working-on

services/zine/gittask.py

@@ -0,0 +1,62 @@
+import subprocess
+from pathlib import Path
+import asyncio
+from settings import SHOUTS_REPO
+
+class GitTask:
+	''' every shout update use a new task '''
+	queue = asyncio.Queue()
+
+	def __init__(self, input, username, user_email, comment):
+		self.slug = input["slug"]
+		self.shout_body = input["body"]
+		self.username = username
+		self.user_email = user_email
+		self.comment = comment
+
+		GitTask.queue.put_nowait(self)
+	
+	def init_repo(self):
+		repo_path = "%s" % (SHOUTS_REPO)
+		
+		Path(repo_path).mkdir()
+		
+		cmd = "cd %s && git init && " \
+			"git config user.name 'discours' && " \
+			"git config user.email 'discours@discours.io' && " \
+			"touch initial && git add initial && " \
+			"git commit -m 'init repo'" \
+			% (repo_path)
+		output = subprocess.check_output(cmd, shell=True)
+		print(output)
+
+	def execute(self):
+		repo_path = "%s" % (SHOUTS_REPO)
+		
+		if not Path(repo_path).exists():
+			self.init_repo()
+
+		#cmd = "cd %s && git checkout master" % (repo_path)
+		#output = subprocess.check_output(cmd, shell=True)
+		#print(output)
+
+		shout_filename = "%s.mdx" % (self.slug)
+		shout_full_filename = "%s/%s" % (repo_path, shout_filename)
+		with open(shout_full_filename, mode='w', encoding='utf-8') as shout_file:
+			shout_file.write(bytes(self.shout_body,'utf-8').decode('utf-8','ignore'))
+
+		author = "%s <%s>" % (self.username, self.user_email)
+		cmd = "cd %s && git add %s && git commit -m '%s' --author='%s'" % \
+			(repo_path, shout_filename, self.comment, author)
+		output = subprocess.check_output(cmd, shell=True)
+		print(output)
+	
+	@staticmethod
+	async def git_task_worker():
+		print("[resolvers.git] worker start")
+		while True:
+			task = await GitTask.queue.get()
+			try:
+				task.execute()
+			except Exception as err:
+				print("[resolvers.git] worker error: %s" % (err))

services/zine/reactions.py

@@ -0,0 +1,159 @@
+import asyncio
+from sqlalchemy import and_, desc, func
+from sqlalchemy.orm import joinedload
+from base.orm import local_session
+from orm.reaction import Reaction, ReactionKind
+from orm.topic import ShoutTopic
+
+
+def kind_to_rate(kind) -> int:
+	if kind in [
+		ReactionKind.AGREE,
+		ReactionKind.LIKE,
+		ReactionKind.PROOF,
+		ReactionKind.ACCEPT
+	]: return 1
+	elif kind in [
+		ReactionKind.DISAGREE,
+		ReactionKind.DISLIKE,
+		ReactionKind.DISPROOF,
+		ReactionKind.REJECT
+	]: return -1
+	else: return 0
+
+class ReactionsStorage:
+	limit = 200
+	reactions = []
+	rating_by_shout = {}
+	reactions_by_shout = {}
+	reactions_by_topic = {}  # TODO: get sum reactions for all shouts in topic
+	reactions_by_author = {}
+	lock = asyncio.Lock()
+	period = 3*60  # 3 mins
+
+	@staticmethod
+	async def prepare_all(session):
+		stmt = session.query(Reaction).\
+			filter(Reaction.deletedAt == None).\
+			order_by(desc("createdAt")).\
+			limit(ReactionsStorage.limit)
+		reactions = []
+		for row in session.execute(stmt):
+			reaction = row.Reaction
+			reactions.append(reaction)
+		reactions.sort(key=lambda x: x.createdAt, reverse=True)
+		async with ReactionsStorage.lock:
+			print("[service.reactions] %d recently published reactions " % len(reactions))
+			ReactionsStorage.reactions = reactions
+
+	@staticmethod
+	async def prepare_by_author(session):
+		try:
+			by_authors = session.query(Reaction.createdBy, func.count('*').label("count")).\
+				where(and_(Reaction.deletedAt == None)).\
+				group_by(Reaction.createdBy).all()
+		except Exception as e:
+			print(e)
+			by_authors = {}
+		async with ReactionsStorage.lock:
+			ReactionsStorage.reactions_by_author = dict([stat for stat in by_authors])
+			print("[service.reactions] %d reacted users" % len(by_authors))
+
+	@staticmethod
+	async def prepare_by_shout(session):
+		try:
+			by_shouts = session.query(Reaction.shout, func.count('*').label("count")).\
+				where(and_(Reaction.deletedAt == None)).\
+				group_by(Reaction.shout).all()
+		except Exception as e:
+			print(e)
+			by_shouts = {}
+		async with ReactionsStorage.lock:
+			ReactionsStorage.reactions_by_shout = dict([stat for stat in by_shouts])
+			print("[service.reactions] %d reacted shouts" % len(by_shouts))
+
+	@staticmethod
+	async def calc_ratings(session):
+		rating_by_shout = {}
+		for shout in ReactionsStorage.reactions_by_shout.keys():
+			rating_by_shout[shout] = 0
+			shout_reactions_by_kinds = session.query(Reaction).\
+				where(and_(Reaction.deletedAt == None, Reaction.shout == shout)).\
+				group_by(Reaction.kind, Reaction.id).all()
+			for reaction in shout_reactions_by_kinds:
+				rating_by_shout[shout] +=  kind_to_rate(reaction.kind)
+		async with ReactionsStorage.lock:
+			ReactionsStorage.rating_by_shout = rating_by_shout
+
+	@staticmethod
+	async def prepare_by_topic(session):
+		# TODO: optimize
+		by_topics = session.query(Reaction, func.count('*').label("count")).\
+			options(
+				joinedload(ShoutTopic),
+				joinedload(Reaction.shout)
+			).\
+			join(ShoutTopic, ShoutTopic.shout == Reaction.shout).\
+			filter(Reaction.deletedAt == None).\
+			group_by(ShoutTopic.topic).all()
+		reactions_by_topic = {}
+		for t, reactions in by_topics:
+			if not reactions_by_topic.get(t):
+				reactions_by_topic[t] = 0
+			for r in reactions:
+				reactions_by_topic[t] += r.count
+		async with ReactionsStorage.lock:
+			ReactionsStorage.reactions_by_topic = reactions_by_topic
+
+	@staticmethod
+	async def recent():
+		async with ReactionsStorage.lock:
+			return ReactionsStorage.reactions
+
+	@staticmethod
+	async def total():
+		async with ReactionsStorage.lock:
+			return len(ReactionsStorage.reactions)
+
+	@staticmethod
+	async def by_shout(shout):
+		async with ReactionsStorage.lock:
+			stat = ReactionsStorage.reactions_by_shout.get(shout)
+			stat = stat if stat else 0
+			return stat
+	
+	@staticmethod
+	async def shout_rating(shout):
+		async with ReactionsStorage.lock:
+			return ReactionsStorage.rating_by_shout.get(shout)
+
+	@staticmethod
+	async def by_author(slug):
+		async with ReactionsStorage.lock:
+			stat = ReactionsStorage.reactions_by_author.get(slug)
+			stat = stat if stat else 0
+			return stat
+
+	@staticmethod
+	async def by_topic(topic):
+		async with ReactionsStorage.lock:
+			stat = ReactionsStorage.reactions_by_topic.get(topic)
+			stat = stat if stat else 0
+			return stat
+
+	@staticmethod
+	async def worker():
+		while True:
+			try:
+				with local_session() as session:
+					await ReactionsStorage.prepare_all(session)
+					print("[service.reactions] all reactions prepared")
+					await ReactionsStorage.prepare_by_shout(session)
+					print("[service.reactions] reactions by shouts prepared")
+					await ReactionsStorage.calc_ratings(session)
+					print("[service.reactions] reactions ratings prepared")
+					await ReactionsStorage.prepare_by_topic(session)
+					print("[service.reactions] reactions topics prepared")
+			except Exception as err:
+				print("[service.reactions] errror: %s" % (err))
+			await asyncio.sleep(ReactionsStorage.period)

services/zine/shoutauthor.py

@@ -0,0 +1,42 @@
+
+import asyncio
+from base.orm import local_session
+from orm.shout import ShoutAuthor
+
+
+class ShoutAuthorStorage:
+	authors_by_shout = {}
+	lock = asyncio.Lock()
+	period = 30*60 #sec
+
+	@staticmethod
+	async def load(session):
+		self = ShoutAuthorStorage
+		authors = session.query(ShoutAuthor).all()
+		for author in authors:
+			user = author.user
+			shout = author.shout
+			if shout in self.authors_by_shout:
+				self.authors_by_shout[shout].append(user)
+			else:
+				self.authors_by_shout[shout] = [user]
+		print('[service.shoutauthor] %d shouts ' % len(self.authors_by_shout))
+
+	@staticmethod
+	async def get_authors(shout):
+		self = ShoutAuthorStorage
+		async with self.lock:
+			return self.authors_by_shout.get(shout, [])
+
+	@staticmethod
+	async def worker():
+		self = ShoutAuthorStorage
+		while True:
+			try:
+				with local_session() as session:
+					async with self.lock:
+						await self.load(session)
+						print("[service.shoutauthor] updated")
+			except Exception as err:
+				print("[service.shoutauthor] errror: %s" % (err))
+			await asyncio.sleep(self.period)

services/zine/shoutscache.py

@@ -0,0 +1,153 @@
+
+import asyncio
+from datetime import datetime, timedelta
+from sqlalchemy import and_, desc, func, select
+from sqlalchemy.orm import selectinload
+from base.orm import local_session
+from orm.reaction import Reaction
+from orm.shout import Shout
+from services.zine.reactions import ReactionsStorage
+from services.stat.viewed import ViewedByDay
+
+
+class ShoutsCache:
+	limit = 200
+	period = 60*60 #1 hour
+	lock = asyncio.Lock()
+
+	@staticmethod
+	async def prepare_recent_published():
+		with local_session() as session:
+			stmt = select(Shout).\
+				options(selectinload(Shout.authors), selectinload(Shout.topics)).\
+				where(Shout.publishedAt != None).\
+				order_by(desc("publishedAt")).\
+				limit(ShoutsCache.limit)
+			shouts = []
+			for row in session.execute(stmt):
+				shout = row.Shout
+				shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
+				shouts.append(shout)
+		async with ShoutsCache.lock:
+			ShoutsCache.recent_published = shouts
+			print("[service.shoutscache] %d recently published shouts " % len(shouts))
+
+	@staticmethod
+	async def prepare_recent_all():
+		with local_session() as session:
+			stmt = select(Shout).\
+				options(selectinload(Shout.authors), selectinload(Shout.topics)).\
+				order_by(desc("createdAt")).\
+				limit(ShoutsCache.limit)
+			shouts = []
+			for row in session.execute(stmt):
+				shout = row.Shout
+				shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
+				shouts.append(shout)
+		async with ShoutsCache.lock:
+			ShoutsCache.recent_all = shouts
+			print("[service.shoutscache] %d recently created shouts " % len(shouts))
+
+	@staticmethod
+	async def prepare_recent_reacted():
+		with local_session() as session:
+			stmt = select(Shout, func.max(Reaction.createdAt).label("reactionCreatedAt")).\
+				options(
+					selectinload(Shout.authors),
+					selectinload(Shout.topics),
+				).\
+				join(Reaction, Reaction.shout == Shout.slug).\
+				where(and_(Shout.publishedAt != None, Reaction.deletedAt == None)).\
+				group_by(Shout.slug).\
+				order_by(desc("reactionCreatedAt")).\
+				limit(ShoutsCache.limit)
+			shouts = []
+			for row in session.execute(stmt):
+				shout = row.Shout
+				shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
+				shouts.append(shout)
+			async with ShoutsCache.lock:
+				ShoutsCache.recent_reacted = shouts
+				print("[service.shoutscache] %d recently reacted shouts " % len(shouts))
+
+
+	@staticmethod
+	async def prepare_top_overall():
+		with local_session() as session:
+			# with reacted times counter
+			stmt = select(Shout, 
+				func.count(Reaction.id).label("reacted")).\
+				options(selectinload(Shout.authors), selectinload(Shout.topics), selectinload(Shout.reactions)).\
+				join(Reaction).\
+				where(and_(Shout.publishedAt != None, Reaction.deletedAt == None)).\
+				group_by(Shout.slug).\
+				order_by(desc("reacted")).\
+				limit(ShoutsCache.limit)
+			shouts = []
+			# with rating synthetic counter
+			for row in session.execute(stmt):
+				shout = row.Shout
+				shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
+				shouts.append(shout)
+			shouts.sort(key = lambda shout: shout.rating, reverse = True)
+			async with ShoutsCache.lock:
+				print("[service.shoutscache] %d top shouts " % len(shouts))
+				ShoutsCache.top_overall = shouts
+
+	@staticmethod
+	async def prepare_top_month():
+		month_ago = datetime.now() - timedelta(days = 30)
+		with local_session() as session:
+			stmt = select(Shout, func.count(Reaction.id).label("reacted")).\
+				options(selectinload(Shout.authors), selectinload(Shout.topics)).\
+				join(Reaction).\
+				where(and_(Shout.createdAt > month_ago, Shout.publishedAt != None)).\
+				group_by(Shout.slug).\
+				order_by(desc("reacted")).\
+				limit(ShoutsCache.limit)
+			shouts = []
+			for row in session.execute(stmt):
+				shout = row.Shout
+				shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
+				shouts.append(shout)
+			shouts.sort(key = lambda shout: shout.rating, reverse = True)
+			async with ShoutsCache.lock:
+				print("[service.shoutscache] %d top month shouts " % len(shouts))
+				ShoutsCache.top_month = shouts
+
+	@staticmethod
+	async def prepare_top_viewed():
+		month_ago = datetime.now() - timedelta(days = 30)
+		with local_session() as session:
+			stmt = select(Shout, func.sum(ViewedByDay.value).label("viewed")).\
+				options(selectinload(Shout.authors), selectinload(Shout.topics)).\
+				join(ViewedByDay).\
+				where(and_(ViewedByDay.day > month_ago, Shout.publishedAt != None)).\
+				group_by(Shout.slug).\
+				order_by(desc("viewed")).\
+				limit(ShoutsCache.limit)
+			shouts = []
+			for row in session.execute(stmt):
+				shout = row.Shout
+				shout.rating = await ReactionsStorage.shout_rating(shout.slug) or 0
+				shouts.append(shout)
+		# shouts.sort(key = lambda shout: shout.viewed, reverse = True)
+		async with ShoutsCache.lock:
+			print("[service.shoutscache] %d top viewed shouts " % len(shouts))
+			ShoutsCache.top_viewed = shouts
+
+	@staticmethod
+	async def worker():
+		while True:
+			try:
+				await ShoutsCache.prepare_top_month()
+				await ShoutsCache.prepare_top_overall()
+				await ShoutsCache.prepare_top_viewed()
+				await ShoutsCache.prepare_recent_published()
+				await ShoutsCache.prepare_recent_all()
+				await ShoutsCache.prepare_recent_reacted()
+				print("[service.shoutscache] updated")
+			except Exception as err:
+				print("[service.shoutscache] error: %s" % (err))
+				raise err
+			await asyncio.sleep(ShoutsCache.period)

services/zine/topics.py

@@ -0,0 +1,56 @@
+import asyncio
+from orm.topic import Topic
+
+
+class TopicStorage:
+	topics = {}
+	lock = asyncio.Lock()
+
+	@staticmethod
+	def init(session):
+		self = TopicStorage
+		topics = session.query(Topic)
+		self.topics = dict([(topic.slug, topic) for topic in topics])
+		for topic in self.topics.values():
+			self.load_parents(topic) # TODO: test
+		
+		print('[service.topics] %d ' % len(self.topics.keys()))
+
+	@staticmethod
+	def load_parents(topic):
+		self = TopicStorage
+		parents = []
+		for parent in self.topics.values():
+			if topic.slug in parent.children:
+				parents.append(parent.slug)
+		topic.parents = parents
+		return topic
+
+	@staticmethod
+	async def get_topics_all():
+		self = TopicStorage
+		async with self.lock:
+			return self.topics.values()
+
+	@staticmethod
+	async def get_topics_by_slugs(slugs):
+		self = TopicStorage
+		async with self.lock:
+			if not slugs:
+				return self.topics.values()
+			topics = filter(lambda topic: topic.slug in slugs, self.topics.values())
+			return list(topics)
+
+	@staticmethod
+	async def get_topics_by_community(community):
+		self = TopicStorage
+		async with self.lock:
+			topics = filter(lambda topic: topic.community == community, self.topics.values())
+			return list(topics)
+
+	@staticmethod
+	async def add_topic(topic):
+		self = TopicStorage
+		async with self.lock:
+			self.topics[topic.slug] = topic
+			self.load_parents(topic)