bookmarked not count, migation fixed

migration/__init__.py

@@ -1,5 +1,6 @@
 ''' cmd managed migration '''
 import csv
+import asyncio
 from datetime import datetime
 import json
 import subprocess
@@ -74,7 +75,7 @@ def topics_handle(storage):
 	return storage
 
 
-def shouts_handle(storage, args):
+async def shouts_handle(storage, args):
 	''' migrating content items one by one '''
 	counter = 0
 	discours_author = 0
@@ -89,7 +90,7 @@ def shouts_handle(storage, args):
 		if '-' in args and slug not in args: continue
 
 		# migrate
-		shout = migrateShout(entry, storage)
+		shout = await migrateShout(entry, storage)
 		storage['shouts']['by_oid'][entry['_id']] = shout
 		storage['shouts']['by_slug'][shout['slug']] = shout
 		# shouts.topics
@@ -123,13 +124,13 @@ def shouts_handle(storage, args):
 	return storage
 
 
-def comments_handle(storage):
+async def comments_handle(storage):
 	id_map = {}
 	ignored_counter = 0
 	missed_shouts = {}
 	for oldcomment in storage['reactions']['data']:
 		if not oldcomment.get('deleted'):
-			reaction = migrateComment(oldcomment, storage)
+			reaction = await migrateComment(oldcomment, storage)
 			if type(reaction) == str:
 				missed_shouts[reaction] = oldcomment
 			elif type(reaction) == Reaction:
@@ -166,11 +167,11 @@ def export_one(slug, storage):
 	export_slug(slug, storage)
 
 
-def all_handle(storage, args):
+async def all_handle(storage, args):
 	print('[migration] handle everything')
 	users_handle(storage)
 	topics_handle(storage)
-	shouts_handle(storage, args)
+	await shouts_handle(storage, args)
 	comments_handle(storage)
 	# export_email_subscriptions()
 	print('[migration] done!')
@@ -213,26 +214,26 @@ def data_load():
 	content_data = []
 	try:
 		users_data = json.loads(open('migration/data/users.json').read())
-		print('[migration] ' + str(len(users_data)) + ' users ')
+		print('[migration.load] ' + str(len(users_data)) + ' users ')
 		tags_data = json.loads(open('migration/data/tags.json').read())
 		storage['topics']['tags'] = tags_data
-		print('[migration] ' + str(len(tags_data)) + ' tags ')
+		print('[migration.load] ' + str(len(tags_data)) + ' tags ')
 		cats_data = json.loads(
 			open('migration/data/content_item_categories.json').read())
 		storage['topics']['cats'] = cats_data
-		print('[migration] ' + str(len(cats_data)) + ' cats ')
+		print('[migration.load] ' + str(len(cats_data)) + ' cats ')
 		comments_data = json.loads(open('migration/data/comments.json').read())
 		storage['reactions']['data'] = comments_data
-		print('[migration] ' + str(len(comments_data)) + ' comments ')
+		print('[migration.load] ' + str(len(comments_data)) + ' comments ')
 		content_data = json.loads(open('migration/data/content_items.json').read())
 		storage['shouts']['data'] = content_data
-		print('[migration] ' + str(len(content_data)) + ' content items ')
+		print('[migration.load] ' + str(len(content_data)) + ' content items ')
 		# fill out storage
 		for x in users_data:
 			storage['users']['by_oid'][x['_id']] = x
 			# storage['users']['by_slug'][x['slug']] = x
 		# no user.slug yet
-		print('[migration] ' + str(len(storage['users']
+		print('[migration.load] ' + str(len(storage['users']
 			  ['by_oid'].keys())) + ' users by oid')
 		for x in tags_data:
 			storage['topics']['by_oid'][x['_id']] = x
@@ -240,20 +241,20 @@ def data_load():
 		for x in cats_data:
 			storage['topics']['by_oid'][x['_id']] = x
 			storage['topics']['by_slug'][x['slug']] = x
-		print('[migration] ' + str(len(storage['topics']
+		print('[migration.load] ' + str(len(storage['topics']
 			  ['by_slug'].keys())) + ' topics by slug')
 		for item in content_data:
 			slug = get_shout_slug(item)
 			storage['content_items']['by_slug'][slug] = item
 			storage['content_items']['by_oid'][item['_id']] = item
-		print('[migration] ' + str(len(content_data)) + ' content items')
+		print('[migration.load] ' + str(len(content_data)) + ' content items')
 		for x in comments_data:
 			storage['reactions']['by_oid'][x['_id']] = x
 			cid = x['contentItem']
 			storage['reactions']['by_content'][cid] = x
 			ci = storage['content_items']['by_oid'].get(cid, {})
 			if 'slug' in ci: storage['reactions']['by_slug'][ci['slug']] = x
-		print('[migration] ' + str(len(storage['reactions']
+		print('[migration.load] ' + str(len(storage['reactions']
 			  ['by_content'].keys())) + ' with comments')
 	except Exception as e: raise e
 	storage['users']['data'] = users_data
@@ -288,29 +289,25 @@ def create_pgdump():
 	])
 
 
-def handle_auto():
+async def handle_auto():
 	print('[migration] no command given, auto mode')
-	mongo_download(os.getenv('MONGODB_URL'))
+	url = os.getenv('MONGODB_URL')
+	if url: mongo_download(url)
 	bson_handle()
-	all_handle(data_load(), sys.argv)
+	await all_handle(data_load(), sys.argv)
 	create_pgdump()
 
-def migrate():
+async def main():
 	if len(sys.argv) > 1:
 		cmd=sys.argv[1]
 		if type(cmd) == str: print('[migration] command: ' + cmd)
-		if cmd == 'mongodb':
+		await handle_auto()
-			mongo_download(sys.argv[2])
-		elif cmd == 'bson':
-			bson_handle()
-		else:
-			storage=data_load()
-			if cmd == '-': export_one(sys.argv[2], storage)
-			else: all_handle(storage, sys.argv)
-	elif len(sys.argv) == 1:
-		handle_auto()
 	else:
-		print('[migration] usage: python migrate.py <command>')
+		print('[migration] usage: python server.py migrate')
-		print('[migration] commands: mongodb, bson, all, all mdx, - <slug>')
 
-if __name__ == '__main__': migrate()
+def migrate():
+	loop = asyncio.get_event_loop()
+	loop.run_until_complete(main())
+ 
+if __name__ == '__main__':
+	migrate()

migration/tables/comments.py

@@ -8,7 +8,7 @@ from services.stat.reacted import ReactedStorage
 
 ts = datetime.now()
 
-def migrate(entry, storage):
+async def migrate(entry, storage):
 	'''
 	{
 	  "_id": "hdtwS8fSyFLxXCgSC",
@@ -72,7 +72,7 @@ def migrate(entry, storage):
 				# creating reaction from old comment
 				day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0)
 				reaction = Reaction.create(**reaction_dict)
-				ReactedStorage.increment(reaction)
+				await ReactedStorage.increment(reaction)
 				
 				reaction_dict['id'] = reaction.id
 				for comment_rating_old in entry.get('ratings',[]):
@@ -89,7 +89,7 @@ def migrate(entry, storage):
 					try:
 						# creating reaction from old rating
 						rr = Reaction.create(**re_reaction_dict)
-						ReactedStorage.increment(rr)
+						await ReactedStorage.increment(rr)
 				
 					except Exception as e:
 						print('[migration] comment rating error: %r' % re_reaction_dict)

migration/tables/content_items.py

@@ -28,7 +28,7 @@ def get_shout_slug(entry):
 			if slug: break
 	return slug
 
-def migrate(entry, storage):
+async def migrate(entry, storage):
 	# init, set title and layout
 	r = {
 		'layout': type2layout[entry['type']],
@@ -218,7 +218,7 @@ def migrate(entry, storage):
 					else:
 						day = (reaction_dict.get('createdAt') or ts).replace(hour=0, minute=0, second=0, microsecond=0)
 						rea = Reaction.create(**reaction_dict)
-						ReactedStorage.increment(rea)
+						await ReactedStorage.increment(rea)
 					# shout_dict['ratings'].append(reaction_dict)
 	except:
 		print('[migration] content_item.ratings error: \n%r' % content_rating)

orm/shout.py

@@ -67,6 +67,5 @@ class Shout(Base):
             "viewed": await ViewedStorage.get_shout(self.slug),
             "reacted": len(await ReactedStorage.get_shout(self.slug)),
             "commented": len(await ReactedStorage.get_comments(self.slug)),
-            "rating": await ReactedStorage.get_rating(self.slug),
+            "rating": await ReactedStorage.get_rating(self.slug)
-            "bookmarked": len(await ReactedStorage.get_bookmarked(self.slug))
 		}

requirements.txt

@@ -16,3 +16,4 @@ transliterate
 requests
 bcrypt
 websockets
+bson

resolvers/reactions.py

@@ -1,4 +1,5 @@
 from sqlalchemy import desc
+from sqlalchemy.orm import joinedload, selectinload
 from orm.reaction import Reaction
 from base.orm import local_session
 from orm.shout import ShoutReactionsFollower
@@ -112,31 +113,31 @@ async def get_shout_reactions(_, info, slug, page, size):
             filter(Reaction.shout == slug).\
             limit(size).offset(offset).all()
     for r in reactions:
-        r.createdBy = await UserStorage.get_user(r.createdBy)
+        r.createdBy = await UserStorage.get_user(r.createdBy or 'discours')
     return reactions
 
 
 @query.field("reactionsAll")
-def get_all_reactions(_, info, page=1, size=10):
+async def get_all_reactions(_, info, page=1, size=10):
     offset = page * size
     reactions = []
     with local_session() as session:
-        # raw sql: statement = text(open('queries/reactions-all.sql', 'r').read()))
+        reactions = session.query(Reaction).\
-        statement = session.query(Reaction).\
             filter(Reaction.deletedAt == None).\
             order_by(desc("createdAt")).\
             offset(offset).limit(size)
-        reactions = []
+        for r in reactions:
-        for row in session.execute(statement):
+            r.createdBy = await UserStorage.get_user(r.createdBy or 'discours')
-            reaction = row.Reaction
+        reactions = list(reactions)
-            reactions.append(reaction)
         reactions.sort(key=lambda x: x.createdAt, reverse=True)
     return reactions
 
 @query.field("reactionsByAuthor")
-def get_reactions_by_author(_, info, slug, page=1, size=50):
+async def get_reactions_by_author(_, info, slug, page=1, size=50):
     offset = page * size
     reactions = []
     with local_session() as session:
-        reactions = session.query(Reaction).filter(Reaction.createdBy == slug).limit(size).offset(offset).all()
+        reactions = session.query(Reaction).filter(Reaction.createdBy == slug).limit(size).offset(offset)
+    for r in reactions:
+        r.createdBy = await UserStorage.get_user(r.createdBy or 'discours')
     return reactions

schema.graphql

@@ -414,7 +414,6 @@ type Stat {
   reacted: Int
   rating: Int
   commented: Int
-  bookmarked: Int
 }
 
 type Community {

services/stat/reacted.py

@@ -1,7 +1,7 @@
 import asyncio
 from datetime import datetime
 from sqlalchemy.types import Enum
-from sqlalchemy import Column, DateTime, ForeignKey
+from sqlalchemy import Column, DateTime, ForeignKey, Boolean
 # from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy import Enum
 import enum
@@ -37,6 +37,7 @@ def kind_to_rate(kind) -> int:
 		ReactionKind.REJECT
 	]: return -1
 	else: return 0
+ 
 class ReactedByDay(Base):
 	__tablename__ = "reacted_by_day"
 
@@ -46,6 +47,8 @@ class ReactedByDay(Base):
 	replyTo = Column(ForeignKey('reaction.id'), nullable=True)
 	kind: int = Column(Enum(ReactionKind), nullable=False, comment="Reaction kind")
 	day = Column(DateTime, primary_key=True, default=datetime.now)
+	comment = Column(Boolean, nullable=True)
+	
 
 class ReactedStorage:
 	reacted = {
@@ -63,34 +66,6 @@ class ReactedStorage:
 	period = 30*60  # sec
 	lock = asyncio.Lock()
 
-	@staticmethod
-	def init(session):
-		self = ReactedStorage
-		all_reactions = session.query(ReactedByDay).all()
-		print('[stat.reacted] %d reactions total' % len(all_reactions))
-		for reaction in all_reactions:
-			shout = reaction.shout
-			topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all()
-			kind = reaction.kind
-   
-			self.reacted['shouts'][shout] = self.reacted['shouts'].get(shout, [])
-			self.reacted['shouts'][shout].append(reaction)
-			self.rating['shouts'][shout] = self.rating['shouts'].get(shout, 0) + kind_to_rate(kind)
-   
-			for t in topics:
-				self.reacted['topics'][t] = self.reacted['topics'].get(t, [])
-				self.reacted['topics'][t].append(reaction)
-				self.rating['topics'][t] = self.rating['topics'].get(t, 0) + kind_to_rate(kind) # rating
-	
-			if reaction.replyTo:
-				self.reacted['reactions'][reaction.replyTo] = self.reacted['reactions'].get(reaction.replyTo, [])
-				self.reacted['reactions'][reaction.replyTo].append(reaction)
-				self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
-		ttt = self.reacted['topics'].values()
-		print('[stat.reacted] %d topics reacted' % len(ttt))
-		print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts']))
-		print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions']))
-
 	@staticmethod
 	async def get_shout(shout_slug):
 		self = ReactedStorage
@@ -103,6 +78,24 @@ class ReactedStorage:
 		async with self.lock:
 			return self.reacted['topics'].get(topic_slug, [])
 
+	@staticmethod
+	async def get_comments(shout_slug):
+		self = ReactedStorage
+		async with self.lock:
+			return list(filter(lambda r: r.comment, self.reacted['shouts'].get(shout_slug, [])))
+
+	@staticmethod
+	async def get_topic_comments(topic_slug):
+		self = ReactedStorage
+		async with self.lock:
+			return list(filter(lambda r: r.comment, self.reacted['topics'].get(topic_slug, [])))
+
+	@staticmethod
+	async def get_reaction_comments(reaction_id):
+		self = ReactedStorage
+		async with self.lock:
+			return list(filter(lambda r: r.comment, self.reacted['reactions'].get(reaction_id)))
+
 	@staticmethod
 	async def get_reaction(reaction_id):
 		self = ReactedStorage
@@ -137,17 +130,52 @@ class ReactedStorage:
 		return rating
 
 	@staticmethod
-	async def increment(shout_slug, kind, reply_id = None):
+	async def increment(reaction):
 		self = ReactedStorage
-		reaction: ReactedByDay = None
 		async with self.lock:
 			with local_session() as session:
-				reaction = ReactedByDay.create(shout=shout_slug, kind=kind, reply=reply_id)
+				r = {
-				self.reacted['shouts'][shout_slug] = self.reacted['shouts'].get(shout_slug, [])
+					"day": datetime.now().replace(hour=0, minute=0, second=0, microsecond=0),
-				self.reacted['shouts'][shout_slug].append(reaction)
+					"reaction": reaction.id,
-				if reply_id:
+					"kind": reaction.kind,
-					self.reacted['reaction'][reply_id] = self.reacted['reactions'].get(shout_slug, [])
+					"shout": reaction.shout
-					self.reacted['reaction'][reply_id].append(reaction)
+				}
-					self.rating['reactions'][reply_id] = self.rating['reactions'].get(reply_id, 0) + kind_to_rate(kind)
+				if reaction.replyTo: r['replyTo'] = reaction.replyTo
+				if reaction.body: r['comment'] = True
+				reaction = ReactedByDay.create(**r)
+				self.reacted['shouts'][reaction.shout] = self.reacted['shouts'].get(reaction.shout, [])
+				self.reacted['shouts'][reaction.shout].append(reaction)
+				if reaction.replyTo:
+					self.reacted['reaction'][reaction.replyTo] = self.reacted['reactions'].get(reaction.shout, [])
+					self.reacted['reaction'][reaction.replyTo].append(reaction)
+					self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
 				else:
-					self.rating['shouts'][shout_slug] = self.rating['shouts'].get(shout_slug, 0) + kind_to_rate(kind)
+					self.rating['shouts'][reaction.replyTo] = self.rating['shouts'].get(reaction.shout, 0) + kind_to_rate(reaction.kind)
+
+	@staticmethod
+	def init(session):
+		self = ReactedStorage
+		all_reactions = session.query(ReactedByDay).all()
+		print('[stat.reacted] %d reactions total' % len(all_reactions))
+		for reaction in all_reactions:
+			shout = reaction.shout
+			topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all()
+			kind = reaction.kind
+   
+			self.reacted['shouts'][shout] = self.reacted['shouts'].get(shout, [])
+			self.reacted['shouts'][shout].append(reaction)
+			self.rating['shouts'][shout] = self.rating['shouts'].get(shout, 0) + kind_to_rate(kind)
+   
+			for t in topics:
+				self.reacted['topics'][t] = self.reacted['topics'].get(t, [])
+				self.reacted['topics'][t].append(reaction)
+				self.rating['topics'][t] = self.rating['topics'].get(t, 0) + kind_to_rate(kind) # rating
+	
+			if reaction.replyTo:
+				self.reacted['reactions'][reaction.replyTo] = self.reacted['reactions'].get(reaction.replyTo, [])
+				self.reacted['reactions'][reaction.replyTo].append(reaction)
+				self.rating['reactions'][reaction.replyTo] = self.rating['reactions'].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
+		ttt = self.reacted['topics'].values()
+		print('[stat.reacted] %d topics reacted' % len(ttt))
+		print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts']))
+		print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions']))

services/stat/topicstat.py

@@ -67,7 +67,6 @@ class TopicStat:
 			"viewed": await ViewedStorage.get_topic(topic),
 			"reacted" : len(await ReactedStorage.get_topic(topic)),
 			"commented": len(await ReactedStorage.get_topic_comments(topic)),
-			"bookmarked": len(await ReactedStorage.get_topic_bookmarked(topic)),
 			"rating" : await ReactedStorage.get_topic_rating(topic),
 		}