fix comments migration; remove unused columns

migrate.py

@@ -9,6 +9,7 @@ from migration.tables.content_items import get_metadata, migrate as migrateShout
 from migration.tables.content_item_categories import migrate as migrateCategory
 from migration.tables.tags import migrate as migrateTag
 from migration.tables.comments import migrate as migrateComment
+from migration.tables.comments import migrate_2stage as migrateComment_2stage
 from migration.utils import DateTimeEncoder
 from orm import Community, Topic
 from dateutil.parser import parse as date_parse
@@ -206,41 +207,16 @@ def export_slug(slug, export_articles, export_authors, content_dict):
 	export_body(shout, content_dict)
 	comments([slug, ])
 
-def comments(sluglist, export_comments, export_articles, shouts_by_slug, content_dict):
+def comments(comments_data):
-	''' migrating comments on content items one '''
+	id_map = {}
-	if len(sluglist) == 0:
+	for comment in comments_data:
-		export_articles = json.loads(open('../src/data/articles.json').read())
+		comment = migrateComment(comment)
-		print(str(len(export_articles.items())) + ' articles were exported before')
+		id = comment.get('id')
-		if len(sluglist) == 0: sluglist = list(export_articles.keys())
+		old_id = comment.get('old_id')
-
+		id_map[old_id] = id
-	if len(sluglist) > 0:
+	for comment in comments_data:
-		print('exporting comments for: ')
+		migrateComment_2stage(comment, id_map)
-		print(' '.join(sluglist))
+	print(str(len(id_map)) + ' comments exported')
-		for slug in sluglist:
-			shout = shouts_by_slug[slug]
-			old_id = shout['old_id']
-			content_item = content_dict.get(old_id, {})
-			if content_item.get('commentedAt', False):
-				comments = [ migrateComment(c) for c in comments_by_post.get(old_id, []) ]
-				if len(comments) > 0: 
-					export_comments[slug] = comments
-					sys.stdout.write('.')
-	else:
-
-		print('exporting comments for top 10 commented articles...')
-		comments_by_shoutslug = {}
-		for content_item in content_data:
-			old_id = content_item['_id']
-			if content_item.get('commentedAt', False):
-				comments = [ migrateComment(c) for c in comments_by_post.get(old_id, []) ]
-				if len(comments) > 0:
-					shout = shouts_by_oid.get(old_id, { 'slug': 'abandoned-comments' })
-					comments_by_shoutslug[shout['slug']] = comments
-		
-		top = dict(sorted(comments_by_shoutslug.items(), reverse=True, key=lambda c: len(c[1]))[:10])
-		export_comments.update(top)
-			
-		print(str(len(export_comments.keys())) + ' articls with comments exported\n')
 
 
 def export_finish(export_articles = {}, export_authors = {}, export_topics = {}, export_comments = {}):
@@ -342,16 +318,14 @@ if __name__ == '__main__':
 				elif cmd == "shouts":
 					shouts(content_data, shouts_by_slug, shouts_by_oid) # NOTE: listens limit
 				elif cmd == "comments":
-					for comment in comments_data:
+					comments(comments_data)
-						migrateComment(comment)
 				elif cmd == "export_shouts":
 					export_shouts(shouts_by_slug, export_articles, export_authors, content_dict)
 				elif cmd == "all":
 					users(users_by_oid, users_by_slug, users_data)
 					topics(export_topics, topics_by_slug, topics_by_cat, topics_by_tag, cats_data, tags_data)
 					shouts(content_data, shouts_by_slug, shouts_by_oid)
-					for comment in comments_data:
+					comments(comments_data)
-						migrateComment(comment)
 				elif cmd == 'slug':
 					export_slug(sys.argv[2], export_articles, export_authors, content_dict)
 				#export_finish(export_articles, export_authors, export_topics, export_comments)

migration/tables/comments.py

@@ -38,36 +38,29 @@ def migrate(entry):
 		deletedBy: Int
 		ratings: [CommentRating]
 		views: Int
-        old_id: String
-        old_thread: String
 	}
 	'''
 	with local_session() as session:
-        shout = session.query(Shout).filter(Shout.old_id == entry['_id']).first()
+		shout = session.query(Shout).filter(Shout.old_id == entry['contentItem']).first()
 		if not shout: shout = session.query(Shout).first()
-        author = session.query(User).filter(User.old_id == entry['_id']).first() # FIXME
+		author = session.query(User).filter(User.old_id == entry['createdBy']).first()
 		comment_dict = {
-            'old_id': entry['_id'],
 			'author': author.id if author else 0,
 			'createdAt': date_parse(entry['createdAt']),
 			'body': html2text(entry['body']),
 			'shout': shout.id
 		}
-        #TODO save as CommentRating
-        #if 'rating' in entry:
-        #  comment_dict['rating'] = entry['rating']
 		if entry.get('deleted'):
 			comment_dict['deletedAt'] = date_parse(entry['updatedAt'])
 			comment_dict['deletedBy'] = str(entry['updatedBy'])
 		if entry.get('updatedAt'):
 			comment_dict['updatedAt'] = date_parse(entry['updatedAt'])
 		  # comment_dict['updatedBy'] = str(entry.get('updatedBy', 0)) invalid keyword for Comment
-        if 'thread' in entry:
-          comment_dict['old_thread'] = entry['thread']
 		# print(comment_dict)
 		comment = Comment.create(**comment_dict)
 		comment_dict['id'] = comment.id
 		comment_dict['ratings'] = []
+		comment_dict['old_id'] = entry['_id']
 		# print(comment)
 		for comment_rating_old in entry.get('ratings',[]):
 			rater = session.query(User).filter(User.old_id == comment_rating_old['createdBy']).first()
@@ -87,3 +80,15 @@ def migrate(entry):
 					print(comment_rating_dict)
 					raise e
 		return comment_dict
+
+def migrate_2stage(entry, id_map):
+	old_reply_to = entry.get('replyTo')
+	if not old_reply_to:
+		return
+	old_id = entry['_id']
+	id = id_map.get(old_id)
+	with local_session() as session:
+		comment = session.query(Comment).filter(Comment.id == id).first()
+		reply_to = id_map.get(old_reply_to)
+		comment.replyTo = reply_to
+		session.commit()

orm/comment.py

@@ -28,8 +28,5 @@ class Comment(Base):
 	shout: int = Column(ForeignKey("shout.id"), nullable=False, comment="Shout ID")
 	replyTo: int = Column(ForeignKey("comment.id"), nullable=True, comment="comment ID")
 	ratings = relationship(CommentRating, foreign_keys=CommentRating.comment_id)
-	old_id: str = Column(String, nullable = True)
-	old_thread: str = Column(String, nullable = True)
-	
 
 	# TODO: work in progress, udpate this code