comment and migration update

migration/html2text.py

@@ -388,7 +388,7 @@ class HTML2Text(HTMLParser.HTMLParser):
             parent_style = {}
             if start:
                 if self.tag_stack:
-                  parent_style = self.tag_stack[-1][2]
+                    parent_style = self.tag_stack[-1][2]
                 tag_style = element_style(attrs, self.style_def, parent_style)
                 self.tag_stack.append((tag, attrs, tag_style))
             else:
@@ -418,7 +418,7 @@ class HTML2Text(HTMLParser.HTMLParser):
                     elif attrs['class'] == 'lead':
                         self.o('==') # NOTE: but CriticMarkup uses {== ==}
                         self.span_lead = True
-            elif self.:
+            else:
                 if self.span_hightlight:
                     self.o('`')
                     self.span_hightlight = False
@@ -600,7 +600,7 @@ class HTML2Text(HTMLParser.HTMLParser):
         # if self.google_doc:
         # prevent white space immediately after 'begin emphasis' marks ('**' and '_')
         lstripped_data = data.lstrip()
-        if self.drop_white_space and not (self.pre or self.code or self.span_hightlight or self.span_lead):
+        if self.drop_white_space and not (self.pre or self.code):
             data = lstripped_data   
             if puredata: # and not self.pre:
                 data = re.sub('\s+', ' ', data)

migration/tables/comments.py

@@ -1,12 +1,13 @@
 from dateutil.parser import parse as date_parse
 import json
+import datetime
 from os.path import abspath
 from orm import Shout, Comment, CommentRating, User
 from orm.base import local_session
 from migration.html2text import html2text
 
-users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
-topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
+# users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
+# topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
 
 def migrate(entry):
     '''
@@ -39,39 +40,46 @@ def migrate(entry):
         deletedAt: DateTime
         deletedBy: Int
         rating: Int
-        ratigns: [Rating]
+        ratigns: [CommentRating]
         views: Int
         old_id: String
+        old_thread: String
     }
     '''
     with local_session() as session:
-        shout_id = session.query(Shout).filter(Shout.old_id == entry['_id']).first()
-        author_dict = users_dict[entry['createdBy']]
-        print(author_dict)
-        author_id = author_dict['id']
+        shout = session.query(Shout).filter(Shout.old_id == entry['_id']).first()
+        if not shout: print(entry)
+        assert shout, '=== NO SHOUT IN COMMENT ERROR ==='
+        author = session.query(User).filter(User.old_id == entry['_id']).first()
         comment_dict = {
             'old_id': entry['_id'],
-            'author': author_id,
+            'author': author.id if author else 0,
             'createdAt': date_parse(entry['createdAt']),
             'body': html2text(entry['body']),
-            'shout': shout_id
+            'shout': shout
         }
         if 'rating' in entry:
           comment_dict['rating'] = entry['rating']
-        if 'deleted' in entry:
-          comment_dict['deleted'] = entry['deleted']
+        if entry.get('deleted'):
+          comment_dict['deletedAt'] = entry['updatedAt']
+          comment_dict['deletedBy'] = entry['updatedBy']
         if 'thread' in entry:
           comment_dict['old_thread'] = entry['thread']
-        print(entry.keys())
+        # print(entry.keys())
         comment = Comment.create(**comment_dict)
-        
         for comment_rating_old in entry.get('ratings',[]):
             rater_id = session.query(User).filter(User.old_id == comment_rating_old['createdBy']).first()
             comment_rating_dict = {
-                'value': cr['value'],
-                'createdBy': rater_id,
-                'createdAt': date_parse(comment_rating_old['createdAt']) or ts
+                'value': comment_rating_old['value'],
+                'createdBy': rater_id or 0,
+                'createdAt': comment_rating_old.get('createdAt', datetime.datetime.now()),
+                'comment_id': comment.id
             }
-            comment_rating = CommentRating.create(**comment_rating_dict)
-            comment['ratings'].append(comment_rating)
+            try:
+              comment_rating = CommentRating.create(**comment_rating_dict)
+              # TODO: comment rating append resolver
+              # comment['ratings'].append(comment_rating)
+            except Exception as e:
+              print(comment_rating)
+              pass # raise e
         return comment

migration/tables/content_item_categories.py

@@ -9,7 +9,7 @@ def migrate(entry):
         children: [String] # and children
     }
     '''
-    return {
+    topic_dict = {
         'slug': entry['slug'],
         'createdBy': entry['createdBy'], # NOTE: uses an old user id
         'createdAt': entry['createdAt'],
@@ -17,4 +17,11 @@ def migrate(entry):
         'parents': [],
         'children': [],
         'old_id': entry['_id']
-    }
+    }
+    
+    with local_session() as session:
+        topic = session.query(Topic).filter(Topic.slug == topic_slug).first()
+        if not topic:
+            topic = Topic.create(**topic_dict)
+            topic_dict['id'] = topic.id
+        return topic_dict

migration/tables/content_items.py

@@ -184,6 +184,7 @@ def migrate(entry):
     with local_session() as session:
         user = session.query(User).filter(User.slug == slug).first()
     r['authors'].append({
+        'id': user.id,
         'slug': slug,
         'name': name,
         'userpic': userpic
@@ -197,7 +198,7 @@ def migrate(entry):
     if entry['published']:
         ext = 'md'
         open('migration/content/' +
-             r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
+            r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
         try:
             shout_dict = r.copy()
             shout_dict['authors'] = [user, ]
@@ -211,21 +212,6 @@ def migrate(entry):
                 else:
                     shout_dict['publishedAt'] = ts
             del shout_dict['published']
-            
-            # shout comments
-            if entry.get('commentedAt', False):
-                try:
-                    old_comments = comments_by_post.get(shout_dict['old_id'], [])
-                    if len(old_comments) > 0:
-                        shout_dict['comments'] = []
-                        
-                        # migrate comments
-                        for entry in old_comments:
-                            comment = migrateComment(entry)
-                            shout_dict['comments'].append(comment)
-                except KeyError:
-                    print(shout_dict.keys())
-                    raise 'error'
 
             try:
                 topic_slugs = shout_dict['topics']
@@ -248,18 +234,18 @@ def migrate(entry):
                         )
                         shout.ratings.append(shout_rating.id)
                     '''
-                
+                # adding topics to created shout
                 for topic_slug in topic_slugs:
-                    topic_dict = topics_dict.get(topic_slug)
-                    if topic_dict:
-                        topic = Topic.create(**topic_dict)
+                        if not topic: 
+                            topic_dict = topics_dict.get(topic_slug)
+                            if topic_dict:
+                                topic = Topic.create(**topic_dict)
                         shout.topics = [ topic, ]
                         shout.save()
-                    
             except Exception as e:
-              r['error'] = 'db error'
-              # pass
-              raise e
+                r['error'] = 'db error'
+                # pass
+                raise e
         except Exception as e:
             if not r['body']: r['body'] = 'body moved'
             raise e

migration/tables/users.py

@@ -81,8 +81,6 @@ def migrate(entry, limit=668):
       res['slug'] = res['email'].split('@')[0]
   else:
       old = res['old_id']
-      del res['old_id']
       user = User.create(**res.copy())
       res['id'] = user.id
-      res['old_id'] = old
       return res