schema and migration wip

migrate.py

@@ -9,14 +9,14 @@ def users():
     data = json.loads(open('migration/data/users.json').read())
     newdata = {}
     counter = 0
-    try:
+    #try:
     for entry in data:
         oid = entry['_id']
         newdata[oid] = migrateUser(entry)
         counter += 1
-    except Exception:
+    #except Exception:
-        print(str(counter) + '/' + str(len(data)) + ' users entries were migrated')
+    #    print(str(counter) + '/' + str(len(data)) + ' users entries were migrated')
-        print('try to remove database first')
+    #    print('try to remove database first')
     open('migration/data/users.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) )
     print(str(counter) + ' users entries were migrated')
 
@@ -47,6 +47,9 @@ def shouts():
         oid = entry['_id']
         newdata[oid] = migrateShout(entry)
         counter += 1
+        print(str(counter) + ': ' + newdata['slug'])
+        if counter > 9:
+            break
 
     open('migration/data/shouts.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) )
     print(str(counter) + ' shouts were migrated')

migration/tables/comments.py

@@ -31,6 +31,6 @@ def migrate(entry):
         'org': 'discours.io',
         'slug': entry['slug'],
         'createdAt': entry['createdAt'],
-        'body': markdown(entry['body']),
+        'body': markdown.feed(entry['body']),
         'replyTo': entry['']
     }

migration/tables/content_item_categories.py

@@ -1,5 +1,5 @@
 def migrate(entry):
-    ```
+    '''
     type Topic {
         slug: String! # ID
         createdBy: Int! # User
@@ -8,12 +8,13 @@ def migrate(entry):
         parents: [String] # NOTE: topic can have parent topics
         children: [String] # and children
     }
-    ```
+    '''
     return {
         'slug': entry['slug'],
         'createdBy': entry['createdBy'], # NOTE: uses an old user id
         'createdAt': entry['createdAt'],
         'value': entry['title'].lower(),
         'parents': [],
-        'children': []
+        'children': [],
+        'old_id': entry['_id']
     }

migration/tables/content_items.py

@@ -1,6 +1,7 @@
 from migration.html2md import Converter
 from dateutil.parser import parse
 from os.path import abspath
+import frontmatter
 import json
 from orm import Shout
 
@@ -70,15 +71,20 @@ def migrate(entry):
     if entry.get('updatedAt') is not None:
         r['updatedAt'] = entry['updatedAt']
     if entry.get('type') == 'Literature':
-        r['body'] = entry['media'][0]['literatureBody']
+        r['body'] = markdown.feed(entry['media'][0]['literatureBody'])
     elif entry.get('type') == 'Video':
         r['body'] = '<ShoutVideo src=\"' + entry['media'][0]['youtubeId'] + '\" />'
     elif entry.get('type') == 'Music':
         r['body'] = '<ShoutMusic media={\"' + json.dumps(entry['media']) +'\"} />'
+    elif entry.get('body') is not None:
+        r['body'] = markdown.feed(entry['body'])
     else:
         r['body'] = '## ' + r['title']
-    # TODO: compile md with graymatter
+    body = r['body']
-    open('migration/content/' + r['slug'] + '.md', 'w').write(mdfile)
+    del r['body']
+    metadata = frontmatter.dumps(r)
+    open('migration/content/' + r['slug'] + '.md', 'w').write(metadata + '\n' + body)
+    r['body'] = body
     shout = Shout.create(**r.copy())
     r['id'] = shout['id']
     return r

migration/tables/users.py

@@ -1,6 +1,8 @@
-from orm import User
+from orm import User, Role
+import frontmatter
 from dateutil.parser import parse
-
+from migration.html2md import Converter
+markdown = Converter()
 counter = 0
 
 def migrate(entry):
@@ -30,7 +32,6 @@ def migrate(entry):
 
         '''
         res = {}
-        try:
         res['old_id'] = entry['_id']
         res['password'] = entry['services']['password'].get('bcrypt', '')
         res['username'] = entry['emails'][0]['address']
@@ -44,11 +45,15 @@ def migrate(entry):
         res['notifications'] = []
         res['links'] = []
         res['muted'] = False
-            res['viewname'] = 'anonymous'
+        res['bio'] = markdown.feed(entry.get('bio', ''))
         if entry['profile']:
             res['slug'] = entry['profile'].get('path')
             res['userpic'] = entry['profile'].get('image', {'url': ''}).get('url', '')
-                viewname = entry['profile'].get('firstName', '') + ' ' + entry['profile'].get('lastName', '')
+            fn = entry['profile'].get('firstName', '')
+            ln = entry['profile'].get('lastName', '')
+            viewname = res['slug'] if res['slug'] else 'anonymous'
+            viewname = fn if fn else viewname
+            viewname = (viewname + ' ' + ln) if ln else viewname
             viewname = entry['profile']['path'] if len(viewname) < 2 else viewname
             res['viewname'] = viewname
             fb = entry['profile'].get('facebook', False)
@@ -67,9 +72,6 @@ def migrate(entry):
                 res['slug'] = res['links'][0].split('/')[-1]
         if not res['slug']:
             res['slug'] = res['email'].split('@')[0]
-        except Exception:
-            print(entry['profile'])
-            raise Exception
         else:
             old = res['old_id']
             del res['old_id']

orm/user.py

@@ -38,7 +38,7 @@ class User(Base):
 	userpic: str = Column(String, nullable=True, comment="Userpic")
 	viewname: str = Column(String, nullable=True, comment="Display name")
 	rating: int = Column(Integer, nullable=True, comment="Rating")
-	slug: str = Column(String, unique=True, comment="Author's slug")
+	slug: str = Column(String, unique=True, comment="User's slug")
 	muted: bool = Column(Boolean, default=False)
 	emailConfirmed: bool = Column(Boolean, default=False)
 	createdAt: DateTime = Column(DateTime, nullable=False, comment="Created at")

schema.graphql

@@ -127,7 +127,7 @@ type Role {
 }
 
 type Rating {
-  createdBy: String!
+  createdBy: Int!
   value: Int!
 }