schema and migration wip

migrate.py

@@ -9,14 +9,14 @@ def users():
     data = json.loads(open('migration/data/users.json').read())
     newdata = {}
     counter = 0
-    try:
+    #try:
-        for entry in data:
+    for entry in data:
-            oid = entry['_id']
+        oid = entry['_id']
-            newdata[oid] = migrateUser(entry)
+        newdata[oid] = migrateUser(entry)
-            counter += 1
+        counter += 1
-    except Exception:
+    #except Exception:
-        print(str(counter) + '/' + str(len(data)) + ' users entries were migrated')
+    #    print(str(counter) + '/' + str(len(data)) + ' users entries were migrated')
-        print('try to remove database first')
+    #    print('try to remove database first')
     open('migration/data/users.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) )
     print(str(counter) + ' users entries were migrated')
 
@@ -47,7 +47,10 @@ def shouts():
         oid = entry['_id']
         newdata[oid] = migrateShout(entry)
         counter += 1
-        
+        print(str(counter) + ': ' + newdata['slug'])
+        if counter > 9:
+            break
+
     open('migration/data/shouts.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) )
     print(str(counter) + ' shouts were migrated')
 

migration/tables/comments.py

@@ -31,6 +31,6 @@ def migrate(entry):
         'org': 'discours.io',
         'slug': entry['slug'],
         'createdAt': entry['createdAt'],
-        'body': markdown(entry['body']),
+        'body': markdown.feed(entry['body']),
         'replyTo': entry['']
     }

migration/tables/content_item_categories.py

@@ -1,5 +1,5 @@
 def migrate(entry):
-    ```
+    '''
     type Topic {
         slug: String! # ID
         createdBy: Int! # User
@@ -8,12 +8,13 @@ def migrate(entry):
         parents: [String] # NOTE: topic can have parent topics
         children: [String] # and children
     }
-    ```
+    '''
     return {
         'slug': entry['slug'],
         'createdBy': entry['createdBy'], # NOTE: uses an old user id
         'createdAt': entry['createdAt'],
         'value': entry['title'].lower(),
         'parents': [],
-        'children': []
+        'children': [],
+        'old_id': entry['_id']
     }

migration/tables/content_items.py

@@ -1,6 +1,7 @@
 from migration.html2md import Converter
 from dateutil.parser import parse
 from os.path import abspath
+import frontmatter
 import json
 from orm import Shout
 
@@ -70,15 +71,20 @@ def migrate(entry):
     if entry.get('updatedAt') is not None:
         r['updatedAt'] = entry['updatedAt']
     if entry.get('type') == 'Literature':
-        r['body'] = entry['media'][0]['literatureBody']
+        r['body'] = markdown.feed(entry['media'][0]['literatureBody'])
     elif entry.get('type') == 'Video':
         r['body'] = '<ShoutVideo src=\"' + entry['media'][0]['youtubeId'] + '\" />'
     elif entry.get('type') == 'Music':
         r['body'] = '<ShoutMusic media={\"' + json.dumps(entry['media']) +'\"} />'
+    elif entry.get('body') is not None:
+        r['body'] = markdown.feed(entry['body'])
     else:
         r['body'] = '## ' + r['title']
-    # TODO: compile md with graymatter
+    body = r['body']
-    open('migration/content/' + r['slug'] + '.md', 'w').write(mdfile)
+    del r['body']
+    metadata = frontmatter.dumps(r)
+    open('migration/content/' + r['slug'] + '.md', 'w').write(metadata + '\n' + body)
+    r['body'] = body
     shout = Shout.create(**r.copy())
     r['id'] = shout['id']
     return r

migration/tables/users.py

@@ -1,6 +1,8 @@
-from orm import User
+from orm import User, Role
+import frontmatter
 from dateutil.parser import parse
-
+from migration.html2md import Converter
+markdown = Converter()
 counter = 0
 
 def migrate(entry):
@@ -30,46 +32,46 @@ def migrate(entry):
 
         '''
         res = {}
-        try:
+        res['old_id'] = entry['_id']
-            res['old_id'] = entry['_id']
+        res['password'] = entry['services']['password'].get('bcrypt', '')
-            res['password'] = entry['services']['password'].get('bcrypt', '')
+        res['username'] = entry['emails'][0]['address']
-            res['username'] = entry['emails'][0]['address']
+        res['email'] = res['username']
-            res['email'] = res['username']
+        res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt']))
-            res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt']))
+        res['emailConfirmed'] = entry['emails'][0]['verified']
-            res['emailConfirmed'] = entry['emails'][0]['verified']
+        res['createdAt'] = parse(entry['createdAt'])
-            res['createdAt'] = parse(entry['createdAt'])
+        res['rating'] = entry['rating'] # number
-            res['rating'] = entry['rating'] # number
+        res['roles'] = [] # entry['roles'] # roles without org is for discours.io
-            res['roles'] = [] # entry['roles'] # roles without org is for discours.io
+        res['ratings'] = [] # entry['ratings']
-            res['ratings'] = [] # entry['ratings']
+        res['notifications'] = []
-            res['notifications'] = []
+        res['links'] = []
-            res['links'] = []
+        res['muted'] = False
-            res['muted'] = False
+        res['bio'] = markdown.feed(entry.get('bio', ''))
-            res['viewname'] = 'anonymous'
+        if entry['profile']:
-            if entry['profile']:
+            res['slug'] = entry['profile'].get('path')
-                res['slug'] = entry['profile'].get('path')
+            res['userpic'] = entry['profile'].get('image', {'url': ''}).get('url', '')
-                res['userpic'] = entry['profile'].get('image', {'url': ''}).get('url', '')
+            fn = entry['profile'].get('firstName', '')
-                viewname = entry['profile'].get('firstName', '') + ' ' + entry['profile'].get('lastName', '')
+            ln = entry['profile'].get('lastName', '')
-                viewname = entry['profile']['path'] if len(viewname) < 2 else viewname
+            viewname = res['slug'] if res['slug'] else 'anonymous'
-                res['viewname'] = viewname
+            viewname = fn if fn else viewname
-                fb = entry['profile'].get('facebook', False)
+            viewname = (viewname + ' ' + ln) if ln else viewname
-                if fb:
+            viewname = entry['profile']['path'] if len(viewname) < 2 else viewname
-                    res['links'].append(fb)
+            res['viewname'] = viewname
-                vk = entry['profile'].get('vkontakte', False)
+            fb = entry['profile'].get('facebook', False)
-                if vk:
+            if fb:
-                    res['links'].append(vk)
+                res['links'].append(fb)
-                tr = entry['profile'].get('twitter', False)
+            vk = entry['profile'].get('vkontakte', False)
-                if tr:
+            if vk:
-                    res['links'].append(tr)
+                res['links'].append(vk)
-                ws = entry['profile'].get('website', False)
+            tr = entry['profile'].get('twitter', False)
-                if ws:
+            if tr:
-                    res['links'].append(ws)
+                res['links'].append(tr)
-                if not res['slug']:
+            ws = entry['profile'].get('website', False)
-                    res['slug'] = res['links'][0].split('/')[-1]
+            if ws:
+                res['links'].append(ws)
             if not res['slug']:
-                res['slug'] = res['email'].split('@')[0]
+                res['slug'] = res['links'][0].split('/')[-1]
-        except Exception:
+        if not res['slug']:
-            print(entry['profile'])
+            res['slug'] = res['email'].split('@')[0]
-            raise Exception
         else:
             old = res['old_id']
             del res['old_id']

orm/user.py

@@ -38,7 +38,7 @@ class User(Base):
 	userpic: str = Column(String, nullable=True, comment="Userpic")
 	viewname: str = Column(String, nullable=True, comment="Display name")
 	rating: int = Column(Integer, nullable=True, comment="Rating")
-	slug: str = Column(String, unique=True, comment="Author's slug")
+	slug: str = Column(String, unique=True, comment="User's slug")
 	muted: bool = Column(Boolean, default=False)
 	emailConfirmed: bool = Column(Boolean, default=False)
 	createdAt: DateTime = Column(DateTime, nullable=False, comment="Created at")

schema.graphql

@@ -127,7 +127,7 @@ type Role {
 }
 
 type Rating {
-  createdBy: String!
+  createdBy: Int!
   value: Int!
 }