From 892c929c35b8465cb158f5a161f0cae9604a468d Mon Sep 17 00:00:00 2001 From: Untone Date: Fri, 20 Aug 2021 18:10:15 +0300 Subject: [PATCH] schema and migration wip --- migrate.py | 21 +++--- migration/tables/comments.py | 2 +- migration/tables/content_item_categories.py | 7 +- migration/tables/content_items.py | 12 ++- migration/tables/users.py | 84 +++++++++++---------- orm/user.py | 2 +- schema.graphql | 2 +- 7 files changed, 71 insertions(+), 59 deletions(-) diff --git a/migrate.py b/migrate.py index b98eb9cc..67581f9d 100644 --- a/migrate.py +++ b/migrate.py @@ -9,14 +9,14 @@ def users(): data = json.loads(open('migration/data/users.json').read()) newdata = {} counter = 0 - try: - for entry in data: - oid = entry['_id'] - newdata[oid] = migrateUser(entry) - counter += 1 - except Exception: - print(str(counter) + '/' + str(len(data)) + ' users entries were migrated') - print('try to remove database first') + #try: + for entry in data: + oid = entry['_id'] + newdata[oid] = migrateUser(entry) + counter += 1 + #except Exception: + # print(str(counter) + '/' + str(len(data)) + ' users entries were migrated') + # print('try to remove database first') open('migration/data/users.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) ) print(str(counter) + ' users entries were migrated') @@ -47,7 +47,10 @@ def shouts(): oid = entry['_id'] newdata[oid] = migrateShout(entry) counter += 1 - + print(str(counter) + ': ' + newdata['slug']) + if counter > 9: + break + open('migration/data/shouts.dict.json','w').write( json.dumps(newdata, cls=DateTimeEncoder) ) print(str(counter) + ' shouts were migrated') diff --git a/migration/tables/comments.py b/migration/tables/comments.py index a1aa85ad..1eb92621 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -31,6 +31,6 @@ def migrate(entry): 'org': 'discours.io', 'slug': entry['slug'], 'createdAt': entry['createdAt'], - 'body': markdown(entry['body']), + 'body': markdown.feed(entry['body']), 'replyTo': entry[''] } \ No newline at end of file diff --git a/migration/tables/content_item_categories.py b/migration/tables/content_item_categories.py index 47b9383f..ffb1db5a 100644 --- a/migration/tables/content_item_categories.py +++ b/migration/tables/content_item_categories.py @@ -1,5 +1,5 @@ def migrate(entry): - ``` + ''' type Topic { slug: String! # ID createdBy: Int! # User @@ -8,12 +8,13 @@ def migrate(entry): parents: [String] # NOTE: topic can have parent topics children: [String] # and children } - ``` + ''' return { 'slug': entry['slug'], 'createdBy': entry['createdBy'], # NOTE: uses an old user id 'createdAt': entry['createdAt'], 'value': entry['title'].lower(), 'parents': [], - 'children': [] + 'children': [], + 'old_id': entry['_id'] } \ No newline at end of file diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index a352b0b9..8c581f49 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -1,6 +1,7 @@ from migration.html2md import Converter from dateutil.parser import parse from os.path import abspath +import frontmatter import json from orm import Shout @@ -70,15 +71,20 @@ def migrate(entry): if entry.get('updatedAt') is not None: r['updatedAt'] = entry['updatedAt'] if entry.get('type') == 'Literature': - r['body'] = entry['media'][0]['literatureBody'] + r['body'] = markdown.feed(entry['media'][0]['literatureBody']) elif entry.get('type') == 'Video': r['body'] = '' elif entry.get('type') == 'Music': r['body'] = '' + elif entry.get('body') is not None: + r['body'] = markdown.feed(entry['body']) else: r['body'] = '## ' + r['title'] - # TODO: compile md with graymatter - open('migration/content/' + r['slug'] + '.md', 'w').write(mdfile) + body = r['body'] + del r['body'] + metadata = frontmatter.dumps(r) + open('migration/content/' + r['slug'] + '.md', 'w').write(metadata + '\n' + body) + r['body'] = body shout = Shout.create(**r.copy()) r['id'] = shout['id'] return r diff --git a/migration/tables/users.py b/migration/tables/users.py index 6e8f42dd..2eac49e7 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -1,6 +1,8 @@ -from orm import User +from orm import User, Role +import frontmatter from dateutil.parser import parse - +from migration.html2md import Converter +markdown = Converter() counter = 0 def migrate(entry): @@ -30,46 +32,46 @@ def migrate(entry): ''' res = {} - try: - res['old_id'] = entry['_id'] - res['password'] = entry['services']['password'].get('bcrypt', '') - res['username'] = entry['emails'][0]['address'] - res['email'] = res['username'] - res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt'])) - res['emailConfirmed'] = entry['emails'][0]['verified'] - res['createdAt'] = parse(entry['createdAt']) - res['rating'] = entry['rating'] # number - res['roles'] = [] # entry['roles'] # roles without org is for discours.io - res['ratings'] = [] # entry['ratings'] - res['notifications'] = [] - res['links'] = [] - res['muted'] = False - res['viewname'] = 'anonymous' - if entry['profile']: - res['slug'] = entry['profile'].get('path') - res['userpic'] = entry['profile'].get('image', {'url': ''}).get('url', '') - viewname = entry['profile'].get('firstName', '') + ' ' + entry['profile'].get('lastName', '') - viewname = entry['profile']['path'] if len(viewname) < 2 else viewname - res['viewname'] = viewname - fb = entry['profile'].get('facebook', False) - if fb: - res['links'].append(fb) - vk = entry['profile'].get('vkontakte', False) - if vk: - res['links'].append(vk) - tr = entry['profile'].get('twitter', False) - if tr: - res['links'].append(tr) - ws = entry['profile'].get('website', False) - if ws: - res['links'].append(ws) - if not res['slug']: - res['slug'] = res['links'][0].split('/')[-1] + res['old_id'] = entry['_id'] + res['password'] = entry['services']['password'].get('bcrypt', '') + res['username'] = entry['emails'][0]['address'] + res['email'] = res['username'] + res['wasOnlineAt'] = parse(entry.get('loggedInAt', entry['createdAt'])) + res['emailConfirmed'] = entry['emails'][0]['verified'] + res['createdAt'] = parse(entry['createdAt']) + res['rating'] = entry['rating'] # number + res['roles'] = [] # entry['roles'] # roles without org is for discours.io + res['ratings'] = [] # entry['ratings'] + res['notifications'] = [] + res['links'] = [] + res['muted'] = False + res['bio'] = markdown.feed(entry.get('bio', '')) + if entry['profile']: + res['slug'] = entry['profile'].get('path') + res['userpic'] = entry['profile'].get('image', {'url': ''}).get('url', '') + fn = entry['profile'].get('firstName', '') + ln = entry['profile'].get('lastName', '') + viewname = res['slug'] if res['slug'] else 'anonymous' + viewname = fn if fn else viewname + viewname = (viewname + ' ' + ln) if ln else viewname + viewname = entry['profile']['path'] if len(viewname) < 2 else viewname + res['viewname'] = viewname + fb = entry['profile'].get('facebook', False) + if fb: + res['links'].append(fb) + vk = entry['profile'].get('vkontakte', False) + if vk: + res['links'].append(vk) + tr = entry['profile'].get('twitter', False) + if tr: + res['links'].append(tr) + ws = entry['profile'].get('website', False) + if ws: + res['links'].append(ws) if not res['slug']: - res['slug'] = res['email'].split('@')[0] - except Exception: - print(entry['profile']) - raise Exception + res['slug'] = res['links'][0].split('/')[-1] + if not res['slug']: + res['slug'] = res['email'].split('@')[0] else: old = res['old_id'] del res['old_id'] diff --git a/orm/user.py b/orm/user.py index aa3e24df..af149195 100644 --- a/orm/user.py +++ b/orm/user.py @@ -38,7 +38,7 @@ class User(Base): userpic: str = Column(String, nullable=True, comment="Userpic") viewname: str = Column(String, nullable=True, comment="Display name") rating: int = Column(Integer, nullable=True, comment="Rating") - slug: str = Column(String, unique=True, comment="Author's slug") + slug: str = Column(String, unique=True, comment="User's slug") muted: bool = Column(Boolean, default=False) emailConfirmed: bool = Column(Boolean, default=False) createdAt: DateTime = Column(DateTime, nullable=False, comment="Created at") diff --git a/schema.graphql b/schema.graphql index bef45f26..0ef597d1 100644 --- a/schema.graphql +++ b/schema.graphql @@ -127,7 +127,7 @@ type Role { } type Rating { - createdBy: String! + createdBy: Int! value: Int! }