From 450cf89b5061cfadaed4a9abc995c4869e7aeba9 Mon Sep 17 00:00:00 2001 From: tonyrewin Date: Thu, 14 Jul 2022 16:41:53 +0300 Subject: [PATCH] psql-migration-fixes --- migrate.py | 3 +++ migration/html2text/__init__.py | 2 +- migration/tables/comments.py | 2 +- migration/tables/content_items.py | 34 +++++++++++-------------------- orm/community.py | 5 +++-- orm/user.py | 9 ++++---- settings.py | 2 +- 7 files changed, 26 insertions(+), 31 deletions(-) diff --git a/migrate.py b/migrate.py index 754c96c0..baa00e87 100644 --- a/migrate.py +++ b/migrate.py @@ -7,6 +7,9 @@ from migration.tables.content_items import get_shout_slug, migrate as migrateSho from migration.tables.topics import migrate as migrateTopic from migration.tables.comments import migrate as migrateComment from migration.tables.comments import migrate_2stage as migrateComment_2stage +from orm.base import local_session +from orm.community import Community +from orm.user import User OLD_DATE = '2016-03-05 22:22:00.350000' diff --git a/migration/html2text/__init__.py b/migration/html2text/__init__.py index dc43c552..26810d42 100644 --- a/migration/html2text/__init__.py +++ b/migration/html2text/__init__.py @@ -1037,5 +1037,5 @@ def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = config.BO if h: h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth) h = h.handle(html.strip()) - print('[html2text] %d bytes' % len(html)) + # print('[html2text] %d bytes' % len(html)) return h diff --git a/migration/tables/comments.py b/migration/tables/comments.py index 8d270563..009023c8 100644 --- a/migration/tables/comments.py +++ b/migration/tables/comments.py @@ -57,7 +57,7 @@ def migrate(entry, storage): author = session.query(User).filter(User.oid == entry['createdBy']).first() shout_dict = storage['shouts']['by_oid'][shout_oid] if shout_dict: - comment_dict['shout'] = shout_dict['oid'] + comment_dict['shout'] = shout_dict['slug'] comment_dict['createdBy'] = author.slug if author else 'discours' # FIXME if entry.get('deleted'): comment_dict['deletedAt'] = date_parse(entry['updatedAt']) or ts # comment_dict['deletedBy'] = session.query(User).filter(User.oid == (entry.get('updatedBy') or dd['oid'])).first() diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 7db195d6..f7d567bd 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -7,14 +7,6 @@ from orm.base import local_session from migration.extract import prepare_body from orm.community import Community -DISCOURS_USER = { - 'id': 9999999, - 'slug': 'discours', - 'name': 'Дискурс', - 'email': 'welcome@discours.io', - 'userpic': 'https://discours.io/images/logo-mini.svg', - 'createdAt': '2016-03-05 22:22:00.350000' -} OLD_DATE = '2016-03-05 22:22:00.350000' ts = datetime.now() type2layout = { @@ -38,7 +30,7 @@ def migrate(entry, storage): r = { 'layout': type2layout[entry['type']], 'title': entry['title'], - 'community': 0, + 'community': Community.default_community.id, 'authors': [], 'topics': set([]), 'rating': 0, @@ -70,12 +62,7 @@ def migrate(entry, storage): 'wasOnlineAt': ts } else: - userdata = { - 'name': 'Дискурс', - 'slug': 'discours', - 'email': 'welcome@discours.io', - 'userpic': 'https://discours.io/image/logo-mini.svg' - } + userdata = User.default_user.dict() assert userdata, 'no user found for %s from ' % [oid, len(users_by_oid.keys())] r['authors'] = [userdata, ] @@ -151,7 +138,7 @@ def migrate(entry, storage): try: s = Shout.create(**shout_dict) - except sqlalchemy.exc.IntegrityError: + except sqlalchemy.exc.IntegrityError as e: with local_session() as session: s = session.query(Shout).filter(Shout.slug == shout_dict['slug']).first() bump = False @@ -168,6 +155,7 @@ def migrate(entry, storage): s.update(shout_dict) else: print('[migration] something went wrong with shout: \n%r' % shout_dict) + raise e session.commit() except: print(s) @@ -182,15 +170,17 @@ def migrate(entry, storage): if newslug: with local_session() as session: shout_topic_old = session.query(ShoutTopic)\ - .filter(ShoutTopic.shout == s.slug)\ + .filter(ShoutTopic.shout == shout_dict['slug'])\ .filter(ShoutTopic.topic == oldslug).first() if shout_topic_old: shout_topic_old.update({ 'slug': newslug }) else: shout_topic_new = session.query(ShoutTopic)\ - .filter(ShoutTopic.shout == s.slug)\ + .filter(ShoutTopic.shout == shout_dict['slug'])\ .filter(ShoutTopic.topic == newslug).first() - if not shout_topic_new: ShoutTopic.create(**{ 'shout': s.slug, 'topic': newslug }) + if not shout_topic_new: + try: ShoutTopic.create(**{ 'shout': shout_dict['slug'], 'topic': newslug }) + except: print('[migration] shout topic error: ' + newslug) session.commit() if newslug not in shout_dict['topics']: shout_dict['topics'].append(newslug) @@ -208,12 +198,12 @@ def migrate(entry, storage): shout_rating_dict = { 'value': shout_rating_old['value'], 'rater': rater.slug, - 'shout': s.slug + 'shout': shout_dict['slug'] } cts = shout_rating_old.get('createdAt') if cts: shout_rating_dict['ts'] = date_parse(cts) shout_rating = session.query(ShoutRating).\ - filter(ShoutRating.shout == s.slug).\ + filter(ShoutRating.shout == shout_dict['slug']).\ filter(ShoutRating.rater == rater.slug).first() if shout_rating: shout_rating_dict['value'] = int(shout_rating_dict['value'] or 0) + int(shout_rating.value or 0) @@ -225,7 +215,7 @@ def migrate(entry, storage): # raise Exception # shout views - ShoutViewByDay.create( shout = s.slug, value = entry.get('views', 1) ) + ShoutViewByDay.create( shout = shout_dict['slug'], value = entry.get('views', 1) ) del shout_dict['ratings'] shout_dict['oid'] = entry.get('_id') storage['shouts']['by_oid'][entry['_id']] = shout_dict diff --git a/orm/community.py b/orm/community.py index d81ed49f..7ecb917f 100644 --- a/orm/community.py +++ b/orm/community.py @@ -1,4 +1,5 @@ from datetime import datetime +from enum import unique from sqlalchemy import Column, Integer, String, ForeignKey, DateTime from sqlalchemy.orm import relationship, backref from orm.base import Base, local_session @@ -16,7 +17,7 @@ class Community(Base): __tablename__ = 'community' name: str = Column(String, nullable=False, comment="Name") - slug: str = Column(String, nullable = False) + slug: str = Column(String, nullable = False, unique=True, comment="Slug") desc: str = Column(String, nullable=False, default='') pic: str = Column(String, nullable=False, default='') createdAt: str = Column(DateTime, nullable=False, default = datetime.now, comment="Created at") @@ -30,7 +31,7 @@ class Community(Base): default = Community.create( name = "Дискурс", slug = "discours", - createdBy = 0 #TODO: use default user + createdBy = "discours" ) Community.default_community = default diff --git a/orm/user.py b/orm/user.py index 9093392f..93062159 100644 --- a/orm/user.py +++ b/orm/user.py @@ -73,13 +73,14 @@ class User(Base): @staticmethod def init_table(): with local_session() as session: - default = session.query(User).filter(User.slug == "default").first() + default = session.query(User).filter(User.slug == "discours").first() if not default: default = User.create( id = 0, - email = "discours@discours.io", - username = "discours", - slug = "default" + email = "welcome@discours.io", + username = "welcome@discours.io", + slug = "discours", + userpic = 'https://discours.io/images/logo-mini.svg', ) User.default_user = default diff --git a/settings.py b/settings.py index 11ed4a5c..cdb92226 100644 --- a/settings.py +++ b/settings.py @@ -10,7 +10,7 @@ RESET_PWD_URL = environ.get("RESET_PWD_URL") or "https://localhost:8080/reset_pw CONFIRM_EMAIL_URL = environ.get("CONFIRM_EMAIL_URL") or "https://new.discours.io" ERROR_URL_ON_FRONTEND = environ.get("ERROR_URL_ON_FRONTEND") or "https://new.discours.io" -DB_URL = environ.get("DATABASE_URL") or environ.get("DB_URL") or "postgres://localhost:5432" or "sqlite:///db.sqlite3" +DB_URL = environ.get("DATABASE_URL") or environ.get("DB_URL") or "postgresql://localhost:5432/discoursio" or "sqlite:///db.sqlite3" JWT_ALGORITHM = "HS256" JWT_SECRET_KEY = "8f1bd7696ffb482d8486dfbc6e7d16dd-secret-key" JWT_LIFE_SPAN = 24 * 60 * 60 # seconds