configured isort, black, flake8

migration/tables/__init__.py

@@ -1 +0,0 @@
-__all__ = (["users", "topics", "content_items", "comments"],)

migration/tables/comments.py

@@ -5,61 +5,48 @@ from dateutil.parser import parse as date_parse
 from base.orm import local_session
 from migration.html2text import html2text
 from orm.reaction import Reaction, ReactionKind
-from orm.shout import ShoutReactionsFollower
+from orm.shout import Shout, ShoutReactionsFollower
 from orm.topic import TopicFollower
 from orm.user import User
-from orm.shout import Shout
 
 ts = datetime.now(tz=timezone.utc)
 
 
 def auto_followers(session, topics, reaction_dict):
     # creating shout's reactions following for reaction author
-    following1 = session.query(
-        ShoutReactionsFollower
-    ).where(
-        ShoutReactionsFollower.follower == reaction_dict["createdBy"]
-    ).filter(
-        ShoutReactionsFollower.shout == reaction_dict["shout"]
-    ).first()
+    following1 = (
+        session.query(ShoutReactionsFollower)
+        .where(ShoutReactionsFollower.follower == reaction_dict["createdBy"])
+        .filter(ShoutReactionsFollower.shout == reaction_dict["shout"])
+        .first()
+    )
     if not following1:
         following1 = ShoutReactionsFollower.create(
-            follower=reaction_dict["createdBy"],
-            shout=reaction_dict["shout"],
-            auto=True
+            follower=reaction_dict["createdBy"], shout=reaction_dict["shout"], auto=True
         )
         session.add(following1)
     # creating topics followings for reaction author
     for t in topics:
-        tf = session.query(
-            TopicFollower
-        ).where(
-            TopicFollower.follower == reaction_dict["createdBy"]
-        ).filter(
-            TopicFollower.topic == t['id']
-        ).first()
+        tf = (
+            session.query(TopicFollower)
+            .where(TopicFollower.follower == reaction_dict["createdBy"])
+            .filter(TopicFollower.topic == t["id"])
+            .first()
+        )
         if not tf:
             topic_following = TopicFollower.create(
-                follower=reaction_dict["createdBy"],
-                topic=t['id'],
-                auto=True
+                follower=reaction_dict["createdBy"], topic=t["id"], auto=True
             )
             session.add(topic_following)
 
 
 def migrate_ratings(session, entry, reaction_dict):
     for comment_rating_old in entry.get("ratings", []):
-        rater = (
-            session.query(User)
-            .filter(User.oid == comment_rating_old["createdBy"])
-            .first()
-        )
+        rater = session.query(User).filter(User.oid == comment_rating_old["createdBy"]).first()
         re_reaction_dict = {
             "shout": reaction_dict["shout"],
             "replyTo": reaction_dict["id"],
-            "kind": ReactionKind.LIKE
-            if comment_rating_old["value"] > 0
-            else ReactionKind.DISLIKE,
+            "kind": ReactionKind.LIKE if comment_rating_old["value"] > 0 else ReactionKind.DISLIKE,
             "createdBy": rater.id if rater else 1,
         }
         cts = comment_rating_old.get("createdAt")
@@ -68,18 +55,15 @@ def migrate_ratings(session, entry, reaction_dict):
         try:
             # creating reaction from old rating
             rr = Reaction.create(**re_reaction_dict)
-            following2 = session.query(
-                ShoutReactionsFollower
-            ).where(
-                ShoutReactionsFollower.follower == re_reaction_dict['createdBy']
-            ).filter(
-                ShoutReactionsFollower.shout == rr.shout
-            ).first()
+            following2 = (
+                session.query(ShoutReactionsFollower)
+                .where(ShoutReactionsFollower.follower == re_reaction_dict["createdBy"])
+                .filter(ShoutReactionsFollower.shout == rr.shout)
+                .first()
+            )
             if not following2:
                 following2 = ShoutReactionsFollower.create(
-                    follower=re_reaction_dict['createdBy'],
-                    shout=rr.shout,
-                    auto=True
+                    follower=re_reaction_dict["createdBy"], shout=rr.shout, auto=True
                 )
                 session.add(following2)
             session.add(rr)
@@ -150,9 +134,7 @@ async def migrate(entry, storage):
             else:
                 stage = "author and old id found"
                 try:
-                    shout = session.query(
-                        Shout
-                    ).where(Shout.slug == old_shout["slug"]).one()
+                    shout = session.query(Shout).where(Shout.slug == old_shout["slug"]).one()
                     if shout:
                         reaction_dict["shout"] = shout.id
                         reaction_dict["createdBy"] = author.id if author else 1
@@ -178,9 +160,9 @@ async def migrate(entry, storage):
 
 
 def migrate_2stage(old_comment, idmap):
-    if old_comment.get('body'):
-        new_id = idmap.get(old_comment.get('oid'))
-        new_id = idmap.get(old_comment.get('_id'))
+    if old_comment.get("body"):
+        new_id = idmap.get(old_comment.get("oid"))
+        new_id = idmap.get(old_comment.get("_id"))
         if new_id:
             new_replyto_id = None
             old_replyto_id = old_comment.get("replyTo")
@@ -190,17 +172,20 @@ def migrate_2stage(old_comment, idmap):
                 comment = session.query(Reaction).where(Reaction.id == new_id).first()
                 try:
                     if new_replyto_id:
-                        new_reply = session.query(Reaction).where(Reaction.id == new_replyto_id).first()
+                        new_reply = (
+                            session.query(Reaction).where(Reaction.id == new_replyto_id).first()
+                        )
                         if not new_reply:
                             print(new_replyto_id)
                             raise Exception("cannot find reply by id!")
                         comment.replyTo = new_reply.id
                         session.add(comment)
-                    srf = session.query(ShoutReactionsFollower).where(
-                        ShoutReactionsFollower.shout == comment.shout
-                    ).filter(
-                        ShoutReactionsFollower.follower == comment.createdBy
-                    ).first()
+                    srf = (
+                        session.query(ShoutReactionsFollower)
+                        .where(ShoutReactionsFollower.shout == comment.shout)
+                        .filter(ShoutReactionsFollower.follower == comment.createdBy)
+                        .first()
+                    )
                     if not srf:
                         srf = ShoutReactionsFollower.create(
                             shout=comment.shout, follower=comment.createdBy, auto=True

migration/tables/content_items.py

@@ -1,16 +1,18 @@
-from datetime import datetime, timezone
 import json
+import re
+from datetime import datetime, timezone
+
 from dateutil.parser import parse as date_parse
 from sqlalchemy.exc import IntegrityError
 from transliterate import translit
+
 from base.orm import local_session
 from migration.extract import extract_html, extract_media
 from orm.reaction import Reaction, ReactionKind
-from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
+from orm.shout import Shout, ShoutReactionsFollower, ShoutTopic
+from orm.topic import Topic, TopicFollower
 from orm.user import User
-from orm.topic import TopicFollower, Topic
 from services.stat.viewed import ViewedStorage
-import re
 
 OLD_DATE = "2016-03-05 22:22:00.350000"
 ts = datetime.now(tz=timezone.utc)
@@ -33,7 +35,7 @@ def get_shout_slug(entry):
             slug = friend.get("slug", "")
             if slug:
                 break
-    slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+    slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
     return slug
 
 
@@ -41,27 +43,27 @@ def create_author_from_app(app):
     user = None
     userdata = None
     # check if email is used
-    if app['email']:
+    if app["email"]:
         with local_session() as session:
-            user = session.query(User).where(User.email == app['email']).first()
+            user = session.query(User).where(User.email == app["email"]).first()
             if not user:
                 # print('[migration] app %r' % app)
-                name = app.get('name')
+                name = app.get("name")
                 if name:
                     slug = translit(name, "ru", reversed=True).lower()
-                    slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
-                    print('[migration] created slug %s' % slug)
+                    slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
+                    print("[migration] created slug %s" % slug)
                     # check if slug is used
                     if slug:
                         user = session.query(User).where(User.slug == slug).first()
 
                         # get slug from email
                         if user:
-                            slug = app['email'].split('@')[0]
+                            slug = app["email"].split("@")[0]
                             user = session.query(User).where(User.slug == slug).first()
                             # one more try
                             if user:
-                                slug += '-author'
+                                slug += "-author"
                                 user = session.query(User).where(User.slug == slug).first()
 
                 # create user with application data
@@ -79,7 +81,7 @@ def create_author_from_app(app):
                     user = User.create(**userdata)
                     session.add(user)
                     session.commit()
-                    userdata['id'] = user.id
+                    userdata["id"] = user.id
 
             userdata = user.dict()
         return userdata
@@ -91,11 +93,12 @@ async def create_shout(shout_dict):
     s = Shout.create(**shout_dict)
     author = s.authors[0]
     with local_session() as session:
-        srf = session.query(ShoutReactionsFollower).where(
-            ShoutReactionsFollower.shout == s.id
-        ).filter(
-            ShoutReactionsFollower.follower == author.id
-        ).first()
+        srf = (
+            session.query(ShoutReactionsFollower)
+            .where(ShoutReactionsFollower.shout == s.id)
+            .filter(ShoutReactionsFollower.follower == author.id)
+            .first()
+        )
         if not srf:
             srf = ShoutReactionsFollower.create(shout=s.id, follower=author.id, auto=True)
             session.add(srf)
@@ -116,14 +119,14 @@ async def get_user(entry, storage):
         elif user_oid:
             userdata = storage["users"]["by_oid"].get(user_oid)
         if not userdata:
-            print('no userdata by oid, anonymous')
+            print("no userdata by oid, anonymous")
             userdata = anondict
             print(app)
     # cleanup slug
     if userdata:
         slug = userdata.get("slug", "")
         if slug:
-            slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+            slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
             userdata["slug"] = slug
     else:
         userdata = anondict
@@ -137,11 +140,14 @@ async def migrate(entry, storage):
     r = {
         "layout": type2layout[entry["type"]],
         "title": entry["title"],
-        "authors": [author, ],
+        "authors": [
+            author,
+        ],
         "slug": get_shout_slug(entry),
         "cover": (
-            "https://images.discours.io/unsafe/" +
-            entry["thumborId"] if entry.get("thumborId") else entry.get("image", {}).get("url")
+            "https://images.discours.io/unsafe/" + entry["thumborId"]
+            if entry.get("thumborId")
+            else entry.get("image", {}).get("url")
         ),
         "visibility": "public" if entry.get("published") else "community",
         "publishedAt": date_parse(entry.get("publishedAt")) if entry.get("published") else None,
@@ -150,11 +156,11 @@ async def migrate(entry, storage):
         "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts,
         "createdBy": author.id,
         "topics": await add_topics_follower(entry, storage, author),
-        "body": extract_html(entry, cleanup=True)
+        "body": extract_html(entry, cleanup=True),
     }
 
     # main topic patch
-    r['mainTopic'] = r['topics'][0]
+    r["mainTopic"] = r["topics"][0]
 
     # published author auto-confirm
     if entry.get("published"):
@@ -177,14 +183,16 @@ async def migrate(entry, storage):
         shout_dict["oid"] = entry.get("_id", "")
         shout = await create_shout(shout_dict)
     except IntegrityError as e:
-        print('[migration] create_shout integrity error', e)
+        print("[migration] create_shout integrity error", e)
         shout = await resolve_create_shout(shout_dict)
     except Exception as e:
         raise Exception(e)
 
     # udpate data
     shout_dict = shout.dict()
-    shout_dict["authors"] = [author.dict(), ]
+    shout_dict["authors"] = [
+        author.dict(),
+    ]
 
     # shout topics aftermath
     shout_dict["topics"] = await topics_aftermath(r, storage)
@@ -193,7 +201,9 @@ async def migrate(entry, storage):
     await content_ratings_to_reactions(entry, shout_dict["slug"])
 
     # shout views
-    await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1), viewer='old-discours')
+    await ViewedStorage.increment(
+        shout_dict["slug"], amount=entry.get("views", 1), viewer="old-discours"
+    )
     # del shout_dict['ratings']
 
     storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
@@ -205,7 +215,9 @@ async def add_topics_follower(entry, storage, user):
     topics = set([])
     category = entry.get("category")
     topics_by_oid = storage["topics"]["by_oid"]
-    oids = [category, ] + entry.get("tags", [])
+    oids = [
+        category,
+    ] + entry.get("tags", [])
     for toid in oids:
         tslug = topics_by_oid.get(toid, {}).get("slug")
         if tslug:
@@ -217,23 +229,18 @@ async def add_topics_follower(entry, storage, user):
             try:
                 tpc = session.query(Topic).where(Topic.slug == tpcslug).first()
                 if tpc:
-                    tf = session.query(
-                        TopicFollower
-                    ).where(
-                        TopicFollower.follower == user.id
-                    ).filter(
-                        TopicFollower.topic == tpc.id
-                    ).first()
+                    tf = (
+                        session.query(TopicFollower)
+                        .where(TopicFollower.follower == user.id)
+                        .filter(TopicFollower.topic == tpc.id)
+                        .first()
+                    )
                     if not tf:
-                        tf = TopicFollower.create(
-                            topic=tpc.id,
-                            follower=user.id,
-                            auto=True
-                        )
+                        tf = TopicFollower.create(topic=tpc.id, follower=user.id, auto=True)
                         session.add(tf)
                         session.commit()
             except IntegrityError:
-                print('[migration.shout] hidden by topic ' + tpc.slug)
+                print("[migration.shout] hidden by topic " + tpc.slug)
     # main topic
     maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug"))
     if maintopic in ttt:
@@ -254,7 +261,7 @@ async def process_user(userdata, storage, oid):
         if not user:
             try:
                 slug = userdata["slug"].lower().strip()
-                slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+                slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
                 userdata["slug"] = slug
                 user = User.create(**userdata)
                 session.add(user)
@@ -282,9 +289,9 @@ async def resolve_create_shout(shout_dict):
         s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
         bump = False
         if s:
-            if s.createdAt != shout_dict['createdAt']:
+            if s.createdAt != shout_dict["createdAt"]:
                 # create new with different slug
-                shout_dict["slug"] += '-' + shout_dict["layout"]
+                shout_dict["slug"] += "-" + shout_dict["layout"]
                 try:
                     await create_shout(shout_dict)
                 except IntegrityError as e:
@@ -295,10 +302,7 @@ async def resolve_create_shout(shout_dict):
                 for key in shout_dict:
                     if key in s.__dict__:
                         if s.__dict__[key] != shout_dict[key]:
-                            print(
-                                "[migration] shout already exists, but differs in %s"
-                                % key
-                            )
+                            print("[migration] shout already exists, but differs in %s" % key)
                             bump = True
                     else:
                         print("[migration] shout already exists, but lacks %s" % key)
@@ -344,9 +348,7 @@ async def topics_aftermath(entry, storage):
                     )
                     if not shout_topic_new:
                         try:
-                            ShoutTopic.create(
-                                **{"shout": shout.id, "topic": new_topic.id}
-                            )
+                            ShoutTopic.create(**{"shout": shout.id, "topic": new_topic.id})
                         except Exception:
                             print("[migration] shout topic error: " + newslug)
                 session.commit()
@@ -363,9 +365,7 @@ async def content_ratings_to_reactions(entry, slug):
         with local_session() as session:
             for content_rating in entry.get("ratings", []):
                 rater = (
-                    session.query(User)
-                    .filter(User.oid == content_rating["createdBy"])
-                    .first()
+                    session.query(User).filter(User.oid == content_rating["createdBy"]).first()
                 ) or User.default_user
                 shout = session.query(Shout).where(Shout.slug == slug).first()
                 cts = content_rating.get("createdAt")
@@ -375,7 +375,7 @@ async def content_ratings_to_reactions(entry, slug):
                     if content_rating["value"] > 0
                     else ReactionKind.DISLIKE,
                     "createdBy": rater.id,
-                    "shout": shout.id
+                    "shout": shout.id,
                 }
                 reaction = (
                     session.query(Reaction)

migration/tables/remarks.py

@@ -1,42 +1,35 @@
-from base.orm import local_session
-from migration.extract import extract_md
-from migration.html2text import html2text
-from orm.reaction import Reaction, ReactionKind
+# from base.orm import local_session
+
+# from migration.extract import extract_md
+# from migration.html2text import html2text
+# from orm.reaction import Reaction, ReactionKind
 
 
-def migrate(entry, storage):
-    post_oid = entry['contentItem']
-    print(post_oid)
-    shout_dict = storage['shouts']['by_oid'].get(post_oid)
-    if shout_dict:
-        print(shout_dict['body'])
-        remark = {
-            "shout": shout_dict['id'],
-            "body": extract_md(
-                html2text(entry['body']),
-                shout_dict
-            ),
-            "kind": ReactionKind.REMARK
-        }
-
-        if entry.get('textBefore'):
-            remark['range'] = str(
-                    shout_dict['body']
-                        .index(
-                            entry['textBefore'] or ''
-                        )
-                ) + ':' + str(
-                    shout_dict['body']
-                        .index(
-                            entry['textAfter'] or ''
-                        ) + len(
-                            entry['textAfter'] or ''
-                        )
-                )
-
-        with local_session() as session:
-            rmrk = Reaction.create(**remark)
-            session.commit()
-            del rmrk["_sa_instance_state"]
-            return rmrk
-    return
+# def migrate(entry, storage):
+#     post_oid = entry["contentItem"]
+#     print(post_oid)
+#     shout_dict = storage["shouts"]["by_oid"].get(post_oid)
+#     if shout_dict:
+#         print(shout_dict["body"])
+#         remark = {
+#             "shout": shout_dict["id"],
+#             "body": extract_md(html2text(entry["body"]), shout_dict),
+#             "kind": ReactionKind.REMARK,
+#         }
+#
+#         if entry.get("textBefore"):
+#             remark["range"] = (
+#                 str(shout_dict["body"].index(entry["textBefore"] or ""))
+#                 + ":"
+#                 + str(
+#                     shout_dict["body"].index(entry["textAfter"] or "")
+#                     + len(entry["textAfter"] or "")
+#                 )
+#             )
+#
+#         with local_session() as session:
+#             rmrk = Reaction.create(**remark)
+#             session.commit()
+#             del rmrk["_sa_instance_state"]
+#             return rmrk
+#     return

migration/tables/topics.py

@@ -1,5 +1,4 @@
 from base.orm import local_session
-from migration.extract import extract_md
 from migration.html2text import html2text
 from orm import Topic
 
@@ -10,7 +9,7 @@ def migrate(entry):
         "slug": entry["slug"],
         "oid": entry["_id"],
         "title": entry["title"].replace(" ", " "),
-        "body": extract_md(html2text(body_orig))
+        "body": html2text(body_orig),
     }
 
     with local_session() as session:

migration/tables/users.py

@@ -8,7 +8,7 @@ from base.orm import local_session
 from orm.user import AuthorFollower, User, UserRating
 
 
-def migrate(entry):
+def migrate(entry):  # noqa: C901
     if "subscribedTo" in entry:
         del entry["subscribedTo"]
     email = entry["emails"][0]["address"]
@@ -23,7 +23,7 @@ def migrate(entry):
         "muted": False,  # amnesty
         "links": [],
         "name": "anonymous",
-        "password": entry["services"]["password"].get("bcrypt")
+        "password": entry["services"]["password"].get("bcrypt"),
     }
 
     if "updatedAt" in entry:
@@ -33,9 +33,13 @@ def migrate(entry):
     if entry.get("profile"):
         # slug
         slug = entry["profile"].get("path").lower()
-        slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip()
+        slug = re.sub("[^0-9a-zA-Z]+", "-", slug).strip()
         user_dict["slug"] = slug
-        bio = (entry.get("profile", {"bio": ""}).get("bio") or "").replace('\(', '(').replace('\)', ')')
+        bio = (
+            (entry.get("profile", {"bio": ""}).get("bio") or "")
+            .replace(r"\(", "(")
+            .replace(r"\)", ")")
+        )
         bio_text = BeautifulSoup(bio, features="lxml").text
 
         if len(bio_text) > 120:
@@ -46,8 +50,7 @@ def migrate(entry):
         # userpic
         try:
             user_dict["userpic"] = (
-                "https://images.discours.io/unsafe/"
-                + entry["profile"]["thumborId"]
+                "https://images.discours.io/unsafe/" + entry["profile"]["thumborId"]
             )
         except KeyError:
             try:
@@ -62,11 +65,7 @@ def migrate(entry):
         name = (name + " " + ln) if ln else name
         if not name:
             name = slug if slug else "anonymous"
-        name = (
-            entry["profile"]["path"].lower().strip().replace(" ", "-")
-            if len(name) < 2
-            else name
-        )
+        name = entry["profile"]["path"].lower().strip().replace(" ", "-") if len(name) < 2 else name
         user_dict["name"] = name
 
         # links
@@ -95,9 +94,7 @@ def migrate(entry):
     except IntegrityError:
         print("[migration] cannot create user " + user_dict["slug"])
         with local_session() as session:
-            old_user = (
-                session.query(User).filter(User.slug == user_dict["slug"]).first()
-            )
+            old_user = session.query(User).filter(User.slug == user_dict["slug"]).first()
             old_user.oid = oid
             old_user.password = user_dict["password"]
             session.commit()
@@ -114,7 +111,7 @@ def post_migrate():
         "slug": "old-discours",
         "username": "old-discours",
         "email": "old@discours.io",
-        "name": "Просмотры на старой версии сайта"
+        "name": "Просмотры на старой версии сайта",
     }
 
     with local_session() as session:
@@ -147,12 +144,8 @@ def migrate_2stage(entry, id_map):
                 }
 
                 user_rating = UserRating.create(**user_rating_dict)
-                if user_rating_dict['value'] > 0:
-                    af = AuthorFollower.create(
-                        author=user.id,
-                        follower=rater.id,
-                        auto=True
-                    )
+                if user_rating_dict["value"] > 0:
+                    af = AuthorFollower.create(author=user.id, follower=rater.id, auto=True)
                     session.add(af)
                 session.add(user_rating)
                 session.commit()