configured isort, black, flake8

.flake8

@@ -1,6 +0,0 @@
-[flake8]
-ignore = E203,W504,W191,W503
-exclude = .git,__pycache__,orm/rbac.py
-max-complexity = 10
-max-line-length = 108
-indent-string = '  '

.github/workflows/checks.yml

@@ -0,0 +1,16 @@
+name: Checks
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Checks
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.10
+    - run: pip install --upgrade pip
+    - run: pip install -r requirements.txt
+    - run: pip install -r requirements-dev.txt
+    - run: check.sh

.pre-commit-config.yaml

@@ -6,11 +6,11 @@ exclude: |
   )
 
 default_language_version:
-  python: python3.8
+  python: python3.10
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+    rev: v4.5.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -21,24 +21,24 @@ repos:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
+      - id: requirements-txt-fixer
 
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.5.3
+    rev: 5.12.0
     hooks:
       - id: isort
 
   - repo: https://github.com/ambv/black
-    rev: 20.8b1
+    rev: 23.10.1
     hooks:
       - id: black
-        args:
-          - --line-length=100
-          - --skip-string-normalization
 
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
     hooks:
       - id: flake8
-        args:
-          - --max-line-length=100
-          - --disable=protected-access
+
+#  - repo: https://github.com/python/mypy
+#    rev: v1.6.1
+#    hooks:
+#      - id: mypy

Procfile

@@ -1,2 +1 @@
 web: python server.py
-

README.md

@@ -42,4 +42,3 @@ Put the header 'Authorization' with token from signIn query or registerUser muta
 # How to debug Ackee
 
 Set ACKEE_TOKEN var
-

ai/preprocess.py

@@ -1,75 +0,0 @@
-import re
-import nltk
-from bs4 import BeautifulSoup
-from nltk.corpus import stopwords
-from pymystem3 import Mystem
-from string import punctuation
-from transformers import BertTokenizer
-
-nltk.download("stopwords")
-
-
-def get_clear_text(text):
-    soup = BeautifulSoup(text, 'html.parser')
-
-    # extract the plain text from the HTML document without tags
-    clear_text = ''
-    for tag in soup.find_all():
-        clear_text += tag.string or ''
-
-    clear_text = re.sub(pattern='[\u202F\u00A0\n]+', repl=' ', string=clear_text)
-
-    # only words
-    clear_text = re.sub(pattern='[^A-ZА-ЯЁ -]', repl='', string=clear_text, flags=re.IGNORECASE)
-
-    clear_text = re.sub(pattern='\s+', repl=' ', string=clear_text)
-
-    clear_text = clear_text.lower()
-
-    mystem = Mystem()
-    russian_stopwords = stopwords.words("russian")
-
-    tokens = mystem.lemmatize(clear_text)
-    tokens = [token for token in tokens if token not in russian_stopwords \
-              and token != " " \
-              and token.strip() not in punctuation]
-
-    clear_text = " ".join(tokens)
-
-    return clear_text
-
-
-# if __name__ == '__main__':
-#
-#     # initialize the tokenizer with the pre-trained BERT model and vocabulary
-#     tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
-#
-#     # split each text into smaller segments of maximum length 512
-#     max_length = 512
-#     segmented_texts = []
-#     for text in [clear_text1, clear_text2]:
-#         segmented_text = []
-#         for i in range(0, len(text), max_length):
-#             segment = text[i:i+max_length]
-#             segmented_text.append(segment)
-#         segmented_texts.append(segmented_text)
-#
-#     # tokenize each segment using the BERT tokenizer
-#     tokenized_texts = []
-#     for segmented_text in segmented_texts:
-#         tokenized_text = []
-#         for segment in segmented_text:
-#             segment_tokens = tokenizer.tokenize(segment)
-#             segment_tokens = ['[CLS]'] + segment_tokens + ['[SEP]']
-#             tokenized_text.append(segment_tokens)
-#         tokenized_texts.append(tokenized_text)
-#
-#     input_ids = []
-#     for tokenized_text in tokenized_texts:
-#         input_id = []
-#         for segment_tokens in tokenized_text:
-#             segment_id = tokenizer.convert_tokens_to_ids(segment_tokens)
-#             input_id.append(segment_id)
-#         input_ids.append(input_id)
-#
-#     print(input_ids)

alembic/env.py

@@ -1,10 +1,9 @@
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
+from sqlalchemy import engine_from_config, pool
 
 from alembic import context
-
+from base.orm import Base
 from settings import DB_URL
 
 # this is the Alembic Config object, which provides
@@ -19,7 +18,6 @@ config.set_section_option(config.config_ini_section, "DB_URL", DB_URL)
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
 
-from base.orm import Base
 target_metadata = [Base.metadata]
 
 # other values from the config, defined by the needs of env.py,
@@ -66,9 +64,7 @@ def run_migrations_online() -> None:
     )
 
     with connectable.connect() as connection:
-        context.configure(
-            connection=connection, target_metadata=target_metadata
-        )
+        context.configure(connection=connection, target_metadata=target_metadata)
 
         with context.begin_transaction():
             context.run_migrations()

alembic/versions/fe943b098418_init_alembic.py

@@ -1,18 +1,18 @@
 """init alembic
 
 Revision ID: fe943b098418
-Revises: 
+Revises:
 Create Date: 2023-08-19 01:37:57.031933
 
 """
 from typing import Sequence, Union
 
-from alembic import op
-import sqlalchemy as sa
+# import sqlalchemy as sa
 
+# from alembic import op
 
 # revision identifiers, used by Alembic.
-revision: str = 'fe943b098418'
+revision: str = "fe943b098418"
 down_revision: Union[str, None] = None
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None

auth/authenticate.py

@@ -2,75 +2,71 @@ from functools import wraps
 from typing import Optional, Tuple
 
 from graphql.type import GraphQLResolveInfo
-from sqlalchemy.orm import joinedload, exc
+from sqlalchemy.orm import exc, joinedload
 from starlette.authentication import AuthenticationBackend
 from starlette.requests import HTTPConnection
 
 from auth.credentials import AuthCredentials, AuthUser
-from base.orm import local_session
-from orm.user import User, Role
-
-from settings import SESSION_TOKEN_HEADER
 from auth.tokenstorage import SessionToken
 from base.exceptions import OperationNotAllowed
+from base.orm import local_session
+from orm.user import Role, User
+from settings import SESSION_TOKEN_HEADER
 
 
 class JWTAuthenticate(AuthenticationBackend):
     async def authenticate(
         self, request: HTTPConnection
     ) -> Optional[Tuple[AuthCredentials, AuthUser]]:
-
         if SESSION_TOKEN_HEADER not in request.headers:
-            return AuthCredentials(scopes={}), AuthUser(user_id=None, username='')
+            return AuthCredentials(scopes={}), AuthUser(user_id=None, username="")
 
         token = request.headers.get(SESSION_TOKEN_HEADER)
         if not token:
             print("[auth.authenticate] no token in header %s" % SESSION_TOKEN_HEADER)
             return AuthCredentials(scopes={}, error_message=str("no token")), AuthUser(
-                user_id=None, username=''
+                user_id=None, username=""
             )
 
-        if len(token.split('.')) > 1:
+        if len(token.split(".")) > 1:
             payload = await SessionToken.verify(token)
 
             with local_session() as session:
                 try:
                     user = (
-                        session.query(User).options(
+                        session.query(User)
+                        .options(
                             joinedload(User.roles).options(joinedload(Role.permissions)),
-                            joinedload(User.ratings)
-                        ).filter(
-                            User.id == payload.user_id
-                        ).one()
+                            joinedload(User.ratings),
+                        )
+                        .filter(User.id == payload.user_id)
+                        .one()
                     )
 
                     scopes = {}  # TODO: integrate await user.get_permission()
 
                     return (
-                        AuthCredentials(
-                            user_id=payload.user_id,
-                            scopes=scopes,
-                            logged_in=True
-                        ),
-                        AuthUser(user_id=user.id, username=''),
+                        AuthCredentials(user_id=payload.user_id, scopes=scopes, logged_in=True),
+                        AuthUser(user_id=user.id, username=""),
                     )
                 except exc.NoResultFound:
                     pass
 
-        return AuthCredentials(scopes={}, error_message=str('Invalid token')), AuthUser(user_id=None, username='')
+        return AuthCredentials(scopes={}, error_message=str("Invalid token")), AuthUser(
+            user_id=None, username=""
+        )
 
 
 def login_required(func):
     @wraps(func)
     async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs):
-        # print('[auth.authenticate] login required for %r with info %r' % (func, info))  # debug only
+        # debug only
+        # print('[auth.authenticate] login required for %r with info %r' % (func, info))
         auth: AuthCredentials = info.context["request"].auth
         # print(auth)
         if not auth or not auth.logged_in:
             # raise Unauthorized(auth.error_message or "Please login")
-            return {
-                "error": "Please login first"
-            }
+            return {"error": "Please login first"}
         return await func(parent, info, *args, **kwargs)
 
     return wrap
@@ -79,7 +75,9 @@ def login_required(func):
 def permission_required(resource, operation, func):
     @wraps(func)
     async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs):
-        print('[auth.authenticate] permission_required for %r with info %r' % (func, info))  # debug only
+        print(
+            "[auth.authenticate] permission_required for %r with info %r" % (func, info)
+        )  # debug only
         auth: AuthCredentials = info.context["request"].auth
         if not auth.logged_in:
             raise OperationNotAllowed(auth.error_message or "Please login")

auth/credentials.py

@@ -23,13 +23,11 @@ class AuthCredentials(BaseModel):
     async def permissions(self) -> List[Permission]:
         if self.user_id is None:
             # raise Unauthorized("Please login first")
-            return {
-                "error": "Please login first"
-            }
+            return {"error": "Please login first"}
         else:
             # TODO: implement permissions logix
             print(self.user_id)
-        return NotImplemented()
+        return NotImplemented
 
 
 class AuthUser(BaseModel):
@@ -40,6 +38,6 @@ class AuthUser(BaseModel):
     def is_authenticated(self) -> bool:
         return self.user_id is not None
 
-    @property
-    def display_id(self) -> int:
-        return self.user_id
+    # @property
+    # def display_id(self) -> int:
+    #     return self.user_id

auth/email.py

@@ -2,19 +2,16 @@ import requests
 
 from settings import MAILGUN_API_KEY, MAILGUN_DOMAIN
 
-api_url = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN or 'discours.io')
-noreply = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN or 'discours.io')
-lang_subject = {
-    "ru": "Подтверждение почты",
-    "en": "Confirm email"
-}
+api_url = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN or "discours.io")
+noreply = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN or "discours.io")
+lang_subject = {"ru": "Подтверждение почты", "en": "Confirm email"}
 
 
 async def send_auth_email(user, token, lang="ru", template="email_confirmation"):
     try:
         to = "%s <%s>" % (user.name, user.email)
-        if lang not in ['ru', 'en']:
-            lang = 'ru'
+        if lang not in ["ru", "en"]:
+            lang = "ru"
         subject = lang_subject.get(lang, lang_subject["en"])
         template = template + "_" + lang
         payload = {
@@ -22,16 +19,12 @@ async def send_auth_email(user, token, lang="ru", template="email_confirmation")
             "to": to,
             "subject": subject,
             "template": template,
-            "h:X-Mailgun-Variables": "{ \"token\": \"%s\" }" % token
+            "h:X-Mailgun-Variables": '{ "token": "%s" }' % token,
         }
-        print('[auth.email] payload: %r' % payload)
+        print("[auth.email] payload: %r" % payload)
         # debug
         # print('http://localhost:3000/?modal=auth&mode=confirm-email&token=%s' % token)
-        response = requests.post(
-            api_url,
-            auth=("api", MAILGUN_API_KEY),
-            data=payload
-        )
+        response = requests.post(api_url, auth=("api", MAILGUN_API_KEY), data=payload)
         response.raise_for_status()
     except Exception as e:
         print(e)

auth/identity.py

@@ -7,6 +7,7 @@ from sqlalchemy import or_
 
 from auth.jwtcodec import JWTCodec
 from auth.tokenstorage import TokenStorage
+
 # from base.exceptions import InvalidPassword, InvalidToken
 from base.orm import local_session
 from orm import User
@@ -34,7 +35,7 @@ class Password:
         Verify that password hash is equal to specified hash. Hash format:
 
         $2a$10$Ro0CUfOqk6cXEKf3dyaM7OhSCvnwM9s4wIX9JeLapehKK5YdLxKcm
-        \__/\/ \____________________/\_____________________________/
+        \__/\/ \____________________/\_____________________________/  # noqa: W605
         |   |        Salt                     Hash
         |  Cost
         Version
@@ -57,14 +58,10 @@ class Identity:
         user = User(**orm_user.dict())
         if not user.password:
             # raise InvalidPassword("User password is empty")
-            return {
-                "error": "User password is empty"
-            }
+            return {"error": "User password is empty"}
         if not Password.verify(password, user.password):
             # raise InvalidPassword("Wrong user password")
-            return {
-                "error": "Wrong user password"
-            }
+            return {"error": "Wrong user password"}
         return user
 
     @staticmethod
@@ -87,30 +84,22 @@ class Identity:
     @staticmethod
     async def onetime(token: str) -> User:
         try:
-            print('[auth.identity] using one time token')
+            print("[auth.identity] using one time token")
             payload = JWTCodec.decode(token)
             if not await TokenStorage.exist(f"{payload.user_id}-{payload.username}-{token}"):
                 # raise InvalidToken("Login token has expired, please login again")
-                return {
-                    "error": "Token has expired"
-                }
+                return {"error": "Token has expired"}
         except ExpiredSignatureError:
             # raise InvalidToken("Login token has expired, please try again")
-            return {
-                "error": "Token has expired"
-            }
+            return {"error": "Token has expired"}
         except DecodeError:
             # raise InvalidToken("token format error") from e
-            return {
-                "error": "Token format error"
-            }
+            return {"error": "Token format error"}
         with local_session() as session:
             user = session.query(User).filter_by(id=payload.user_id).first()
             if not user:
                 # raise Exception("user not exist")
-                return {
-                    "error": "User does not exist"
-                }
+                return {"error": "User does not exist"}
             if not user.emailConfirmed:
                 user.emailConfirmed = True
                 session.commit()

auth/jwtcodec.py

@@ -1,8 +1,10 @@
 from datetime import datetime, timezone
+
 import jwt
+
 from base.exceptions import ExpiredToken, InvalidToken
-from validations.auth import TokenPayload, AuthInput
 from settings import JWT_ALGORITHM, JWT_SECRET_KEY
+from validations.auth import AuthInput, TokenPayload
 
 
 class JWTCodec:
@@ -13,12 +15,12 @@ class JWTCodec:
             "username": user.email or user.phone,
             "exp": exp,
             "iat": datetime.now(tz=timezone.utc),
-            "iss": "discours"
+            "iss": "discours",
         }
         try:
             return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM)
         except Exception as e:
-            print('[auth.jwtcodec] JWT encode error %r' % e)
+            print("[auth.jwtcodec] JWT encode error %r" % e)
 
     @staticmethod
     def decode(token: str, verify_exp: bool = True) -> TokenPayload:
@@ -33,18 +35,18 @@ class JWTCodec:
                     # "verify_signature": False
                 },
                 algorithms=[JWT_ALGORITHM],
-                issuer="discours"
+                issuer="discours",
             )
             r = TokenPayload(**payload)
             # print('[auth.jwtcodec] debug token %r' % r)
             return r
         except jwt.InvalidIssuedAtError:
-            print('[auth.jwtcodec] invalid issued at: %r' % payload)
-            raise ExpiredToken('check token issued time')
+            print("[auth.jwtcodec] invalid issued at: %r" % payload)
+            raise ExpiredToken("check token issued time")
         except jwt.ExpiredSignatureError:
-            print('[auth.jwtcodec] expired signature %r' % payload)
-            raise ExpiredToken('check token lifetime')
+            print("[auth.jwtcodec] expired signature %r" % payload)
+            raise ExpiredToken("check token lifetime")
         except jwt.InvalidTokenError:
-            raise InvalidToken('token is not valid')
+            raise InvalidToken("token is not valid")
         except jwt.InvalidSignatureError:
-            raise InvalidToken('token is not valid')
+            raise InvalidToken("token is not valid")

auth/oauth.py

@@ -1,8 +1,9 @@
 from authlib.integrations.starlette_client import OAuth
 from starlette.responses import RedirectResponse
+
 from auth.identity import Identity
 from auth.tokenstorage import TokenStorage
-from settings import OAUTH_CLIENTS, FRONTEND_URL
+from settings import FRONTEND_URL, OAUTH_CLIENTS
 
 oauth = OAuth()
 

auth/tokenstorage.py

@@ -1,9 +1,9 @@
 from datetime import datetime, timedelta, timezone
 
 from auth.jwtcodec import JWTCodec
-from validations.auth import AuthInput
 from base.redis import redis
-from settings import SESSION_TOKEN_LIFE_SPAN, ONETIME_TOKEN_LIFE_SPAN
+from settings import ONETIME_TOKEN_LIFE_SPAN, SESSION_TOKEN_LIFE_SPAN
+from validations.auth import AuthInput
 
 
 async def save(token_key, life_span, auto_delete=True):
@@ -35,7 +35,7 @@ class SessionToken:
 class TokenStorage:
     @staticmethod
     async def get(token_key):
-        print('[tokenstorage.get] ' + token_key)
+        print("[tokenstorage.get] " + token_key)
         # 2041-user@domain.zn-eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoyMDQxLCJ1c2VybmFtZSI6ImFudG9uLnJld2luK3Rlc3QtbG9hZGNoYXRAZ21haWwuY29tIiwiZXhwIjoxNjcxNzgwNjE2LCJpYXQiOjE2NjkxODg2MTYsImlzcyI6ImRpc2NvdXJzIn0.Nml4oV6iMjMmc6xwM7lTKEZJKBXvJFEIZ-Up1C1rITQ
         return await redis.execute("GET", token_key)
 

base/exceptions.py

@@ -1,8 +1,8 @@
 from graphql.error import GraphQLError
 
-
 # TODO: remove traceback from logs for defined exceptions
 
+
 class BaseHttpException(GraphQLError):
     code = 500
     message = "500 Server error"

base/orm.py

@@ -1,15 +1,13 @@
-from typing import TypeVar, Any, Dict, Generic, Callable
+from typing import Any, Callable, Dict, Generic, TypeVar
 
-from sqlalchemy import create_engine, Column, Integer
+from sqlalchemy import Column, Integer, create_engine
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import Session
 from sqlalchemy.sql.schema import Table
 
 from settings import DB_URL
 
-engine = create_engine(
-    DB_URL, echo=False, pool_size=10, max_overflow=20
-)
+engine = create_engine(DB_URL, echo=False, pool_size=10, max_overflow=20)
 
 T = TypeVar("T")
 
@@ -20,7 +18,10 @@ def local_session():
     return Session(bind=engine, expire_on_commit=False)
 
 
-class Base(declarative_base()):
+DeclarativeBase = declarative_base()  # type: Any
+
+
+class Base(DeclarativeBase):
     __table__: Table
     __tablename__: str
     __new__: Callable
@@ -29,7 +30,7 @@ class Base(declarative_base()):
     __abstract__ = True
     __table_args__ = {"extend_existing": True}
 
-    id = Column(Integer, primary_key=True)
+    id: Column | None = Column(Integer, primary_key=True)
 
     def __init_subclass__(cls, **kwargs):
         REGISTRY[cls.__name__] = cls
@@ -47,7 +48,7 @@ class Base(declarative_base()):
 
     def update(self, input):
         column_names = self.__table__.columns.keys()
-        for (name, value) in input.items():
+        for name, value in input.items():
             if name in column_names:
                 setattr(self, name, value)
 

base/redis.py

@@ -1,5 +1,7 @@
-from aioredis import from_url
 from asyncio import sleep
+
+from aioredis import from_url
+
 from settings import REDIS_URL
 
 

generate_gql_types.sh

@@ -0,0 +1 @@
+python -m gql_schema_codegen -p ./schema.graphql -t ./schema_types.py

lint.sh

@@ -1,16 +1,10 @@
 #!/usr/bin/env bash
-set -e
-
-find . -name "*.py[co]" -o -name __pycache__ -exec rm -rf {} +
-#rm -rf .mypy_cache
 
 echo "> isort"
-isort --gitignore --settings-file=setup.cfg .
-echo "> brunette"
-brunette --config=setup.cfg .
+isort .
+echo "> black"
+black .
 echo "> flake8"
-flake8 --config=setup.cfg .
-echo "> mypy"
-mypy --config-file=setup.cfg .
-echo "> prettyjson"
-python3 -m scripts.prettyjson
+flake8 .
+# echo "> mypy"
+# mypy .

main.py

@@ -2,6 +2,7 @@ import asyncio
 import os
 from importlib import import_module
 from os.path import exists
+
 from ariadne import load_schema_from_path, make_executable_schema
 from ariadne.asgi import GraphQL
 from starlette.applications import Starlette
@@ -9,23 +10,24 @@ from starlette.middleware import Middleware
 from starlette.middleware.authentication import AuthenticationMiddleware
 from starlette.middleware.sessions import SessionMiddleware
 from starlette.routing import Route
-from orm import init_tables
 
 from auth.authenticate import JWTAuthenticate
-from auth.oauth import oauth_login, oauth_authorize
+from auth.oauth import oauth_authorize, oauth_login
 from base.redis import redis
 from base.resolvers import resolvers
+from orm import init_tables
 from resolvers.auth import confirm_email_handler
 from resolvers.upload import upload_handler
 from services.main import storages_init
 from services.notifications.notification_service import notification_service
+from services.notifications.sse import sse_subscribe_handler
 from services.stat.viewed import ViewedStorage
+
 # from services.zine.gittask import GitTask
 from settings import DEV_SERVER_PID_FILE_NAME, SENTRY_DSN, SESSION_SECRET_KEY
-from services.notifications.sse import sse_subscribe_handler
 
 import_module("resolvers")
-schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers)  # type: ignore
+schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers)
 
 middleware = [
     Middleware(AuthenticationMiddleware, backend=JWTAuthenticate()),
@@ -46,9 +48,10 @@ async def start_up():
 
     try:
         import sentry_sdk
+
         sentry_sdk.init(SENTRY_DSN)
     except Exception as e:
-        print('[sentry] init error')
+        print("[sentry] init error")
         print(e)
 
 
@@ -57,7 +60,7 @@ async def dev_start_up():
         await redis.connect()
         return
     else:
-        with open(DEV_SERVER_PID_FILE_NAME, 'w', encoding='utf-8') as f:
+        with open(DEV_SERVER_PID_FILE_NAME, "w", encoding="utf-8") as f:
             f.write(str(os.getpid()))
 
     await start_up()
@@ -72,7 +75,7 @@ routes = [
     Route("/oauth/{provider}", endpoint=oauth_login),
     Route("/oauth-authorize", endpoint=oauth_authorize),
     Route("/confirm/{token}", endpoint=confirm_email_handler),
-    Route("/upload", endpoint=upload_handler, methods=['POST']),
+    Route("/upload", endpoint=upload_handler, methods=["POST"]),
     Route("/subscribe/{user_id}", endpoint=sse_subscribe_handler),
 ]
 
@@ -82,9 +85,7 @@ app = Starlette(
     middleware=middleware,
     routes=routes,
 )
-app.mount("/", GraphQL(
-    schema
-))
+app.mount("/", GraphQL(schema))
 
 dev_app = Starlette(
     debug=True,
@@ -93,7 +94,4 @@ dev_app = Starlette(
     middleware=middleware,
     routes=routes,
 )
-dev_app.mount("/", GraphQL(
-    schema,
-    debug=True
-))
+dev_app.mount("/", GraphQL(schema, debug=True))

migrate.sh

@@ -16,4 +16,3 @@ echo "Start migration"
 python3 server.py migrate
 if [ $? -ne 0 ]; then { echo "Migration failed, aborting." ; exit 1; } fi
 echo 'Done!'
-

migration/__init__.py

@@ -12,10 +12,12 @@ from migration.tables.comments import migrate as migrateComment
 from migration.tables.comments import migrate_2stage as migrateComment_2stage
 from migration.tables.content_items import get_shout_slug
 from migration.tables.content_items import migrate as migrateShout
-from migration.tables.remarks import migrate as migrateRemark
+
+# from migration.tables.remarks import migrate as migrateRemark
 from migration.tables.topics import migrate as migrateTopic
-from migration.tables.users import migrate as migrateUser, post_migrate as users_post_migrate
+from migration.tables.users import migrate as migrateUser
 from migration.tables.users import migrate_2stage as migrateUser_2stage
+from migration.tables.users import post_migrate as users_post_migrate
 from orm import init_tables
 from orm.reaction import Reaction
 
@@ -63,16 +65,8 @@ async def topics_handle(storage):
             del storage["topics"]["by_slug"][oldslug]
             storage["topics"]["by_oid"][oid] = storage["topics"]["by_slug"][newslug]
     print("[migration] " + str(counter) + " topics migrated")
-    print(
-        "[migration] "
-        + str(len(storage["topics"]["by_oid"].values()))
-        + " topics by oid"
-    )
-    print(
-        "[migration] "
-        + str(len(storage["topics"]["by_slug"].values()))
-        + " topics by slug"
-    )
+    print("[migration] " + str(len(storage["topics"]["by_oid"].values())) + " topics by oid")
+    print("[migration] " + str(len(storage["topics"]["by_slug"].values())) + " topics by slug")
 
 
 async def shouts_handle(storage, args):
@@ -117,9 +111,10 @@ async def shouts_handle(storage, args):
 
             # print main counter
             counter += 1
-            print('[migration] shouts_handle %d: %s @%s' % (
-                (counter + 1), shout_dict["slug"], author["slug"]
-            ))
+            print(
+                "[migration] shouts_handle %d: %s @%s"
+                % ((counter + 1), shout_dict["slug"], author["slug"])
+            )
 
             b = bs4.BeautifulSoup(shout_dict["body"], "html.parser")
             texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")]
@@ -138,13 +133,13 @@ async def shouts_handle(storage, args):
     print("[migration] " + str(anonymous_author) + " authored by @anonymous")
 
 
-async def remarks_handle(storage):
-    print("[migration] comments")
-    c = 0
-    for entry_remark in storage["remarks"]["data"]:
-        remark = await migrateRemark(entry_remark, storage)
-        c += 1
-    print("[migration] " + str(c) + " remarks migrated")
+# async def remarks_handle(storage):
+#     print("[migration] comments")
+#     c = 0
+#     for entry_remark in storage["remarks"]["data"]:
+#         remark = await migrateRemark(entry_remark, storage)
+#         c += 1
+#     print("[migration] " + str(c) + " remarks migrated")
 
 
 async def comments_handle(storage):
@@ -155,9 +150,9 @@ async def comments_handle(storage):
     for oldcomment in storage["reactions"]["data"]:
         if not oldcomment.get("deleted"):
             reaction = await migrateComment(oldcomment, storage)
-            if type(reaction) == str:
+            if isinstance(reaction, str):
                 missed_shouts[reaction] = oldcomment
-            elif type(reaction) == Reaction:
+            elif isinstance(reaction, Reaction):
                 reaction = reaction.dict()
                 rid = reaction["id"]
                 oid = reaction["oid"]
@@ -214,9 +209,7 @@ def data_load():
         tags_data = json.loads(open("migration/data/tags.json").read())
         storage["topics"]["tags"] = tags_data
         print("[migration.load] " + str(len(tags_data)) + " tags ")
-        cats_data = json.loads(
-            open("migration/data/content_item_categories.json").read()
-        )
+        cats_data = json.loads(open("migration/data/content_item_categories.json").read())
         storage["topics"]["cats"] = cats_data
         print("[migration.load] " + str(len(cats_data)) + " cats ")
         comments_data = json.loads(open("migration/data/comments.json").read())
@@ -235,11 +228,7 @@ def data_load():
             storage["users"]["by_oid"][x["_id"]] = x
             # storage['users']['by_slug'][x['slug']] = x
         # no user.slug yet
-        print(
-            "[migration.load] "
-            + str(len(storage["users"]["by_oid"].keys()))
-            + " users by oid"
-        )
+        print("[migration.load] " + str(len(storage["users"]["by_oid"].keys())) + " users by oid")
         for x in tags_data:
             storage["topics"]["by_oid"][x["_id"]] = x
             storage["topics"]["by_slug"][x["slug"]] = x
@@ -247,9 +236,7 @@ def data_load():
             storage["topics"]["by_oid"][x["_id"]] = x
             storage["topics"]["by_slug"][x["slug"]] = x
         print(
-            "[migration.load] "
-            + str(len(storage["topics"]["by_slug"].keys()))
-            + " topics by slug"
+            "[migration.load] " + str(len(storage["topics"]["by_slug"].keys())) + " topics by slug"
         )
         for item in content_data:
             slug = get_shout_slug(item)

migration/bson2json.py

@@ -1,8 +1,9 @@
+import gc
 import json
 import os
 
 import bson
-import gc
+
 from .utils import DateTimeEncoder
 
 
@@ -15,10 +16,10 @@ def json_tables():
         "email_subscriptions": [],
         "users": [],
         "comments": [],
-        "remarks": []
+        "remarks": [],
     }
     for table in data.keys():
-        print('[migration] bson2json for ' + table)
+        print("[migration] bson2json for " + table)
         gc.collect()
         lc = []
         bs = open("dump/discours/" + table + ".bson", "rb").read()

migration/export.py

@@ -71,47 +71,29 @@ def export_slug(slug, storage):
 
 
 def export_email_subscriptions():
-    email_subscriptions_data = json.loads(
-        open("migration/data/email_subscriptions.json").read()
-    )
+    email_subscriptions_data = json.loads(open("migration/data/email_subscriptions.json").read())
     for data in email_subscriptions_data:
         # TODO: migrate to mailgun list manually
         # migrate_email_subscription(data)
         pass
-    print(
-        "[migration] "
-        + str(len(email_subscriptions_data))
-        + " email subscriptions exported"
-    )
+    print("[migration] " + str(len(email_subscriptions_data)) + " email subscriptions exported")
 
 
 def export_shouts(storage):
     # update what was just migrated or load json again
     if len(storage["users"]["by_slugs"].keys()) == 0:
-        storage["users"]["by_slugs"] = json.loads(
-            open(EXPORT_DEST + "authors.json").read()
-        )
-        print(
-            "[migration] "
-            + str(len(storage["users"]["by_slugs"].keys()))
-            + " exported authors "
-        )
+        storage["users"]["by_slugs"] = json.loads(open(EXPORT_DEST + "authors.json").read())
+        print("[migration] " + str(len(storage["users"]["by_slugs"].keys())) + " exported authors ")
     if len(storage["shouts"]["by_slugs"].keys()) == 0:
-        storage["shouts"]["by_slugs"] = json.loads(
-            open(EXPORT_DEST + "articles.json").read()
-        )
+        storage["shouts"]["by_slugs"] = json.loads(open(EXPORT_DEST + "articles.json").read())
         print(
-            "[migration] "
-            + str(len(storage["shouts"]["by_slugs"].keys()))
-            + " exported articles "
+            "[migration] " + str(len(storage["shouts"]["by_slugs"].keys())) + " exported articles "
         )
     for slug in storage["shouts"]["by_slugs"].keys():
         export_slug(slug, storage)
 
 
-def export_json(
-    export_articles={}, export_authors={}, export_topics={}, export_comments={}
-):
+def export_json(export_articles={}, export_authors={}, export_topics={}, export_comments={}):
     open(EXPORT_DEST + "authors.json", "w").write(
         json.dumps(
             export_authors,
@@ -152,8 +134,4 @@ def export_json(
             ensure_ascii=False,
         )
     )
-    print(
-        "[migration] "
-        + str(len(export_comments.items()))
-        + " exported articles with comments"
-    )
+    print("[migration] " + str(len(export_comments.items())) + " exported articles with comments")

migration/extract.py

@@ -1,11 +1,8 @@
-import base64
 import os
 import re
-import uuid
 
 from bs4 import BeautifulSoup
 
-
 TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)"
 contentDir = os.path.join(
     os.path.dirname(os.path.realpath(__file__)), "..", "..", "discoursio-web", "content"
@@ -27,76 +24,79 @@ def replace_tooltips(body):
     return newbody
 
 
-
-def extract_footnotes(body, shout_dict):
-    parts = body.split("&&&")
-    lll = len(parts)
-    newparts = list(parts)
-    placed = False
-    if lll & 1:
-        if lll > 1:
-            i = 1
-            print("[extract] found %d footnotes in body" % (lll - 1))
-            for part in parts[1:]:
-                if i & 1:
-                    placed = True
-                    if 'a class="footnote-url" href=' in part:
-                        print("[extract] footnote: " + part)
-                        fn = 'a class="footnote-url" href="'
-                        exxtracted_link = part.split(fn, 1)[1].split('"', 1)[0]
-                        extracted_body = part.split(fn, 1)[1].split('>', 1)[1].split('</a>', 1)[0]
-                        print("[extract] footnote link: " + extracted_link)
-                        with local_session() as session:
-                            Reaction.create({
-                                "shout": shout_dict['id'],
-                                "kind": ReactionKind.FOOTNOTE,
-                                "body": extracted_body,
-                                "range": str(body.index(fn + link) - len('<')) + ':' + str(body.index(extracted_body) + len('</a>'))
-                            })
-                        newparts[i] = "<a href='#'>ℹ️</a>"
-                else:
-                    newparts[i] = part
-                i += 1
-    return ("".join(newparts), placed)
+# def extract_footnotes(body, shout_dict):
+#     parts = body.split("&&&")
+#     lll = len(parts)
+#     newparts = list(parts)
+#     placed = False
+#     if lll & 1:
+#         if lll > 1:
+#             i = 1
+#             print("[extract] found %d footnotes in body" % (lll - 1))
+#             for part in parts[1:]:
+#                 if i & 1:
+#                     placed = True
+#                     if 'a class="footnote-url" href=' in part:
+#                         print("[extract] footnote: " + part)
+#                         fn = 'a class="footnote-url" href="'
+#                         exxtracted_link = part.split(fn, 1)[1].split('"', 1)[0]
+#                         extracted_body = part.split(fn, 1)[1].split(">", 1)[1].split("</a>", 1)[0]
+#                         print("[extract] footnote link: " + extracted_link)
+#                         with local_session() as session:
+#                             Reaction.create(
+#                                 {
+#                                     "shout": shout_dict["id"],
+#                                     "kind": ReactionKind.FOOTNOTE,
+#                                     "body": extracted_body,
+#                                     "range": str(body.index(fn + link) - len("<"))
+#                                     + ":"
+#                                     + str(body.index(extracted_body) + len("</a>")),
+#                                 }
+#                             )
+#                         newparts[i] = "<a href='#'>ℹ️</a>"
+#                 else:
+#                     newparts[i] = part
+#                 i += 1
+#     return ("".join(newparts), placed)
 
 
-def place_tooltips(body):
-    parts = body.split("&&&")
-    lll = len(parts)
-    newparts = list(parts)
-    placed = False
-    if lll & 1:
-        if lll > 1:
-            i = 1
-            print("[extract] found %d tooltips" % (lll - 1))
-            for part in parts[1:]:
-                if i & 1:
-                    placed = True
-                    if 'a class="footnote-url" href=' in part:
-                        print("[extract] footnote: " + part)
-                        fn = 'a class="footnote-url" href="'
-                        link = part.split(fn, 1)[1].split('"', 1)[0]
-                        extracted_part = (
-                            part.split(fn, 1)[0] + " " + part.split("/", 1)[-1]
-                        )
-                        newparts[i] = (
-                            "<Tooltip"
-                            + (' link="' + link + '" ' if link else "")
-                            + ">"
-                            + extracted_part
-                            + "</Tooltip>"
-                        )
-                    else:
-                        newparts[i] = "<Tooltip>%s</Tooltip>" % part
-                        # print('[extract] ' + newparts[i])
-                else:
-                    # print('[extract] ' + part[:10] + '..')
-                    newparts[i] = part
-                i += 1
-    return ("".join(newparts), placed)
+# def place_tooltips(body):
+#     parts = body.split("&&&")
+#     lll = len(parts)
+#     newparts = list(parts)
+#     placed = False
+#     if lll & 1:
+#         if lll > 1:
+#             i = 1
+#             print("[extract] found %d tooltips" % (lll - 1))
+#             for part in parts[1:]:
+#                 if i & 1:
+#                     placed = True
+#                     if 'a class="footnote-url" href=' in part:
+#                         print("[extract] footnote: " + part)
+#                         fn = 'a class="footnote-url" href="'
+#                         link = part.split(fn, 1)[1].split('"', 1)[0]
+#                         extracted_part = part.split(fn, 1)[0] + " " + part.split("/", 1)[-1]
+#                         newparts[i] = (
+#                             "<Tooltip"
+#                             + (' link="' + link + '" ' if link else "")
+#                             + ">"
+#                             + extracted_part
+#                             + "</Tooltip>"
+#                         )
+#                     else:
+#                         newparts[i] = "<Tooltip>%s</Tooltip>" % part
+#                         # print('[extract] ' + newparts[i])
+#                 else:
+#                     # print('[extract] ' + part[:10] + '..')
+#                     newparts[i] = part
+#                 i += 1
+#     return ("".join(newparts), placed)
 
 
-IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,((?:[A-Za-z\d+\/]{4})*(?:[A-Za-z\d+\/]{3}="
+IMG_REGEX = (
+    r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,((?:[A-Za-z\d+\/]{4})*(?:[A-Za-z\d+\/]{3}="
+)
 IMG_REGEX += r"|[A-Za-z\d+\/]{2}==)))\)"
 
 parentDir = "/".join(os.getcwd().split("/")[:-1])
@@ -104,29 +104,29 @@ public = parentDir + "/discoursio-web/public"
 cache = {}
 
 
-def reextract_images(body, oid):
-    # change if you prefer regexp
-    matches = list(re.finditer(IMG_REGEX, body, re.IGNORECASE | re.MULTILINE))[1:]
-    i = 0
-    for match in matches:
-        print("[extract] image " + match.group(1))
-        ext = match.group(3)
-        name = oid + str(i)
-        link = public + "/upload/image-" + name + "." + ext
-        img = match.group(4)
-        title = match.group(1)  # NOTE: this is not the title
-        if img not in cache:
-            content = base64.b64decode(img + "==")
-            print(str(len(img)) + " image bytes been written")
-            open("../" + link, "wb").write(content)
-            cache[img] = name
-            i += 1
-        else:
-            print("[extract] image cached " + cache[img])
-        body.replace(
-            str(match), "![" + title + "](" + cdn + link + ")"
-        )  # WARNING: this does not work
-    return body
+# def reextract_images(body, oid):
+#     # change if you prefer regexp
+#     matches = list(re.finditer(IMG_REGEX, body, re.IGNORECASE | re.MULTILINE))[1:]
+#     i = 0
+#     for match in matches:
+#         print("[extract] image " + match.group(1))
+#         ext = match.group(3)
+#         name = oid + str(i)
+#         link = public + "/upload/image-" + name + "." + ext
+#         img = match.group(4)
+#         title = match.group(1)  # NOTE: this is not the title
+#         if img not in cache:
+#             content = base64.b64decode(img + "==")
+#             print(str(len(img)) + " image bytes been written")
+#             open("../" + link, "wb").write(content)
+#             cache[img] = name
+#             i += 1
+#         else:
+#             print("[extract] image cached " + cache[img])
+#         body.replace(
+#             str(match), "![" + title + "](" + cdn + link + ")"
+#         )  # WARNING: this does not work
+#     return body
 
 
 IMAGES = {
@@ -137,163 +137,11 @@ IMAGES = {
 
 b64 = ";base64,"
 
-
-def extract_imageparts(bodyparts, prefix):
-    # recursive loop
-    newparts = list(bodyparts)
-    for current in bodyparts:
-        i = bodyparts.index(current)
-        for mime in IMAGES.keys():
-            if mime == current[-len(mime) :] and (i + 1 < len(bodyparts)):
-                print("[extract] " + mime)
-                next = bodyparts[i + 1]
-                ext = IMAGES[mime]
-                b64end = next.index(")")
-                b64encoded = next[:b64end]
-                name = prefix + "-" + str(len(cache))
-                link = "/upload/image-" + name + "." + ext
-                print("[extract] name: " + name)
-                print("[extract] link: " + link)
-                print("[extract] %d bytes" % len(b64encoded))
-                if b64encoded not in cache:
-                    try:
-                        content = base64.b64decode(b64encoded + "==")
-                        open(public + link, "wb").write(content)
-                        print(
-                            "[extract] "
-                            + str(len(content))
-                            + " image bytes been written"
-                        )
-                        cache[b64encoded] = name
-                    except Exception:
-                        raise Exception
-                        # raise Exception('[extract] error decoding image %r' %b64encoded)
-                else:
-                    print("[extract] cached link " + cache[b64encoded])
-                    name = cache[b64encoded]
-                    link = cdn + "/upload/image-" + name + "." + ext
-                newparts[i] = (
-                    current[: -len(mime)]
-                    + current[-len(mime) :]
-                    + link
-                    + next[-b64end:]
-                )
-                newparts[i + 1] = next[:-b64end]
-                break
-    return (
-        extract_imageparts(
-            newparts[i] + newparts[i + 1] + b64.join(bodyparts[(i + 2) :]), prefix
-        )
-        if len(bodyparts) > (i + 1)
-        else "".join(newparts)
-    )
-
-
-def extract_dataimages(parts, prefix):
-    newparts = list(parts)
-    for part in parts:
-        i = parts.index(part)
-        if part.endswith("]("):
-            [ext, rest] = parts[i + 1].split(b64)
-            name = prefix + "-" + str(len(cache))
-            if ext == "/jpeg":
-                ext = "jpg"
-            else:
-                ext = ext.replace("/", "")
-            link = "/upload/image-" + name + "." + ext
-            print("[extract] filename: " + link)
-            b64end = rest.find(")")
-            if b64end != -1:
-                b64encoded = rest[:b64end]
-                print("[extract] %d text bytes" % len(b64encoded))
-                # write if not cached
-                if b64encoded not in cache:
-                    try:
-                        content = base64.b64decode(b64encoded + "==")
-                        open(public + link, "wb").write(content)
-                        print("[extract] " + str(len(content)) + " image bytes")
-                        cache[b64encoded] = name
-                    except Exception:
-                        raise Exception
-                        # raise Exception('[extract] error decoding image %r' %b64encoded)
-                else:
-                    print("[extract] 0 image bytes, cached for " + cache[b64encoded])
-                    name = cache[b64encoded]
-
-                # update link with CDN
-                link = cdn + "/upload/image-" + name + "." + ext
-
-                # patch newparts
-                newparts[i + 1] = link + rest[b64end:]
-            else:
-                raise Exception("cannot find the end of base64 encoded string")
-        else:
-            print("[extract] dataimage skipping part " + str(i))
-            continue
-    return "".join(newparts)
-
-
 di = "data:image"
 
 
-def extract_md_images(body, prefix):
-    newbody = ""
-    body = (
-        body.replace("\n! [](" + di, "\n ![](" + di)
-        .replace("\n[](" + di, "\n![](" + di)
-        .replace(" [](" + di, " ![](" + di)
-    )
-    parts = body.split(di)
-    if len(parts) > 1:
-        newbody = extract_dataimages(parts, prefix)
-    else:
-        newbody = body
-    return newbody
-
-
-def cleanup_md(body):
-    newbody = (
-        body.replace("<", "")
-        .replace(">", "")
-        .replace("{", "(")
-        .replace("}", ")")
-        .replace("…", "...")
-        .replace(" __ ", " ")
-        .replace("_ _", " ")
-        .replace("****", "")
-        .replace("\u00a0", " ")
-        .replace("\u02c6", "^")
-        .replace("\u00a0", " ")
-        .replace("\ufeff", "")
-        .replace("\u200b", "")
-        .replace("\u200c", "")
-    )  # .replace('\u2212', '-')
-    return newbody
-
-
-def extract_md(body, shout_dict = None):
-    newbody = body
-    if newbody:
-        newbody = cleanup_md(newbody)
-        if not newbody:
-            raise Exception("cleanup error")
-
-        if shout_dict:
-
-            uid = shout_dict['id'] or uuid.uuid4()
-            newbody = extract_md_images(newbody, uid)
-            if not newbody:
-                raise Exception("extract_images error")
-
-            newbody, placed = extract_footnotes(body, shout_dict)
-            if not newbody:
-                raise Exception("extract_footnotes error")
-
-    return newbody
-
-
 def extract_media(entry):
-    ''' normalized media extraction method '''
+    """normalized media extraction method"""
     # media [ { title pic url body } ]}
     kind = entry.get("type")
     if not kind:
@@ -323,12 +171,7 @@ def extract_media(entry):
                 url = "https://vimeo.com/" + m["vimeoId"]
         # body
         body = m.get("body") or m.get("literatureBody") or ""
-        media.append({
-            "url": url,
-            "pic": pic,
-            "title": title,
-            "body": body
-        })
+        media.append({"url": url, "pic": pic, "title": title, "body": body})
     return media
 
 
@@ -398,9 +241,7 @@ def cleanup_html(body: str) -> str:
         r"<h4>\s*</h4>",
         r"<div>\s*</div>",
     ]
-    regex_replace = {
-        r"<br>\s*</p>": "</p>"
-    }
+    regex_replace = {r"<br>\s*</p>": "</p>"}
     changed = True
     while changed:
         # we need several iterations to clean nested tags this way
@@ -414,16 +255,17 @@ def cleanup_html(body: str) -> str:
             changed = True
     return new_body
 
-def extract_html(entry, shout_id = None, cleanup=False):
-    body_orig = (entry.get("body") or "").replace('\(', '(').replace('\)', ')')
+
+def extract_html(entry, shout_id=None, cleanup=False):
+    body_orig = (entry.get("body") or "").replace(r"\(", "(").replace(r"\)", ")")
     if cleanup:
         # we do that before bs parsing to catch the invalid html
         body_clean = cleanup_html(body_orig)
         if body_clean != body_orig:
             print(f"[migration] html cleaned for slug {entry.get('slug', None)}")
         body_orig = body_clean
-    if shout_id:
-        extract_footnotes(body_orig, shout_id)
+    # if shout_id:
+    #     extract_footnotes(body_orig, shout_id)
     body_html = str(BeautifulSoup(body_orig, features="html.parser"))
     if cleanup:
         # we do that after bs parsing because it can add dummy tags

migration/html2text/__init__.py

@@ -33,7 +33,7 @@ __version__ = (2020, 1, 16)
 # TODO: Support decoded entities with UNIFIABLE.
 
 
-class HTML2Text(html.parser.HTMLParser):
+class HTML2Text(html.parser.HTMLParser):  # noqa: C901
     def __init__(
         self,
         out: Optional[OutCallback] = None,
@@ -85,7 +85,7 @@ class HTML2Text(html.parser.HTMLParser):
         self.tag_callback = None
         self.open_quote = config.OPEN_QUOTE  # covered in cli
         self.close_quote = config.CLOSE_QUOTE  # covered in cli
-        self.header_id = None
+        self.header_id: str | None = None
         self.span_highlight = False
         self.span_lead = False
 
@@ -119,9 +119,7 @@ class HTML2Text(html.parser.HTMLParser):
         self.lastWasList = False
         self.style = 0
         self.style_def = {}  # type: Dict[str, Dict[str, str]]
-        self.tag_stack = (
-            []
-        )  # type: List[Tuple[str, Dict[str, Optional[str]], Dict[str, str]]]
+        self.tag_stack = []  # type: List[Tuple[str, Dict[str, Optional[str]], Dict[str, str]]]
         self.emphasis = 0
         self.drop_white_space = 0
         self.inheader = False
@@ -227,7 +225,7 @@ class HTML2Text(html.parser.HTMLParser):
                 return i
         return None
 
-    def handle_emphasis(
+    def handle_emphasis(  # noqa: C901
         self, start: bool, tag_style: Dict[str, str], parent_style: Dict[str, str]
     ) -> None:
         """
@@ -300,7 +298,7 @@ class HTML2Text(html.parser.HTMLParser):
             if strikethrough:
                 self.quiet -= 1
 
-    def handle_tag(
+    def handle_tag(  # noqa: C901
         self, tag: str, attrs: Dict[str, Optional[str]], start: bool
     ) -> None:
         self.current_tag = tag
@@ -333,9 +331,7 @@ class HTML2Text(html.parser.HTMLParser):
                 tag_style = element_style(attrs, self.style_def, parent_style)
                 self.tag_stack.append((tag, attrs, tag_style))
             else:
-                dummy, attrs, tag_style = (
-                    self.tag_stack.pop() if self.tag_stack else (None, {}, {})
-                )
+                dummy, attrs, tag_style = self.tag_stack.pop() if self.tag_stack else (None, {}, {})
                 if self.tag_stack:
                     parent_style = self.tag_stack[-1][2]
 
@@ -385,11 +381,7 @@ class HTML2Text(html.parser.HTMLParser):
                 ):
                     self.o("`")  # NOTE: same as <code>
                     self.span_highlight = True
-                elif (
-                    self.current_class == "lead"
-                    and not self.inheader
-                    and not self.span_highlight
-                ):
+                elif self.current_class == "lead" and not self.inheader and not self.span_highlight:
                     # self.o("==") # NOTE:  CriticMarkup {==
                     self.span_lead = True
             else:
@@ -479,11 +471,7 @@ class HTML2Text(html.parser.HTMLParser):
                 and not self.span_lead
                 and not self.span_highlight
             ):
-                if (
-                    start
-                    and self.preceding_data
-                    and self.preceding_data[-1] == self.strong_mark[0]
-                ):
+                if start and self.preceding_data and self.preceding_data[-1] == self.strong_mark[0]:
                     strong = " " + self.strong_mark
                     self.preceding_data += " "
                 else:
@@ -548,13 +536,8 @@ class HTML2Text(html.parser.HTMLParser):
                         "href" in attrs
                         and not attrs["href"].startswith("#_ftn")
                         and attrs["href"] is not None
-                        and not (
-                            self.skip_internal_links and attrs["href"].startswith("#")
-                        )
-                        and not (
-                            self.ignore_mailto_links
-                            and attrs["href"].startswith("mailto:")
-                        )
+                        and not (self.skip_internal_links and attrs["href"].startswith("#"))
+                        and not (self.ignore_mailto_links and attrs["href"].startswith("mailto:"))
                     ):
                         self.astack.append(attrs)
                         self.maybe_automatic_link = attrs["href"]
@@ -591,7 +574,7 @@ class HTML2Text(html.parser.HTMLParser):
 
         if tag == "img" and start and not self.ignore_images:
             # skip cloudinary images
-            if "src" in attrs and "cloudinary" not in attrs["src"]:
+            if "src" in attrs and ("cloudinary" not in attrs["src"]):
                 assert attrs["src"] is not None
                 if not self.images_to_alt:
                     attrs["href"] = attrs["src"]
@@ -638,9 +621,7 @@ class HTML2Text(html.parser.HTMLParser):
                     self.o("![" + escape_md(alt) + "]")
                     if self.inline_links:
                         href = attrs.get("href") or ""
-                        self.o(
-                            "(" + escape_md(urlparse.urljoin(self.baseurl, href)) + ")"
-                        )
+                        self.o("(" + escape_md(urlparse.urljoin(self.baseurl, href)) + ")")
                     else:
                         i = self.previousIndex(attrs)
                         if i is not None:
@@ -696,9 +677,7 @@ class HTML2Text(html.parser.HTMLParser):
                     # WARNING: does not line up <ol><li>s > 9 correctly.
                     parent_list = None
                     for list in self.list:
-                        self.o(
-                            "   " if parent_list == "ol" and list.name == "ul" else "  "
-                        )
+                        self.o("   " if parent_list == "ol" and list.name == "ul" else "  ")
                         parent_list = list.name
 
                 if li.name == "ul":
@@ -787,7 +766,7 @@ class HTML2Text(html.parser.HTMLParser):
         self.pbr()
         self.br_toggle = "  "
 
-    def o(
+    def o(  # noqa: C901
         self, data: str, puredata: bool = False, force: Union[bool, str] = False
     ) -> None:
         """
@@ -864,9 +843,7 @@ class HTML2Text(html.parser.HTMLParser):
                     self.out(" ")
                 self.space = False
 
-            if self.a and (
-                (self.p_p == 2 and self.links_each_paragraph) or force == "end"
-            ):
+            if self.a and ((self.p_p == 2 and self.links_each_paragraph) or force == "end"):
                 if force == "end":
                     self.out("\n")
 
@@ -925,11 +902,7 @@ class HTML2Text(html.parser.HTMLParser):
 
         if self.maybe_automatic_link is not None:
             href = self.maybe_automatic_link
-            if (
-                href == data
-                and self.absolute_url_matcher.match(href)
-                and self.use_automatic_links
-            ):
+            if href == data and self.absolute_url_matcher.match(href) and self.use_automatic_links:
                 self.o("<" + data + ">")
                 self.empty_link = False
                 return
@@ -980,7 +953,7 @@ class HTML2Text(html.parser.HTMLParser):
 
         return nest_count
 
-    def optwrap(self, text: str) -> str:
+    def optwrap(self, text: str) -> str:  # noqa: C901
         """
         Wrap all paragraphs in the provided text.
 
@@ -1000,9 +973,7 @@ class HTML2Text(html.parser.HTMLParser):
             self.inline_links = False
         for para in text.split("\n"):
             if len(para) > 0:
-                if not skipwrap(
-                    para, self.wrap_links, self.wrap_list_items, self.wrap_tables
-                ):
+                if not skipwrap(para, self.wrap_links, self.wrap_list_items, self.wrap_tables):
                     indent = ""
                     if para.startswith("  " + self.ul_item_mark):
                         # list item continuation: add a double indent to the
@@ -1043,12 +1014,10 @@ class HTML2Text(html.parser.HTMLParser):
         return result
 
 
-def html2text(
-    html: str, baseurl: str = "", bodywidth: Optional[int] = config.BODY_WIDTH
-) -> str:
+def html2text(html: str, baseurl: str = "", bodywidth: int = config.BODY_WIDTH) -> str:
     h = html.strip() or ""
     if h:
-        h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
-        h = h.handle(html.strip())
+        h2t = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
+        h = h2t.handle(html.strip())
         # print('[html2text] %d bytes' % len(html))
     return h

migration/html2text/cli.py

@@ -117,10 +117,7 @@ def main() -> None:
         dest="images_with_size",
         action="store_true",
         default=config.IMAGES_WITH_SIZE,
-        help=(
-            "Write image tags with height and width attrs as raw html to retain "
-            "dimensions"
-        ),
+        help=("Write image tags with height and width attrs as raw html to retain " "dimensions"),
     )
     p.add_argument(
         "-g",
@@ -260,9 +257,7 @@ def main() -> None:
         default=config.CLOSE_QUOTE,
         help="The character used to close quotes",
     )
-    p.add_argument(
-        "--version", action="version", version=".".join(map(str, __version__))
-    )
+    p.add_argument("--version", action="version", version=".".join(map(str, __version__)))
     p.add_argument("filename", nargs="?")
     p.add_argument("encoding", nargs="?", default="utf-8")
     args = p.parse_args()

migration/html2text/utils.py

@@ -4,9 +4,7 @@ from typing import Dict, List, Optional
 from . import config
 
 unifiable_n = {
-    html.entities.name2codepoint[k]: v
-    for k, v in config.UNIFIABLE.items()
-    if k != "nbsp"
+    html.entities.name2codepoint[k]: v for k, v in config.UNIFIABLE.items() if k != "nbsp"
 }
 
 
@@ -68,12 +66,14 @@ def element_style(
     :rtype: dict
     """
     style = parent_style.copy()
-    if attrs.get("class"):
-        for css_class in attrs["class"].split():
+    attrs_class = attrs.get("class")
+    if attrs_class:
+        for css_class in attrs_class.split():
             css_style = style_def.get("." + css_class, {})
             style.update(css_style)
-    if attrs.get("style"):
-        immediate_style = dumb_property_dict(attrs["style"])
+    attrs_style = attrs.get("style")
+    if attrs_style:
+        immediate_style = dumb_property_dict(attrs_style)
         style.update(immediate_style)
 
     return style
@@ -147,18 +147,17 @@ def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
 
     :rtype: int or None
     """
-    if attrs.get("start"):
+    attrs_start = attrs.get("start")
+    if attrs_start:
         try:
-            return int(attrs["start"]) - 1
+            return int(attrs_start) - 1
         except ValueError:
             pass
 
     return 0
 
 
-def skipwrap(
-    para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool
-) -> bool:
+def skipwrap(para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool) -> bool:
     # If it appears to contain a link
     # don't wrap
     if not wrap_links and config.RE_LINK.search(para):
@@ -236,9 +235,7 @@ def reformat_table(lines: List[str], right_margin: int) -> List[str]:
             max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
             max_cols = num_cols
 
-        max_width = [
-            max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
-        ]
+        max_width = [max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)]
 
     # reformat
     new_lines = []
@@ -247,15 +244,13 @@ def reformat_table(lines: List[str], right_margin: int) -> List[str]:
         if set(line.strip()) == set("-|"):
             filler = "-"
             new_cols = [
-                x.rstrip() + (filler * (M - len(x.rstrip())))
-                for x, M in zip(cols, max_width)
+                x.rstrip() + (filler * (M - len(x.rstrip()))) for x, M in zip(cols, max_width)
             ]
             new_lines.append("|-" + "|".join(new_cols) + "|")
         else:
             filler = " "
             new_cols = [
-                x.rstrip() + (filler * (M - len(x.rstrip())))
-                for x, M in zip(cols, max_width)
+                x.rstrip() + (filler * (M - len(x.rstrip()))) for x, M in zip(cols, max_width)
             ]
             new_lines.append("| " + "|".join(new_cols) + "|")
     return new_lines

migration/tables/__init__.py

@@ -1 +0,0 @@
-__all__ = (["users", "topics", "content_items", "comments"],)

migration/tables/comments.py

@@ -5,61 +5,48 @@ from dateutil.parser import parse as date_parse
 from base.orm import local_session
 from migration.html2text import html2text
 from orm.reaction import Reaction, ReactionKind
-from orm.shout import ShoutReactionsFollower
+from orm.shout import Shout, ShoutReactionsFollower
 from orm.topic import TopicFollower
 from orm.user import User
-from orm.shout import Shout
 
 ts = datetime.now(tz=timezone.utc)
 
 
 def auto_followers(session, topics, reaction_dict):
     # creating shout's reactions following for reaction author
-    following1 = session.query(
-        ShoutReactionsFollower
-    ).where(
-        ShoutReactionsFollower.follower == reaction_dict["createdBy"]
-    ).filter(
-        ShoutReactionsFollower.shout == reaction_dict["shout"]
-    ).first()
+    following1 = (
+        session.query(ShoutReactionsFollower)
+        .where(ShoutReactionsFollower.follower == reaction_dict["createdBy"])
+        .filter(ShoutReactionsFollower.shout == reaction_dict["shout"])
+        .first()
+    )
     if not following1:
         following1 = ShoutReactionsFollower.create(
-            follower=reaction_dict["createdBy"],
-            shout=reaction_dict["shout"],
-            auto=True
+            follower=reaction_dict["createdBy"], shout=reaction_dict["shout"], auto=True
         )
         session.add(following1)
     # creating topics followings for reaction author
     for t in topics:
-        tf = session.query(
-            TopicFollower
-        ).where(
-            TopicFollower.follower == reaction_dict["createdBy"]
-        ).filter(
-            TopicFollower.topic == t['id']
-        ).first()
+        tf = (
+            session.query(TopicFollower)
+            .where(TopicFollower.follower == reaction_dict["createdBy"])
+            .filter(TopicFollower.topic == t["id"])
+            .first()
+        )
         if not tf:
             topic_following = TopicFollower.create(
-                follower=reaction_dict["createdBy"],
-                topic=t['id'],
-                auto=True
+                follower=reaction_dict["createdBy"], topic=t["id"], auto=True
             )
             session.add(topic_following)
 
 
 def migrate_ratings(session, entry, reaction_dict):
     for comment_rating_old in entry.get("ratings", []):
-        rater = (
-            session.query(User)
-            .filter(User.oid == comment_rating_old["createdBy"])
-            .first()
-        )
+        rater = session.query(User).filter(User.oid == comment_rating_old["createdBy"]).first()
         re_reaction_dict = {
             "shout": reaction_dict["shout"],
             "replyTo": reaction_dict["id"],
-            "kind": ReactionKind.LIKE
-            if comment_rating_old["value"] > 0
-            else ReactionKind.DISLIKE,
+            "kind": ReactionKind.LIKE if comment_rating_old["value"] > 0 else ReactionKind.DISLIKE,
             "createdBy": rater.id if rater else 1,
         }
         cts = comment_rating_old.get("createdAt")
@@ -68,18 +55,15 @@ def migrate_ratings(session, entry, reaction_dict):
         try:
             # creating reaction from old rating
             rr = Reaction.create(**re_reaction_dict)
-            following2 = session.query(
-                ShoutReactionsFollower
-            ).where(
-                ShoutReactionsFollower.follower == re_reaction_dict['createdBy']
-            ).filter(
-                ShoutReactionsFollower.shout == rr.shout
-            ).first()
+            following2 = (
+                session.query(ShoutReactionsFollower)
+                .where(ShoutReactionsFollower.follower == re_reaction_dict["createdBy"])
+                .filter(ShoutReactionsFollower.shout == rr.shout)
+                .first()
+            )
             if not following2:
                 following2 = ShoutReactionsFollower.create(
-                    follower=re_reaction_dict['createdBy'],
-                    shout=rr.shout,
-                    auto=True
+                    follower=re_reaction_dict["createdBy"], shout=rr.shout, auto=True
                 )
                 session.add(following2)
             session.add(rr)
@@ -150,9 +134,7 @@ async def migrate(entry, storage):
             else:
                 stage = "author and old id found"
                 try:
-                    shout = session.query(
-                        Shout
-                    ).where(Shout.slug == old_shout["slug"]).one()
+                    shout = session.query(Shout).where(Shout.slug == old_shout["slug"]).one()
                     if shout:
                         reaction_dict["shout"] = shout.id
                         reaction_dict["createdBy"] = author.id if author else 1
@@ -178,9 +160,9 @@ async def migrate(entry, storage):
 
 
 def migrate_2stage(old_comment, idmap):
-    if old_comment.get('body'):
-        new_id = idmap.get(old_comment.get('oid'))
-        new_id = idmap.get(old_comment.get('_id'))
+    if old_comment.get("body"):
+        new_id = idmap.get(old_comment.get("oid"))
+        new_id = idmap.get(old_comment.get("_id"))
         if new_id:
             new_replyto_id = None
             old_replyto_id = old_comment.get("replyTo")
@@ -190,17 +172,20 @@ def migrate_2stage(old_comment, idmap):
                 comment = session.query(Reaction).where(Reaction.id == new_id).first()
                 try:
                     if new_replyto_id:
-                        new_reply = session.query(Reaction).where(Reaction.id == new_replyto_id).first()
+                        new_reply = (
+                            session.query(Reaction).where(Reaction.id == new_replyto_id).first()
+                        )
                         if not new_reply:
                             print(new_replyto_id)
                             raise Exception("cannot find reply by id!")
                         comment.replyTo = new_reply.id
                         session.add(comment)
-                    srf = session.query(ShoutReactionsFollower).where(
-                        ShoutReactionsFollower.shout == comment.shout
-                    ).filter(
-                        ShoutReactionsFollower.follower == comment.createdBy
-                    ).first()
+                    srf = (
+                        session.query(ShoutReactionsFollower)
+                        .where(ShoutReactionsFollower.shout == comment.shout)
+                        .filter(ShoutReactionsFollower.follower == comment.createdBy)
+                        .first()
+                    )
                     if not srf:
                         srf = ShoutReactionsFollower.create(
                             shout=comment.shout, follower=comment.createdBy, auto=True

migration/tables/content_items.py

@@ -1,16 +1,18 @@
-from datetime import datetime, timezone
 import json
+import re
+from datetime import datetime, timezone
+
 from dateutil.parser import parse as date_parse
 from sqlalchemy.exc import IntegrityError
 from transliterate import translit
+
 from base.orm import local_session
 from migration.extract import extract_html, extract_media
 from orm.reaction import Reaction, ReactionKind
-from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
+from orm.shout import Shout, ShoutReactionsFollower, ShoutTopic
+from orm.topic import Topic, TopicFollower
 from orm.user import User
-from orm.topic import TopicFollower, Topic
 from services.stat.viewed import ViewedStorage
-import re
 
 OLD_DATE = "2016-03-05 22:22:00.350000"
 ts = datetime.now(tz=timezone.utc)
@@ -33,7 +35,7 @@ def get_shout_slug(entry):
             slug = friend.get("slug", "")
             if slug:
                 break
-    slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+    slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
     return slug
 
 
@@ -41,27 +43,27 @@ def create_author_from_app(app):
     user = None
     userdata = None
     # check if email is used
-    if app['email']:
+    if app["email"]:
         with local_session() as session:
-            user = session.query(User).where(User.email == app['email']).first()
+            user = session.query(User).where(User.email == app["email"]).first()
             if not user:
                 # print('[migration] app %r' % app)
-                name = app.get('name')
+                name = app.get("name")
                 if name:
                     slug = translit(name, "ru", reversed=True).lower()
-                    slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
-                    print('[migration] created slug %s' % slug)
+                    slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
+                    print("[migration] created slug %s" % slug)
                     # check if slug is used
                     if slug:
                         user = session.query(User).where(User.slug == slug).first()
 
                         # get slug from email
                         if user:
-                            slug = app['email'].split('@')[0]
+                            slug = app["email"].split("@")[0]
                             user = session.query(User).where(User.slug == slug).first()
                             # one more try
                             if user:
-                                slug += '-author'
+                                slug += "-author"
                                 user = session.query(User).where(User.slug == slug).first()
 
                 # create user with application data
@@ -79,7 +81,7 @@ def create_author_from_app(app):
                     user = User.create(**userdata)
                     session.add(user)
                     session.commit()
-                    userdata['id'] = user.id
+                    userdata["id"] = user.id
 
             userdata = user.dict()
         return userdata
@@ -91,11 +93,12 @@ async def create_shout(shout_dict):
     s = Shout.create(**shout_dict)
     author = s.authors[0]
     with local_session() as session:
-        srf = session.query(ShoutReactionsFollower).where(
-            ShoutReactionsFollower.shout == s.id
-        ).filter(
-            ShoutReactionsFollower.follower == author.id
-        ).first()
+        srf = (
+            session.query(ShoutReactionsFollower)
+            .where(ShoutReactionsFollower.shout == s.id)
+            .filter(ShoutReactionsFollower.follower == author.id)
+            .first()
+        )
         if not srf:
             srf = ShoutReactionsFollower.create(shout=s.id, follower=author.id, auto=True)
             session.add(srf)
@@ -116,14 +119,14 @@ async def get_user(entry, storage):
         elif user_oid:
             userdata = storage["users"]["by_oid"].get(user_oid)
         if not userdata:
-            print('no userdata by oid, anonymous')
+            print("no userdata by oid, anonymous")
             userdata = anondict
             print(app)
     # cleanup slug
     if userdata:
         slug = userdata.get("slug", "")
         if slug:
-            slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+            slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
             userdata["slug"] = slug
     else:
         userdata = anondict
@@ -137,11 +140,14 @@ async def migrate(entry, storage):
     r = {
         "layout": type2layout[entry["type"]],
         "title": entry["title"],
-        "authors": [author, ],
+        "authors": [
+            author,
+        ],
         "slug": get_shout_slug(entry),
         "cover": (
-            "https://images.discours.io/unsafe/" +
-            entry["thumborId"] if entry.get("thumborId") else entry.get("image", {}).get("url")
+            "https://images.discours.io/unsafe/" + entry["thumborId"]
+            if entry.get("thumborId")
+            else entry.get("image", {}).get("url")
         ),
         "visibility": "public" if entry.get("published") else "community",
         "publishedAt": date_parse(entry.get("publishedAt")) if entry.get("published") else None,
@@ -150,11 +156,11 @@ async def migrate(entry, storage):
         "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts,
         "createdBy": author.id,
         "topics": await add_topics_follower(entry, storage, author),
-        "body": extract_html(entry, cleanup=True)
+        "body": extract_html(entry, cleanup=True),
     }
 
     # main topic patch
-    r['mainTopic'] = r['topics'][0]
+    r["mainTopic"] = r["topics"][0]
 
     # published author auto-confirm
     if entry.get("published"):
@@ -177,14 +183,16 @@ async def migrate(entry, storage):
         shout_dict["oid"] = entry.get("_id", "")
         shout = await create_shout(shout_dict)
     except IntegrityError as e:
-        print('[migration] create_shout integrity error', e)
+        print("[migration] create_shout integrity error", e)
         shout = await resolve_create_shout(shout_dict)
     except Exception as e:
         raise Exception(e)
 
     # udpate data
     shout_dict = shout.dict()
-    shout_dict["authors"] = [author.dict(), ]
+    shout_dict["authors"] = [
+        author.dict(),
+    ]
 
     # shout topics aftermath
     shout_dict["topics"] = await topics_aftermath(r, storage)
@@ -193,7 +201,9 @@ async def migrate(entry, storage):
     await content_ratings_to_reactions(entry, shout_dict["slug"])
 
     # shout views
-    await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1), viewer='old-discours')
+    await ViewedStorage.increment(
+        shout_dict["slug"], amount=entry.get("views", 1), viewer="old-discours"
+    )
     # del shout_dict['ratings']
 
     storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
@@ -205,7 +215,9 @@ async def add_topics_follower(entry, storage, user):
     topics = set([])
     category = entry.get("category")
     topics_by_oid = storage["topics"]["by_oid"]
-    oids = [category, ] + entry.get("tags", [])
+    oids = [
+        category,
+    ] + entry.get("tags", [])
     for toid in oids:
         tslug = topics_by_oid.get(toid, {}).get("slug")
         if tslug:
@@ -217,23 +229,18 @@ async def add_topics_follower(entry, storage, user):
             try:
                 tpc = session.query(Topic).where(Topic.slug == tpcslug).first()
                 if tpc:
-                    tf = session.query(
-                        TopicFollower
-                    ).where(
-                        TopicFollower.follower == user.id
-                    ).filter(
-                        TopicFollower.topic == tpc.id
-                    ).first()
+                    tf = (
+                        session.query(TopicFollower)
+                        .where(TopicFollower.follower == user.id)
+                        .filter(TopicFollower.topic == tpc.id)
+                        .first()
+                    )
                     if not tf:
-                        tf = TopicFollower.create(
-                            topic=tpc.id,
-                            follower=user.id,
-                            auto=True
-                        )
+                        tf = TopicFollower.create(topic=tpc.id, follower=user.id, auto=True)
                         session.add(tf)
                         session.commit()
             except IntegrityError:
-                print('[migration.shout] hidden by topic ' + tpc.slug)
+                print("[migration.shout] hidden by topic " + tpc.slug)
     # main topic
     maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug"))
     if maintopic in ttt:
@@ -254,7 +261,7 @@ async def process_user(userdata, storage, oid):
         if not user:
             try:
                 slug = userdata["slug"].lower().strip()
-                slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+                slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
                 userdata["slug"] = slug
                 user = User.create(**userdata)
                 session.add(user)
@@ -282,9 +289,9 @@ async def resolve_create_shout(shout_dict):
         s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
         bump = False
         if s:
-            if s.createdAt != shout_dict['createdAt']:
+            if s.createdAt != shout_dict["createdAt"]:
                 # create new with different slug
-                shout_dict["slug"] += '-' + shout_dict["layout"]
+                shout_dict["slug"] += "-" + shout_dict["layout"]
                 try:
                     await create_shout(shout_dict)
                 except IntegrityError as e:
@@ -295,10 +302,7 @@ async def resolve_create_shout(shout_dict):
                 for key in shout_dict:
                     if key in s.__dict__:
                         if s.__dict__[key] != shout_dict[key]:
-                            print(
-                                "[migration] shout already exists, but differs in %s"
-                                % key
-                            )
+                            print("[migration] shout already exists, but differs in %s" % key)
                             bump = True
                     else:
                         print("[migration] shout already exists, but lacks %s" % key)
@@ -344,9 +348,7 @@ async def topics_aftermath(entry, storage):
                     )
                     if not shout_topic_new:
                         try:
-                            ShoutTopic.create(
-                                **{"shout": shout.id, "topic": new_topic.id}
-                            )
+                            ShoutTopic.create(**{"shout": shout.id, "topic": new_topic.id})
                         except Exception:
                             print("[migration] shout topic error: " + newslug)
                 session.commit()
@@ -363,9 +365,7 @@ async def content_ratings_to_reactions(entry, slug):
         with local_session() as session:
             for content_rating in entry.get("ratings", []):
                 rater = (
-                    session.query(User)
-                    .filter(User.oid == content_rating["createdBy"])
-                    .first()
+                    session.query(User).filter(User.oid == content_rating["createdBy"]).first()
                 ) or User.default_user
                 shout = session.query(Shout).where(Shout.slug == slug).first()
                 cts = content_rating.get("createdAt")
@@ -375,7 +375,7 @@ async def content_ratings_to_reactions(entry, slug):
                     if content_rating["value"] > 0
                     else ReactionKind.DISLIKE,
                     "createdBy": rater.id,
-                    "shout": shout.id
+                    "shout": shout.id,
                 }
                 reaction = (
                     session.query(Reaction)

migration/tables/remarks.py

@@ -1,42 +1,35 @@
-from base.orm import local_session
-from migration.extract import extract_md
-from migration.html2text import html2text
-from orm.reaction import Reaction, ReactionKind
+# from base.orm import local_session
+
+# from migration.extract import extract_md
+# from migration.html2text import html2text
+# from orm.reaction import Reaction, ReactionKind
 
 
-def migrate(entry, storage):
-    post_oid = entry['contentItem']
-    print(post_oid)
-    shout_dict = storage['shouts']['by_oid'].get(post_oid)
-    if shout_dict:
-        print(shout_dict['body'])
-        remark = {
-            "shout": shout_dict['id'],
-            "body": extract_md(
-                html2text(entry['body']),
-                shout_dict
-            ),
-            "kind": ReactionKind.REMARK
-        }
-
-        if entry.get('textBefore'):
-            remark['range'] = str(
-                    shout_dict['body']
-                        .index(
-                            entry['textBefore'] or ''
-                        )
-                ) + ':' + str(
-                    shout_dict['body']
-                        .index(
-                            entry['textAfter'] or ''
-                        ) + len(
-                            entry['textAfter'] or ''
-                        )
-                )
-
-        with local_session() as session:
-            rmrk = Reaction.create(**remark)
-            session.commit()
-            del rmrk["_sa_instance_state"]
-            return rmrk
-    return
+# def migrate(entry, storage):
+#     post_oid = entry["contentItem"]
+#     print(post_oid)
+#     shout_dict = storage["shouts"]["by_oid"].get(post_oid)
+#     if shout_dict:
+#         print(shout_dict["body"])
+#         remark = {
+#             "shout": shout_dict["id"],
+#             "body": extract_md(html2text(entry["body"]), shout_dict),
+#             "kind": ReactionKind.REMARK,
+#         }
+#
+#         if entry.get("textBefore"):
+#             remark["range"] = (
+#                 str(shout_dict["body"].index(entry["textBefore"] or ""))
+#                 + ":"
+#                 + str(
+#                     shout_dict["body"].index(entry["textAfter"] or "")
+#                     + len(entry["textAfter"] or "")
+#                 )
+#             )
+#
+#         with local_session() as session:
+#             rmrk = Reaction.create(**remark)
+#             session.commit()
+#             del rmrk["_sa_instance_state"]
+#             return rmrk
+#     return

migration/tables/topics.py

@@ -1,5 +1,4 @@
 from base.orm import local_session
-from migration.extract import extract_md
 from migration.html2text import html2text
 from orm import Topic
 
@@ -10,7 +9,7 @@ def migrate(entry):
         "slug": entry["slug"],
         "oid": entry["_id"],
         "title": entry["title"].replace(" ", " "),
-        "body": extract_md(html2text(body_orig))
+        "body": html2text(body_orig),
     }
 
     with local_session() as session:

migration/tables/users.py

@@ -8,7 +8,7 @@ from base.orm import local_session
 from orm.user import AuthorFollower, User, UserRating
 
 
-def migrate(entry):
+def migrate(entry):  # noqa: C901
     if "subscribedTo" in entry:
         del entry["subscribedTo"]
     email = entry["emails"][0]["address"]
@@ -23,7 +23,7 @@ def migrate(entry):
         "muted": False,  # amnesty
         "links": [],
         "name": "anonymous",
-        "password": entry["services"]["password"].get("bcrypt")
+        "password": entry["services"]["password"].get("bcrypt"),
     }
 
     if "updatedAt" in entry:
@@ -33,9 +33,13 @@ def migrate(entry):
     if entry.get("profile"):
         # slug
         slug = entry["profile"].get("path").lower()
-        slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip()
+        slug = re.sub("[^0-9a-zA-Z]+", "-", slug).strip()
         user_dict["slug"] = slug
-        bio = (entry.get("profile", {"bio": ""}).get("bio") or "").replace('\(', '(').replace('\)', ')')
+        bio = (
+            (entry.get("profile", {"bio": ""}).get("bio") or "")
+            .replace(r"\(", "(")
+            .replace(r"\)", ")")
+        )
         bio_text = BeautifulSoup(bio, features="lxml").text
 
         if len(bio_text) > 120:
@@ -46,8 +50,7 @@ def migrate(entry):
         # userpic
         try:
             user_dict["userpic"] = (
-                "https://images.discours.io/unsafe/"
-                + entry["profile"]["thumborId"]
+                "https://images.discours.io/unsafe/" + entry["profile"]["thumborId"]
             )
         except KeyError:
             try:
@@ -62,11 +65,7 @@ def migrate(entry):
         name = (name + " " + ln) if ln else name
         if not name:
             name = slug if slug else "anonymous"
-        name = (
-            entry["profile"]["path"].lower().strip().replace(" ", "-")
-            if len(name) < 2
-            else name
-        )
+        name = entry["profile"]["path"].lower().strip().replace(" ", "-") if len(name) < 2 else name
         user_dict["name"] = name
 
         # links
@@ -95,9 +94,7 @@ def migrate(entry):
     except IntegrityError:
         print("[migration] cannot create user " + user_dict["slug"])
         with local_session() as session:
-            old_user = (
-                session.query(User).filter(User.slug == user_dict["slug"]).first()
-            )
+            old_user = session.query(User).filter(User.slug == user_dict["slug"]).first()
             old_user.oid = oid
             old_user.password = user_dict["password"]
             session.commit()
@@ -114,7 +111,7 @@ def post_migrate():
         "slug": "old-discours",
         "username": "old-discours",
         "email": "old@discours.io",
-        "name": "Просмотры на старой версии сайта"
+        "name": "Просмотры на старой версии сайта",
     }
 
     with local_session() as session:
@@ -147,12 +144,8 @@ def migrate_2stage(entry, id_map):
                 }
 
                 user_rating = UserRating.create(**user_rating_dict)
-                if user_rating_dict['value'] > 0:
-                    af = AuthorFollower.create(
-                        author=user.id,
-                        follower=rater.id,
-                        auto=True
-                    )
+                if user_rating_dict["value"] > 0:
+                    af = AuthorFollower.create(author=user.id, follower=rater.id, auto=True)
                     session.add(af)
                 session.add(user_rating)
                 session.commit()

orm/__init__.py

@@ -1,7 +1,7 @@
 from base.orm import Base, engine
 from orm.community import Community
 from orm.notification import Notification
-from orm.rbac import Operation, Resource, Permission, Role
+from orm.rbac import Operation, Permission, Resource, Role
 from orm.reaction import Reaction
 from orm.shout import Shout
 from orm.topic import Topic, TopicFollower
@@ -32,5 +32,5 @@ __all__ = [
     "Notification",
     "Reaction",
     "UserRating",
-    "init_tables"
+    "init_tables",
 ]

orm/collection.py

@@ -8,7 +8,7 @@ from base.orm import Base
 class ShoutCollection(Base):
     __tablename__ = "shout_collection"
 
-    id = None  # type: ignore
+    id = None
     shout = Column(ForeignKey("shout.id"), primary_key=True)
     collection = Column(ForeignKey("collection.id"), primary_key=True)
 

orm/community.py

@@ -1,18 +1,17 @@
 from datetime import datetime
 
-from sqlalchemy import Column, String, ForeignKey, DateTime
+from sqlalchemy import Column, DateTime, ForeignKey, String
+
 from base.orm import Base, local_session
 
 
 class CommunityFollower(Base):
     __tablename__ = "community_followers"
 
-    id = None  # type: ignore
-    follower = Column(ForeignKey("user.id"), primary_key=True)
-    community = Column(ForeignKey("community.id"), primary_key=True)
-    joinedAt = Column(
-        DateTime, nullable=False, default=datetime.now, comment="Created at"
-    )
+    id = None
+    follower: Column = Column(ForeignKey("user.id"), primary_key=True)
+    community: Column = Column(ForeignKey("community.id"), primary_key=True)
+    joinedAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
     # role = Column(ForeignKey(Role.id), nullable=False, comment="Role for member")
 
 
@@ -23,19 +22,15 @@ class Community(Base):
     slug = Column(String, nullable=False, unique=True, comment="Slug")
     desc = Column(String, nullable=False, default="")
     pic = Column(String, nullable=False, default="")
-    createdAt = Column(
-        DateTime, nullable=False, default=datetime.now, comment="Created at"
-    )
+    createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
 
     @staticmethod
     def init_table():
         with local_session() as session:
-            d = (
-                session.query(Community).filter(Community.slug == "discours").first()
-            )
+            d = session.query(Community).filter(Community.slug == "discours").first()
             if not d:
                 d = Community.create(name="Дискурс", slug="discours")
                 session.add(d)
                 session.commit()
             Community.default_community = d
-            print('[orm] default community id: %s' % d.id)
+            print("[orm] default community id: %s" % d.id)

orm/notification.py

@@ -1,9 +1,10 @@
 from datetime import datetime
-from sqlalchemy import Column, Enum, ForeignKey, DateTime, Boolean, Integer
+from enum import Enum as Enumeration
+
+from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer
 from sqlalchemy.dialects.postgresql import JSONB
 
 from base.orm import Base
-from enum import Enum as Enumeration
 
 
 class NotificationType(Enumeration):
@@ -14,9 +15,9 @@ class NotificationType(Enumeration):
 class Notification(Base):
     __tablename__ = "notification"
 
-    shout = Column(ForeignKey("shout.id"), index=True)
-    reaction = Column(ForeignKey("reaction.id"), index=True)
-    user = Column(ForeignKey("user.id"), index=True)
+    shout: Column = Column(ForeignKey("shout.id"), index=True)
+    reaction: Column = Column(ForeignKey("reaction.id"), index=True)
+    user: Column = Column(ForeignKey("user.id"), index=True)
     createdAt = Column(DateTime, nullable=False, default=datetime.now, index=True)
     seen = Column(Boolean, nullable=False, default=False, index=True)
     type = Column(Enum(NotificationType), nullable=False)

orm/rbac.py

@@ -1,9 +1,9 @@
 import warnings
 
-from sqlalchemy import String, Column, ForeignKey, UniqueConstraint, TypeDecorator
+from sqlalchemy import Column, ForeignKey, String, TypeDecorator, UniqueConstraint
 from sqlalchemy.orm import relationship
 
-from base.orm import Base, REGISTRY, engine, local_session
+from base.orm import REGISTRY, Base, local_session
 
 # Role Based Access Control #
 
@@ -121,16 +121,23 @@ class Operation(Base):
 
 class Resource(Base):
     __tablename__ = "resource"
-    resourceClass = Column(
-        String, nullable=False, unique=True, comment="Resource class"
-    )
+    resourceClass = Column(String, nullable=False, unique=True, comment="Resource class")
     name = Column(String, nullable=False, unique=True, comment="Resource name")
     # TODO: community = Column(ForeignKey())
 
     @staticmethod
     def init_table():
         with local_session() as session:
-            for res in ["shout", "topic", "reaction", "chat", "message", "invite", "community", "user"]:
+            for res in [
+                "shout",
+                "topic",
+                "reaction",
+                "chat",
+                "message",
+                "invite",
+                "community",
+                "user",
+            ]:
                 r = session.query(Resource).filter(Resource.name == res).first()
                 if not r:
                     r = Resource.create(name=res, resourceClass=res)
@@ -145,29 +152,27 @@ class Permission(Base):
         {"extend_existing": True},
     )
 
-    role = Column(
-        ForeignKey("role.id", ondelete="CASCADE"), nullable=False, comment="Role"
-    )
-    operation = Column(
+    role: Column = Column(ForeignKey("role.id", ondelete="CASCADE"), nullable=False, comment="Role")
+    operation: Column = Column(
         ForeignKey("operation.id", ondelete="CASCADE"),
         nullable=False,
         comment="Operation",
     )
-    resource = Column(
+    resource: Column = Column(
         ForeignKey("resource.id", ondelete="CASCADE"),
         nullable=False,
         comment="Resource",
     )
 
 
-if __name__ == "__main__":
-    Base.metadata.create_all(engine)
-    ops = [
-        Permission(role=1, operation=1, resource=1),
-        Permission(role=1, operation=2, resource=1),
-        Permission(role=1, operation=3, resource=1),
-        Permission(role=1, operation=4, resource=1),
-        Permission(role=2, operation=4, resource=1),
-    ]
-    global_session.add_all(ops)
-    global_session.commit()
+# if __name__ == "__main__":
+#     Base.metadata.create_all(engine)
+#     ops = [
+#         Permission(role=1, operation=1, resource=1),
+#         Permission(role=1, operation=2, resource=1),
+#         Permission(role=1, operation=3, resource=1),
+#         Permission(role=1, operation=4, resource=1),
+#         Permission(role=2, operation=4, resource=1),
+#     ]
+#     global_session.add_all(ops)
+#     global_session.commit()

orm/reaction.py

@@ -27,16 +27,18 @@ class ReactionKind(Enumeration):
 class Reaction(Base):
     __tablename__ = "reaction"
     body = Column(String, nullable=True, comment="Reaction Body")
-    createdAt = Column(
-        DateTime, nullable=False, default=datetime.now, comment="Created at"
-    )
-    createdBy = Column(ForeignKey("user.id"), nullable=False, index=True, comment="Sender")
+    createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
+    createdBy: Column = Column(ForeignKey("user.id"), nullable=False, index=True, comment="Sender")
     updatedAt = Column(DateTime, nullable=True, comment="Updated at")
-    updatedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor")
+    updatedBy: Column = Column(
+        ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor"
+    )
     deletedAt = Column(DateTime, nullable=True, comment="Deleted at")
-    deletedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by")
-    shout = Column(ForeignKey("shout.id"), nullable=False, index=True)
-    replyTo = Column(
+    deletedBy: Column = Column(
+        ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by"
+    )
+    shout: Column = Column(ForeignKey("shout.id"), nullable=False, index=True)
+    replyTo: Column = Column(
         ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID"
     )
     range = Column(String, nullable=True, comment="Range in format <start index>:<end>")

orm/shout.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, JSON
+from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Integer, String
 from sqlalchemy.orm import column_property, relationship
 
 from base.orm import Base, local_session
@@ -12,31 +12,29 @@ from orm.user import User
 class ShoutTopic(Base):
     __tablename__ = "shout_topic"
 
-    id = None  # type: ignore
-    shout = Column(ForeignKey("shout.id"), primary_key=True, index=True)
-    topic = Column(ForeignKey("topic.id"), primary_key=True, index=True)
+    id = None
+    shout: Column = Column(ForeignKey("shout.id"), primary_key=True, index=True)
+    topic: Column = Column(ForeignKey("topic.id"), primary_key=True, index=True)
 
 
 class ShoutReactionsFollower(Base):
     __tablename__ = "shout_reactions_followers"
 
-    id = None  # type: ignore
-    follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
-    shout = Column(ForeignKey("shout.id"), primary_key=True, index=True)
+    id = None
+    follower: Column = Column(ForeignKey("user.id"), primary_key=True, index=True)
+    shout: Column = Column(ForeignKey("shout.id"), primary_key=True, index=True)
     auto = Column(Boolean, nullable=False, default=False)
-    createdAt = Column(
-        DateTime, nullable=False, default=datetime.now, comment="Created at"
-    )
+    createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
     deletedAt = Column(DateTime, nullable=True)
 
 
 class ShoutAuthor(Base):
     __tablename__ = "shout_author"
 
-    id = None  # type: ignore
-    shout = Column(ForeignKey("shout.id"), primary_key=True, index=True)
-    user = Column(ForeignKey("user.id"), primary_key=True, index=True)
-    caption = Column(String, nullable=True, default="")
+    id = None
+    shout: Column = Column(ForeignKey("shout.id"), primary_key=True, index=True)
+    user: Column = Column(ForeignKey("user.id"), primary_key=True, index=True)
+    caption: Column = Column(String, nullable=True, default="")
 
 
 class Shout(Base):
@@ -48,8 +46,8 @@ class Shout(Base):
     publishedAt = Column(DateTime, nullable=True)
     deletedAt = Column(DateTime, nullable=True)
 
-    createdBy = Column(ForeignKey("user.id"), comment="Created By")
-    deletedBy = Column(ForeignKey("user.id"), nullable=True)
+    createdBy: Column = Column(ForeignKey("user.id"), comment="Created By")
+    deletedBy: Column = Column(ForeignKey("user.id"), nullable=True)
 
     slug = Column(String, unique=True)
     cover = Column(String, nullable=True, comment="Cover image url")
@@ -71,11 +69,11 @@ class Shout(Base):
     reactions = relationship(lambda: Reaction)
 
     # TODO: these field should be used or modified
-    community = Column(ForeignKey("community.id"), default=1)
-    lang = Column(String, nullable=False, default='ru', comment="Language")
-    mainTopic = Column(ForeignKey("topic.slug"), nullable=True)
+    community: Column = Column(ForeignKey("community.id"), default=1)
+    lang = Column(String, nullable=False, default="ru", comment="Language")
+    mainTopic: Column = Column(ForeignKey("topic.slug"), nullable=True)
     visibility = Column(String, nullable=True)  # owner authors community public
-    versionOf = Column(ForeignKey("shout.id"), nullable=True)
+    versionOf: Column = Column(ForeignKey("shout.id"), nullable=True)
     oid = Column(String, nullable=True)
 
     @staticmethod
@@ -83,12 +81,7 @@ class Shout(Base):
         with local_session() as session:
             s = session.query(Shout).first()
             if not s:
-                entry = {
-                    "slug": "genesis-block",
-                    "body": "",
-                    "title": "Ничего",
-                    "lang": "ru"
-                }
+                entry = {"slug": "genesis-block", "body": "", "title": "Ничего", "lang": "ru"}
                 s = Shout.create(**entry)
                 session.add(s)
                 session.commit()

orm/topic.py

@@ -8,12 +8,10 @@ from base.orm import Base
 class TopicFollower(Base):
     __tablename__ = "topic_followers"
 
-    id = None  # type: ignore
-    follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
-    topic = Column(ForeignKey("topic.id"), primary_key=True, index=True)
-    createdAt = Column(
-        DateTime, nullable=False, default=datetime.now, comment="Created at"
-    )
+    id = None
+    follower: Column = Column(ForeignKey("user.id"), primary_key=True, index=True)
+    topic: Column = Column(ForeignKey("topic.id"), primary_key=True, index=True)
+    createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
     auto = Column(Boolean, nullable=False, default=False)
 
 
@@ -24,7 +22,5 @@ class Topic(Base):
     title = Column(String, nullable=False, comment="Title")
     body = Column(String, nullable=True, comment="Body")
     pic = Column(String, nullable=True, comment="Picture")
-    community = Column(
-        ForeignKey("community.id"), default=1, comment="Community"
-    )
+    community: Column = Column(ForeignKey("community.id"), default=1, comment="Community")
     oid = Column(String, nullable=True, comment="Old ID")

orm/user.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from sqlalchemy import JSON as JSONType
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String
 from sqlalchemy.orm import relationship
+
 from base.orm import Base, local_session
 from orm.rbac import Role
 
@@ -10,10 +11,10 @@ from orm.rbac import Role
 class UserRating(Base):
     __tablename__ = "user_rating"
 
-    id = None  # type: ignore
-    rater = Column(ForeignKey("user.id"), primary_key=True, index=True)
-    user = Column(ForeignKey("user.id"), primary_key=True, index=True)
-    value = Column(Integer)
+    id = None
+    rater: Column = Column(ForeignKey("user.id"), primary_key=True, index=True)
+    user: Column = Column(ForeignKey("user.id"), primary_key=True, index=True)
+    value: Column = Column(Integer)
 
     @staticmethod
     def init_table():
@@ -23,7 +24,7 @@ class UserRating(Base):
 class UserRole(Base):
     __tablename__ = "user_role"
 
-    id = None  # type: ignore
+    id = None
     user = Column(ForeignKey("user.id"), primary_key=True, index=True)
     role = Column(ForeignKey("role.id"), primary_key=True, index=True)
 
@@ -31,12 +32,10 @@ class UserRole(Base):
 class AuthorFollower(Base):
     __tablename__ = "author_follower"
 
-    id = None  # type: ignore
-    follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
-    author = Column(ForeignKey("user.id"), primary_key=True, index=True)
-    createdAt = Column(
-        DateTime, nullable=False, default=datetime.now, comment="Created at"
-    )
+    id = None
+    follower: Column = Column(ForeignKey("user.id"), primary_key=True, index=True)
+    author: Column = Column(ForeignKey("user.id"), primary_key=True, index=True)
+    createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
     auto = Column(Boolean, nullable=False, default=False)
 
 
@@ -54,12 +53,8 @@ class User(Base):
     slug = Column(String, unique=True, comment="User's slug")
     muted = Column(Boolean, default=False)
     emailConfirmed = Column(Boolean, default=False)
-    createdAt = Column(
-        DateTime, nullable=False, default=datetime.now, comment="Created at"
-    )
-    lastSeen = Column(
-        DateTime, nullable=False, default=datetime.now, comment="Was online at"
-    )
+    createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
+    lastSeen = Column(DateTime, nullable=False, default=datetime.now, comment="Was online at")
     deletedAt = Column(DateTime, nullable=True, comment="Deleted at")
     links = Column(JSONType, nullable=True, comment="Links")
     oauth = Column(String, nullable=True)
@@ -103,4 +98,4 @@ class User(Base):
 
 
 # if __name__ == "__main__":
-#   print(User.get_permission(user_id=1))  # type: ignore
+#   print(User.get_permission(user_id=1))

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.black]
+line-length = 100

requirements-dev.txt

@@ -1,4 +1,8 @@
-isort
-brunette
-flake8
-mypy
+black==23.10.1
+flake8==6.1.0
+gql_schema_codegen==1.0.1
+isort==5.12.0
+mypy==1.6.1
+pre-commit==3.5.0
+pymongo-stubs==0.2.0
+sqlalchemy-stubs==0.4

requirements.txt

@@ -1,40 +1,37 @@
-python-frontmatter~=1.0.0
-aioredis~=2.0.1
 aiohttp
+aioredis~=2.0.1
+alembic==1.11.3
 ariadne>=0.17.0
-PyYAML>=5.4
-pyjwt>=2.6.0
-starlette~=0.23.1
-sqlalchemy>=1.4.41
-graphql-core>=3.0.3
-gql~=3.4.0
-uvicorn>=0.18.3
-pydantic>=1.10.2
-passlib~=1.7.4
-authlib>=1.1.0
-httpx>=0.23.0
-psycopg2-binary
-transliterate~=1.10.2
-requests~=2.28.1
-bcrypt>=4.0.0
-bson~=0.5.10
-flake8
-DateTime~=4.7
 asyncio~=3.4.3
-python-dateutil~=2.8.2
+authlib>=1.1.0
+bcrypt>=4.0.0
 beautifulsoup4~=4.11.1
-lxml
-sentry-sdk>=1.14.0
-# sse_starlette
-graphql-ws
-nltk~=3.8.1
-pymystem3~=0.2.0
-transformers~=4.28.1
 boto3~=1.28.2
 botocore~=1.31.2
-python-multipart~=0.0.6
-alembic==1.11.3
+bson~=0.5.10
+DateTime~=4.7
+gql~=3.4.0
+graphql-core>=3.0.3
+httpx>=0.23.0
+itsdangerous
+lxml
 Mako==1.2.4
 MarkupSafe==2.1.3
+nltk~=3.8.1
+passlib~=1.7.4
+psycopg2-binary
+pydantic>=1.10.2
+pyjwt>=2.6.0
+pymystem3~=0.2.0
+python-dateutil~=2.8.2
+python-frontmatter~=1.0.0
+python-multipart~=0.0.6
+PyYAML>=5.4
+requests~=2.28.1
+sentry-sdk>=1.14.0
+sqlalchemy>=1.4.41
 sse-starlette==1.6.5
-itsdangerous
+starlette~=0.23.1
+transformers~=4.28.1
+transliterate~=1.10.2
+uvicorn>=0.18.3

resetdb.sh

@@ -53,4 +53,3 @@ echo "Start migration"
 python3 server.py migrate
 if [ $? -ne 0 ]; then { echo "Migration failed, aborting." ; exit 1; } fi
 echo 'Done!'
-

resolvers/__init__.py

@@ -1,67 +1,46 @@
+# flake8: noqa
+
 from resolvers.auth import (
-    login,
-    sign_out,
-    is_email_used,
-    register_by_email,
-    confirm_email,
     auth_send_link,
+    confirm_email,
     get_current_user,
+    is_email_used,
+    login,
+    register_by_email,
+    sign_out,
 )
-
-from resolvers.create.migrate import markdown_body
 from resolvers.create.editor import create_shout, delete_shout, update_shout
-
-from resolvers.zine.profile import (
-    load_authors_by,
-    rate_user,
-    update_profile,
-    get_authors_all
-)
-
-from resolvers.zine.reactions import (
-    create_reaction,
-    delete_reaction,
-    update_reaction,
-    reactions_unfollow,
-    reactions_follow,
-    load_reactions_by
-)
-from resolvers.zine.topics import (
-    topic_follow,
-    topic_unfollow,
-    topics_by_author,
-    topics_by_community,
-    topics_all,
-    get_topic
-)
-
-from resolvers.zine.following import (
-    follow,
-    unfollow
-)
-
-from resolvers.zine.load import (
-    load_shout,
-    load_shouts_by
-)
-
-from resolvers.inbox.chats import (
-    create_chat,
-    delete_chat,
-    update_chat
-
-)
+from resolvers.inbox.chats import create_chat, delete_chat, update_chat
+from resolvers.inbox.load import load_chats, load_messages_by, load_recipients
 from resolvers.inbox.messages import (
     create_message,
     delete_message,
+    mark_as_read,
     update_message,
-    mark_as_read
-)
-from resolvers.inbox.load import (
-    load_chats,
-    load_messages_by,
-    load_recipients
 )
 from resolvers.inbox.search import search_recipients
-
 from resolvers.notifications import load_notifications
+from resolvers.zine.following import follow, unfollow
+from resolvers.zine.load import load_shout, load_shouts_by
+from resolvers.zine.profile import (
+    get_authors_all,
+    load_authors_by,
+    rate_user,
+    update_profile,
+)
+from resolvers.zine.reactions import (
+    create_reaction,
+    delete_reaction,
+    load_reactions_by,
+    reactions_follow,
+    reactions_unfollow,
+    update_reaction,
+)
+from resolvers.zine.topics import (
+    get_topic,
+    topic_follow,
+    topic_unfollow,
+    topics_all,
+    topics_by_author,
+    topics_by_community,
+)

resolvers/auth.py

@@ -1,24 +1,30 @@
 # -*- coding: utf-8 -*-
 
+import re
 from datetime import datetime, timezone
 from urllib.parse import quote_plus
 
 from graphql.type import GraphQLResolveInfo
 from starlette.responses import RedirectResponse
 from transliterate import translit
-import re
+
 from auth.authenticate import login_required
 from auth.credentials import AuthCredentials
 from auth.email import send_auth_email
 from auth.identity import Identity, Password
 from auth.jwtcodec import JWTCodec
 from auth.tokenstorage import TokenStorage
-from base.exceptions import (BaseHttpException, InvalidPassword, InvalidToken,
-                             ObjectNotExist, Unauthorized)
+from base.exceptions import (
+    BaseHttpException,
+    InvalidPassword,
+    InvalidToken,
+    ObjectNotExist,
+    Unauthorized,
+)
 from base.orm import local_session
 from base.resolvers import mutation, query
 from orm import Role, User
-from settings import SESSION_TOKEN_HEADER, FRONTEND_URL
+from settings import FRONTEND_URL, SESSION_TOKEN_HEADER
 
 
 @mutation.field("getSession")
@@ -32,17 +38,14 @@ async def get_current_user(_, info):
         user.lastSeen = datetime.now(tz=timezone.utc)
         session.commit()
 
-        return {
-            "token": token,
-            "user": user
-        }
+        return {"token": token, "user": user}
 
 
 @mutation.field("confirmEmail")
 async def confirm_email(_, info, token):
     """confirm owning email address"""
     try:
-        print('[resolvers.auth] confirm email by token')
+        print("[resolvers.auth] confirm email by token")
         payload = JWTCodec.decode(token)
         user_id = payload.user_id
         await TokenStorage.get(f"{user_id}-{payload.username}-{token}")
@@ -53,10 +56,7 @@ async def confirm_email(_, info, token):
             user.lastSeen = datetime.now(tz=timezone.utc)
             session.add(user)
             session.commit()
-            return {
-                "token": session_token,
-                "user": user
-            }
+            return {"token": session_token, "user": user}
     except InvalidToken as e:
         raise InvalidToken(e.message)
     except Exception as e:
@@ -68,9 +68,9 @@ async def confirm_email_handler(request):
     token = request.path_params["token"]  # one time
     request.session["token"] = token
     res = await confirm_email(None, {}, token)
-    print('[resolvers.auth] confirm_email request: %r' % request)
+    print("[resolvers.auth] confirm_email request: %r" % request)
     if "error" in res:
-        raise BaseHttpException(res['error'])
+        raise BaseHttpException(res["error"])
     else:
         response = RedirectResponse(url=FRONTEND_URL)
         response.set_cookie("token", res["token"])  # session token
@@ -87,22 +87,22 @@ def create_user(user_dict):
 
 
 def generate_unique_slug(src):
-    print('[resolvers.auth] generating slug from: ' + src)
+    print("[resolvers.auth] generating slug from: " + src)
     slug = translit(src, "ru", reversed=True).replace(".", "-").lower()
-    slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
+    slug = re.sub("[^0-9a-zA-Z]+", "-", slug)
     if slug != src:
-        print('[resolvers.auth] translited name: ' + slug)
+        print("[resolvers.auth] translited name: " + slug)
     c = 1
     with local_session() as session:
         user = session.query(User).where(User.slug == slug).first()
         while user:
             user = session.query(User).where(User.slug == slug).first()
-            slug = slug + '-' + str(c)
+            slug = slug + "-" + str(c)
             c += 1
         if not user:
             unique_slug = slug
-            print('[resolvers.auth] ' + unique_slug)
-            return quote_plus(unique_slug.replace('\'', '')).replace('+', '-')
+            print("[resolvers.auth] " + unique_slug)
+            return quote_plus(unique_slug.replace("'", "")).replace("+", "-")
 
 
 @mutation.field("registerUser")
@@ -117,12 +117,12 @@ async def register_by_email(_, _info, email: str, password: str = "", name: str
         slug = generate_unique_slug(name)
         user = session.query(User).where(User.slug == slug).first()
         if user:
-            slug = generate_unique_slug(email.split('@')[0])
+            slug = generate_unique_slug(email.split("@")[0])
         user_dict = {
             "email": email,
             "username": email,  # will be used to store phone number or some messenger network id
             "name": name,
-            "slug": slug
+            "slug": slug,
         }
         if password:
             user_dict["password"] = Password.encode(password)
@@ -172,10 +172,7 @@ async def login(_, info, email: str, password: str = "", lang: str = "ru"):
                     user = Identity.password(orm_user, password)
                     session_token = await TokenStorage.create_session(user)
                     print(f"[auth] user {email} authorized")
-                    return {
-                        "token": session_token,
-                        "user": user
-                    }
+                    return {"token": session_token, "user": user}
                 except InvalidPassword:
                     print(f"[auth] {email}: invalid password")
                     raise InvalidPassword("invalid password")  # contains webserver status

resolvers/create/editor.py

@@ -18,21 +18,23 @@ async def create_shout(_, info, inp):
     auth: AuthCredentials = info.context["request"].auth
 
     with local_session() as session:
-        topics = session.query(Topic).filter(Topic.slug.in_(inp.get('topics', []))).all()
+        topics = session.query(Topic).filter(Topic.slug.in_(inp.get("topics", []))).all()
 
-        new_shout = Shout.create(**{
-            "title": inp.get("title"),
-            "subtitle": inp.get('subtitle'),
-            "lead": inp.get('lead'),
-            "description": inp.get('description'),
-            "body": inp.get("body", ''),
-            "layout": inp.get("layout"),
-            "authors": inp.get("authors", []),
-            "slug": inp.get("slug"),
-            "mainTopic": inp.get("mainTopic"),
-            "visibility": "owner",
-            "createdBy": auth.user_id
-        })
+        new_shout = Shout.create(
+            **{
+                "title": inp.get("title"),
+                "subtitle": inp.get("subtitle"),
+                "lead": inp.get("lead"),
+                "description": inp.get("description"),
+                "body": inp.get("body", ""),
+                "layout": inp.get("layout"),
+                "authors": inp.get("authors", []),
+                "slug": inp.get("slug"),
+                "mainTopic": inp.get("mainTopic"),
+                "visibility": "owner",
+                "createdBy": auth.user_id,
+            }
+        )
 
         for topic in topics:
             t = ShoutTopic.create(topic=topic.id, shout=new_shout.id)
@@ -60,14 +62,19 @@ async def create_shout(_, info, inp):
 
 @mutation.field("updateShout")
 @login_required
-async def update_shout(_, info, shout_id, shout_input=None, publish=False):
+async def update_shout(_, info, shout_id, shout_input=None, publish=False):  # noqa: C901
     auth: AuthCredentials = info.context["request"].auth
 
     with local_session() as session:
-        shout = session.query(Shout).options(
-            joinedload(Shout.authors),
-            joinedload(Shout.topics),
-        ).filter(Shout.id == shout_id).first()
+        shout = (
+            session.query(Shout)
+            .options(
+                joinedload(Shout.authors),
+                joinedload(Shout.topics),
+            )
+            .filter(Shout.id == shout_id)
+            .first()
+        )
 
         if not shout:
             return {"error": "shout not found"}
@@ -94,25 +101,34 @@ async def update_shout(_, info, shout_id, shout_input=None, publish=False):
                 session.commit()
 
             for new_topic_to_link in new_topics_to_link:
-                created_unlinked_topic = ShoutTopic.create(shout=shout.id, topic=new_topic_to_link.id)
+                created_unlinked_topic = ShoutTopic.create(
+                    shout=shout.id, topic=new_topic_to_link.id
+                )
                 session.add(created_unlinked_topic)
 
-            existing_topics_input = [topic_input for topic_input in topics_input if topic_input.get("id", 0) > 0]
-            existing_topic_to_link_ids = [existing_topic_input["id"] for existing_topic_input in existing_topics_input
-                                          if existing_topic_input["id"] not in [topic.id for topic in shout.topics]]
+            existing_topics_input = [
+                topic_input for topic_input in topics_input if topic_input.get("id", 0) > 0
+            ]
+            existing_topic_to_link_ids = [
+                existing_topic_input["id"]
+                for existing_topic_input in existing_topics_input
+                if existing_topic_input["id"] not in [topic.id for topic in shout.topics]
+            ]
 
             for existing_topic_to_link_id in existing_topic_to_link_ids:
-                created_unlinked_topic = ShoutTopic.create(shout=shout.id, topic=existing_topic_to_link_id)
+                created_unlinked_topic = ShoutTopic.create(
+                    shout=shout.id, topic=existing_topic_to_link_id
+                )
                 session.add(created_unlinked_topic)
 
-            topic_to_unlink_ids = [topic.id for topic in shout.topics
-                                   if topic.id not in [topic_input["id"] for topic_input in existing_topics_input]]
+            topic_to_unlink_ids = [
+                topic.id
+                for topic in shout.topics
+                if topic.id not in [topic_input["id"] for topic_input in existing_topics_input]
+            ]
 
             shout_topics_to_remove = session.query(ShoutTopic).filter(
-                and_(
-                    ShoutTopic.shout == shout.id,
-                    ShoutTopic.topic.in_(topic_to_unlink_ids)
-                )
+                and_(ShoutTopic.shout == shout.id, ShoutTopic.topic.in_(topic_to_unlink_ids))
             )
 
             for shout_topic_to_remove in shout_topics_to_remove:
@@ -120,13 +136,13 @@ async def update_shout(_, info, shout_id, shout_input=None, publish=False):
 
             shout_input["mainTopic"] = shout_input["mainTopic"]["slug"]
 
-            if shout_input["mainTopic"] == '':
+            if shout_input["mainTopic"] == "":
                 del shout_input["mainTopic"]
 
             shout.update(shout_input)
             updated = True
 
-        if publish and shout.visibility == 'owner':
+        if publish and shout.visibility == "owner":
             shout.visibility = "community"
             shout.publishedAt = datetime.now(tz=timezone.utc)
             updated = True

resolvers/create/migrate.py

@@ -1,11 +0,0 @@
-
-from base.resolvers import query
-from resolvers.auth import login_required
-from migration.extract import extract_md
-
-
-@login_required
-@query.field("markdownBody")
-def markdown_body(_, info, body: str):
-    body = extract_md(body)
-    return body

resolvers/inbox/chats.py

@@ -24,27 +24,24 @@ async def update_chat(_, info, chat_new: Chat):
     chat_id = chat_new["id"]
     chat = await redis.execute("GET", f"chats/{chat_id}")
     if not chat:
-        return {
-            "error": "chat not exist"
-        }
+        return {"error": "chat not exist"}
     chat = dict(json.loads(chat))
 
     # TODO
     if auth.user_id in chat["admins"]:
-        chat.update({
-            "title": chat_new.get("title", chat["title"]),
-            "description": chat_new.get("description", chat["description"]),
-            "updatedAt": int(datetime.now(tz=timezone.utc).timestamp()),
-            "admins": chat_new.get("admins", chat.get("admins") or []),
-            "users": chat_new.get("users", chat["users"])
-        })
+        chat.update(
+            {
+                "title": chat_new.get("title", chat["title"]),
+                "description": chat_new.get("description", chat["description"]),
+                "updatedAt": int(datetime.now(tz=timezone.utc).timestamp()),
+                "admins": chat_new.get("admins", chat.get("admins") or []),
+                "users": chat_new.get("users", chat["users"]),
+            }
+        )
     await redis.execute("SET", f"chats/{chat.id}", json.dumps(chat))
     await redis.execute("COMMIT")
 
-    return {
-        "error": None,
-        "chat": chat
-    }
+    return {"error": None, "chat": chat}
 
 
 @mutation.field("createChat")
@@ -52,7 +49,7 @@ async def update_chat(_, info, chat_new: Chat):
 async def create_chat(_, info, title="", members=[]):
     auth: AuthCredentials = info.context["request"].auth
     chat = {}
-    print('create_chat members: %r' % members)
+    print("create_chat members: %r" % members)
     if auth.user_id not in members:
         members.append(int(auth.user_id))
 
@@ -74,15 +71,12 @@ async def create_chat(_, info, title="", members=[]):
             chat = await redis.execute("GET", f"chats/{c.decode('utf-8')}")
             if chat:
                 chat = json.loads(chat)
-                if chat['title'] == "":
-                    print('[inbox] createChat found old chat')
+                if chat["title"] == "":
+                    print("[inbox] createChat found old chat")
                     print(chat)
                     break
         if chat:
-            return {
-                "chat": chat,
-                "error": "existed"
-            }
+            return {"chat": chat, "error": "existed"}
 
     chat_id = str(uuid.uuid4())
     chat = {
@@ -92,7 +86,7 @@ async def create_chat(_, info, title="", members=[]):
         "createdBy": auth.user_id,
         "createdAt": int(datetime.now(tz=timezone.utc).timestamp()),
         "updatedAt": int(datetime.now(tz=timezone.utc).timestamp()),
-        "admins": members if (len(members) == 2 and title == "") else []
+        "admins": members if (len(members) == 2 and title == "") else [],
     }
 
     for m in members:
@@ -100,10 +94,7 @@ async def create_chat(_, info, title="", members=[]):
     await redis.execute("SET", f"chats/{chat_id}", json.dumps(chat))
     await redis.execute("SET", f"chats/{chat_id}/next_message_id", str(0))
     await redis.execute("COMMIT")
-    return {
-        "error": None,
-        "chat": chat
-    }
+    return {"error": None, "chat": chat}
 
 
 @mutation.field("deleteChat")
@@ -114,11 +105,9 @@ async def delete_chat(_, info, chat_id: str):
     chat = await redis.execute("GET", f"/chats/{chat_id}")
     if chat:
         chat = dict(json.loads(chat))
-        if auth.user_id in chat['admins']:
+        if auth.user_id in chat["admins"]:
             await redis.execute("DEL", f"chats/{chat_id}")
             await redis.execute("SREM", "chats_by_user/" + str(auth.user_id), chat_id)
             await redis.execute("COMMIT")
     else:
-        return {
-            "error": "chat not exist"
-        }
+        return {"error": "chat not exist"}

resolvers/inbox/load.py

@@ -1,28 +1,27 @@
 import json
-# from datetime import datetime, timedelta, timezone
 
 from auth.authenticate import login_required
 from auth.credentials import AuthCredentials
-from base.redis import redis
 from base.orm import local_session
+from base.redis import redis
 from base.resolvers import query
 from orm.user import User
 from resolvers.zine.profile import followed_authors
+
 from .unread import get_unread_counter
 
+# from datetime import datetime, timedelta, timezone
+
 
 async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]):
-    ''' load :limit messages for :chat_id with :offset '''
+    """load :limit messages for :chat_id with :offset"""
     messages = []
     message_ids = []
     if ids:
         message_ids += ids
     try:
         if limit:
-            mids = await redis.lrange(f"chats/{chat_id}/message_ids",
-                                      offset,
-                                      offset + limit
-                                      )
+            mids = await redis.lrange(f"chats/{chat_id}/message_ids", offset, offset + limit)
             mids = [mid.decode("utf-8") for mid in mids]
             message_ids += mids
     except Exception as e:
@@ -30,10 +29,10 @@ async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]):
     if message_ids:
         message_keys = [f"chats/{chat_id}/messages/{mid}" for mid in message_ids]
         messages = await redis.mget(*message_keys)
-        messages = [json.loads(msg.decode('utf-8')) for msg in messages]
+        messages = [json.loads(msg.decode("utf-8")) for msg in messages]
         replies = []
         for m in messages:
-            rt = m.get('replyTo')
+            rt = m.get("replyTo")
             if rt:
                 rt = int(rt)
                 if rt not in message_ids:
@@ -46,14 +45,14 @@ async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]):
 @query.field("loadChats")
 @login_required
 async def load_chats(_, info, limit: int = 50, offset: int = 0):
-    """ load :limit chats of current user with :offset """
+    """load :limit chats of current user with :offset"""
     auth: AuthCredentials = info.context["request"].auth
 
     cids = await redis.execute("SMEMBERS", "chats_by_user/" + str(auth.user_id))
     if cids:
-        cids = list(cids)[offset:offset + limit]
+        cids = list(cids)[offset : offset + limit]
     if not cids:
-        print('[inbox.load] no chats were found')
+        print("[inbox.load] no chats were found")
         cids = []
     onliners = await redis.execute("SMEMBERS", "users-online")
     if not onliners:
@@ -64,62 +63,50 @@ async def load_chats(_, info, limit: int = 50, offset: int = 0):
         c = await redis.execute("GET", "chats/" + cid)
         if c:
             c = dict(json.loads(c))
-            c['messages'] = await load_messages(cid, 5, 0)
-            c['unread'] = await get_unread_counter(cid, auth.user_id)
+            c["messages"] = await load_messages(cid, 5, 0)
+            c["unread"] = await get_unread_counter(cid, auth.user_id)
             with local_session() as session:
-                c['members'] = []
+                c["members"] = []
                 for uid in c["users"]:
                     a = session.query(User).where(User.id == uid).first()
                     if a:
-                        c['members'].append({
-                            "id": a.id,
-                            "slug": a.slug,
-                            "userpic": a.userpic,
-                            "name": a.name,
-                            "lastSeen": a.lastSeen,
-                            "online": a.id in onliners
-                        })
+                        c["members"].append(
+                            {
+                                "id": a.id,
+                                "slug": a.slug,
+                                "userpic": a.userpic,
+                                "name": a.name,
+                                "lastSeen": a.lastSeen,
+                                "online": a.id in onliners,
+                            }
+                        )
                 chats.append(c)
-    return {
-        "chats": chats,
-        "error": None
-    }
+    return {"chats": chats, "error": None}
 
 
 @query.field("loadMessagesBy")
 @login_required
 async def load_messages_by(_, info, by, limit: int = 10, offset: int = 0):
-    ''' load :limit messages of :chat_id with :offset '''
+    """load :limit messages of :chat_id with :offset"""
 
     auth: AuthCredentials = info.context["request"].auth
     userchats = await redis.execute("SMEMBERS", "chats_by_user/" + str(auth.user_id))
-    userchats = [c.decode('utf-8') for c in userchats]
+    userchats = [c.decode("utf-8") for c in userchats]
     # print('[inbox] userchats: %r' % userchats)
     if userchats:
         # print('[inbox] loading messages by...')
         messages = []
-        by_chat = by.get('chat')
+        by_chat = by.get("chat")
         if by_chat in userchats:
             chat = await redis.execute("GET", f"chats/{by_chat}")
             # print(chat)
             if not chat:
-                return {
-                    "messages": [],
-                    "error": "chat not exist"
-                }
+                return {"messages": [], "error": "chat not exist"}
             # everyone's messages in filtered chat
             messages = await load_messages(by_chat, limit, offset)
-        return {
-            "messages": sorted(
-                list(messages),
-                key=lambda m: m['createdAt']
-            ),
-            "error": None
-        }
+        return {"messages": sorted(list(messages), key=lambda m: m["createdAt"]), "error": None}
     else:
-        return {
-            "error": "Cannot access messages of this chat"
-        }
+        return {"error": "Cannot access messages of this chat"}
 
 
 @query.field("loadRecipients")
@@ -138,15 +125,14 @@ async def load_recipients(_, info, limit=50, offset=0):
         chat_users += session.query(User).where(User.emailConfirmed).limit(limit).offset(offset)
         members = []
         for a in chat_users:
-            members.append({
-                "id": a.id,
-                "slug": a.slug,
-                "userpic": a.userpic,
-                "name": a.name,
-                "lastSeen": a.lastSeen,
-                "online": a.id in onliners
-            })
-    return {
-        "members": members,
-        "error": None
-    }
+            members.append(
+                {
+                    "id": a.id,
+                    "slug": a.slug,
+                    "userpic": a.userpic,
+                    "name": a.name,
+                    "lastSeen": a.lastSeen,
+                    "online": a.id in onliners,
+                }
+            )
+    return {"members": members, "error": None}

resolvers/inbox/messages.py

@@ -1,62 +1,54 @@
-import asyncio
 import json
-from typing import Any
 from datetime import datetime, timezone
-from graphql.type import GraphQLResolveInfo
+
 from auth.authenticate import login_required
 from auth.credentials import AuthCredentials
 from base.redis import redis
 from base.resolvers import mutation
-from services.following import FollowingManager, FollowingResult, Following
-from validations.inbox import Message
+from services.following import FollowingManager, FollowingResult
 
 
 @mutation.field("createMessage")
 @login_required
 async def create_message(_, info, chat: str, body: str, replyTo=None):
-    """ create message with :body for :chat_id replying to :replyTo optionally """
+    """create message with :body for :chat_id replying to :replyTo optionally"""
     auth: AuthCredentials = info.context["request"].auth
 
     chat = await redis.execute("GET", f"chats/{chat}")
     if not chat:
-        return {
-            "error": "chat is not exist"
-        }
+        return {"error": "chat is not exist"}
     else:
-        chat = dict(json.loads(chat))
-        message_id = await redis.execute("GET", f"chats/{chat['id']}/next_message_id")
+        chat_dict = dict(json.loads(chat))
+        message_id = await redis.execute("GET", f"chats/{chat_dict['id']}/next_message_id")
         message_id = int(message_id)
         new_message = {
-            "chatId": chat['id'],
+            "chatId": chat_dict["id"],
             "id": message_id,
             "author": auth.user_id,
             "body": body,
-            "createdAt": int(datetime.now(tz=timezone.utc).timestamp())
+            "createdAt": int(datetime.now(tz=timezone.utc).timestamp()),
         }
         if replyTo:
-            new_message['replyTo'] = replyTo
-        chat['updatedAt'] = new_message['createdAt']
-        await redis.execute("SET", f"chats/{chat['id']}", json.dumps(chat))
+            new_message["replyTo"] = replyTo
+        chat_dict["updatedAt"] = new_message["createdAt"]
+        await redis.execute("SET", f"chats/{chat_dict['id']}", json.dumps(chat))
         print(f"[inbox] creating message {new_message}")
         await redis.execute(
-            "SET", f"chats/{chat['id']}/messages/{message_id}", json.dumps(new_message)
+            "SET", f"chats/{chat_dict['id']}/messages/{message_id}", json.dumps(new_message)
         )
-        await redis.execute("LPUSH", f"chats/{chat['id']}/message_ids", str(message_id))
-        await redis.execute("SET", f"chats/{chat['id']}/next_message_id", str(message_id + 1))
+        await redis.execute("LPUSH", f"chats/{chat_dict['id']}/message_ids", str(message_id))
+        await redis.execute("SET", f"chats/{chat_dict['id']}/next_message_id", str(message_id + 1))
 
-        users = chat["users"]
+        users = chat_dict["users"]
         for user_slug in users:
             await redis.execute(
-                "LPUSH", f"chats/{chat['id']}/unread/{user_slug}", str(message_id)
+                "LPUSH", f"chats/{chat_dict['id']}/unread/{user_slug}", str(message_id)
             )
 
-        result = FollowingResult("NEW", 'chat', new_message)
-        await FollowingManager.push('chat', result)
+        result = FollowingResult("NEW", "chat", new_message)
+        await FollowingManager.push("chat", result)
 
-        return {
-            "message": new_message,
-            "error": None
-        }
+        return {"message": new_message, "error": None}
 
 
 @mutation.field("updateMessage")
@@ -81,13 +73,10 @@ async def update_message(_, info, chat_id: str, message_id: int, body: str):
 
     await redis.execute("SET", f"chats/{chat_id}/messages/{message_id}", json.dumps(message))
 
-    result = FollowingResult("UPDATED", 'chat', message)
-    await FollowingManager.push('chat', result)
+    result = FollowingResult("UPDATED", "chat", message)
+    await FollowingManager.push("chat", result)
 
-    return {
-        "message": message,
-        "error": None
-    }
+    return {"message": message, "error": None}
 
 
 @mutation.field("deleteMessage")
@@ -114,7 +103,7 @@ async def delete_message(_, info, chat_id: str, message_id: int):
     for user_id in users:
         await redis.execute("LREM", f"chats/{chat_id}/unread/{user_id}", 0, str(message_id))
 
-    result = FollowingResult("DELETED", 'chat', message)
+    result = FollowingResult("DELETED", "chat", message)
     await FollowingManager.push(result)
 
     return {}
@@ -137,6 +126,4 @@ async def mark_as_read(_, info, chat_id: str, messages: [int]):
     for message_id in messages:
         await redis.execute("LREM", f"chats/{chat_id}/unread/{auth.user_id}", 0, str(message_id))
 
-    return {
-        "error": None
-    }
+    return {"error": None}

resolvers/inbox/search.py

@@ -1,10 +1,11 @@
 import json
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta, timezone
+
 from auth.authenticate import login_required
 from auth.credentials import AuthCredentials
+from base.orm import local_session
 from base.redis import redis
 from base.resolvers import query
-from base.orm import local_session
 from orm.user import AuthorFollower, User
 from resolvers.inbox.load import load_messages
 
@@ -17,7 +18,7 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int =
     auth: AuthCredentials = info.context["request"].auth
     talk_before = await redis.execute("GET", f"/chats_by_user/{auth.user_id}")
     if talk_before:
-        talk_before = list(json.loads(talk_before))[offset:offset + limit]
+        talk_before = list(json.loads(talk_before))[offset : offset + limit]
         for chat_id in talk_before:
             members = await redis.execute("GET", f"/chats/{chat_id}/users")
             if members:
@@ -31,23 +32,24 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int =
 
     with local_session() as session:
         # followings
-        result += session.query(AuthorFollower.author).join(
-            User, User.id == AuthorFollower.follower
-        ).where(
-            User.slug.startswith(query)
-        ).offset(offset + len(result)).limit(more_amount)
+        result += (
+            session.query(AuthorFollower.author)
+            .join(User, User.id == AuthorFollower.follower)
+            .where(User.slug.startswith(query))
+            .offset(offset + len(result))
+            .limit(more_amount)
+        )
 
         more_amount = limit
         # followers
-        result += session.query(AuthorFollower.follower).join(
-            User, User.id == AuthorFollower.author
-        ).where(
-            User.slug.startswith(query)
-        ).offset(offset + len(result)).limit(offset + len(result) + limit)
-    return {
-        "members": list(result),
-        "error": None
-    }
+        result += (
+            session.query(AuthorFollower.follower)
+            .join(User, User.id == AuthorFollower.author)
+            .where(User.slug.startswith(query))
+            .offset(offset + len(result))
+            .limit(offset + len(result) + limit)
+        )
+    return {"members": list(result), "error": None}
 
 
 @query.field("searchMessages")
@@ -57,22 +59,22 @@ async def search_user_chats(by, messages, user_id: int, limit, offset):
     cids.union(set(await redis.execute("SMEMBERS", "chats_by_user/" + str(user_id))))
     messages = []
 
-    by_author = by.get('author')
+    by_author = by.get("author")
     if by_author:
         # all author's messages
         cids.union(set(await redis.execute("SMEMBERS", f"chats_by_user/{by_author}")))
         # author's messages in filtered chat
         messages.union(set(filter(lambda m: m["author"] == by_author, list(messages))))
         for c in cids:
-            c = c.decode('utf-8')
+            c = c.decode("utf-8")
             messages = await load_messages(c, limit, offset)
 
-    body_like = by.get('body')
+    body_like = by.get("body")
     if body_like:
         # search in all messages in all user's chats
         for c in cids:
             # FIXME: use redis scan here
-            c = c.decode('utf-8')
+            c = c.decode("utf-8")
             mmm = await load_messages(c, limit, offset)
             for m in mmm:
                 if body_like in m["body"]:
@@ -83,13 +85,12 @@ async def search_user_chats(by, messages, user_id: int, limit, offset):
 
     days = by.get("days")
     if days:
-        messages.extend(filter(
-            list(messages),
-            key=lambda m: (
-                datetime.now(tz=timezone.utc) - int(m["createdAt"]) < timedelta(days=by["days"])
+        messages.extend(
+            filter(
+                list(messages),
+                key=lambda m: (
+                    datetime.now(tz=timezone.utc) - int(m["createdAt"]) < timedelta(days=by["days"])
+                ),
             )
-        ))
-    return {
-        "messages": messages,
-        "error": None
-    }
+        )
+    return {"messages": messages, "error": None}

resolvers/notifications.py

@@ -1,9 +1,9 @@
-from sqlalchemy import select, desc, and_, update
+from sqlalchemy import and_, desc, select, update
 
-from auth.credentials import AuthCredentials
-from base.resolvers import query, mutation
 from auth.authenticate import login_required
+from auth.credentials import AuthCredentials
 from base.orm import local_session
+from base.resolvers import mutation, query
 from orm import Notification
 
 
@@ -16,25 +16,26 @@ async def load_notifications(_, info, params=None):
     auth: AuthCredentials = info.context["request"].auth
     user_id = auth.user_id
 
-    limit = params.get('limit', 50)
-    offset = params.get('offset', 0)
+    limit = params.get("limit", 50)
+    offset = params.get("offset", 0)
 
-    q = select(Notification).where(
-        Notification.user == user_id
-    ).order_by(desc(Notification.createdAt)).limit(limit).offset(offset)
+    q = (
+        select(Notification)
+        .where(Notification.user == user_id)
+        .order_by(desc(Notification.createdAt))
+        .limit(limit)
+        .offset(offset)
+    )
 
     notifications = []
     with local_session() as session:
-        total_count = session.query(Notification).where(
-            Notification.user == user_id
-        ).count()
+        total_count = session.query(Notification).where(Notification.user == user_id).count()
 
-        total_unread_count = session.query(Notification).where(
-            and_(
-                Notification.user == user_id,
-                Notification.seen == False
-            )
-        ).count()
+        total_unread_count = (
+            session.query(Notification)
+            .where(and_(Notification.user == user_id, Notification.seen == False))  # noqa: E712
+            .count()
+        )
 
         for [notification] in session.execute(q):
             notification.type = notification.type.name
@@ -43,7 +44,7 @@ async def load_notifications(_, info, params=None):
     return {
         "notifications": notifications,
         "totalCount": total_count,
-        "totalUnreadCount": total_unread_count
+        "totalUnreadCount": total_unread_count,
     }
 
 
@@ -54,9 +55,11 @@ async def mark_notification_as_read(_, info, notification_id: int):
     user_id = auth.user_id
 
     with local_session() as session:
-        notification = session.query(Notification).where(
-            and_(Notification.id == notification_id, Notification.user == user_id)
-        ).one()
+        notification = (
+            session.query(Notification)
+            .where(and_(Notification.id == notification_id, Notification.user == user_id))
+            .one()
+        )
         notification.seen = True
         session.commit()
 
@@ -69,12 +72,11 @@ async def mark_all_notifications_as_read(_, info):
     auth: AuthCredentials = info.context["request"].auth
     user_id = auth.user_id
 
-    statement = update(Notification).where(
-        and_(
-            Notification.user == user_id,
-            Notification.seen == False
-        )
-    ).values(seen=True)
+    statement = (
+        update(Notification)
+        .where(and_(Notification.user == user_id, Notification.seen == False))  # noqa: E712
+        .values(seen=True)
+    )
 
     with local_session() as session:
         try:

resolvers/upload.py

@@ -2,33 +2,36 @@ import os
 import shutil
 import tempfile
 import uuid
+
 import boto3
 from botocore.exceptions import BotoCoreError, ClientError
 from starlette.responses import JSONResponse
 
-STORJ_ACCESS_KEY = os.environ.get('STORJ_ACCESS_KEY')
-STORJ_SECRET_KEY = os.environ.get('STORJ_SECRET_KEY')
-STORJ_END_POINT = os.environ.get('STORJ_END_POINT')
-STORJ_BUCKET_NAME = os.environ.get('STORJ_BUCKET_NAME')
-CDN_DOMAIN = os.environ.get('CDN_DOMAIN')
+STORJ_ACCESS_KEY = os.environ.get("STORJ_ACCESS_KEY")
+STORJ_SECRET_KEY = os.environ.get("STORJ_SECRET_KEY")
+STORJ_END_POINT = os.environ.get("STORJ_END_POINT")
+STORJ_BUCKET_NAME = os.environ.get("STORJ_BUCKET_NAME")
+CDN_DOMAIN = os.environ.get("CDN_DOMAIN")
 
 
 async def upload_handler(request):
     form = await request.form()
-    file = form.get('file')
+    file = form.get("file")
 
     if file is None:
-        return JSONResponse({'error': 'No file uploaded'}, status_code=400)
+        return JSONResponse({"error": "No file uploaded"}, status_code=400)
 
     file_name, file_extension = os.path.splitext(file.filename)
 
-    key = 'files/' + str(uuid.uuid4()) + file_extension
+    key = "files/" + str(uuid.uuid4()) + file_extension
 
     # Create an S3 client with Storj configuration
-    s3 = boto3.client('s3',
-                      aws_access_key_id=STORJ_ACCESS_KEY,
-                      aws_secret_access_key=STORJ_SECRET_KEY,
-                      endpoint_url=STORJ_END_POINT)
+    s3 = boto3.client(
+        "s3",
+        aws_access_key_id=STORJ_ACCESS_KEY,
+        aws_secret_access_key=STORJ_SECRET_KEY,
+        endpoint_url=STORJ_END_POINT,
+    )
 
     try:
         # Save the uploaded file to a temporary file
@@ -39,18 +42,13 @@ async def upload_handler(request):
                 Filename=tmp_file.name,
                 Bucket=STORJ_BUCKET_NAME,
                 Key=key,
-                ExtraArgs={
-                    "ContentType": file.content_type
-                }
+                ExtraArgs={"ContentType": file.content_type},
             )
 
-        url = 'https://' + CDN_DOMAIN + '/' + key
+        url = "https://" + CDN_DOMAIN + "/" + key
 
-        return JSONResponse({'url': url, 'originalFilename': file.filename})
+        return JSONResponse({"url": url, "originalFilename": file.filename})
 
     except (BotoCoreError, ClientError) as e:
         print(e)
-        return JSONResponse({'error': 'Failed to upload file'}, status_code=500)
-
-
-
+        return JSONResponse({"error": "Failed to upload file"}, status_code=500)

resolvers/zine/following.py

@@ -1,41 +1,36 @@
-import asyncio
-from base.orm import local_session
-from base.resolvers import mutation
 from auth.authenticate import login_required
 from auth.credentials import AuthCredentials
+from base.resolvers import mutation
+
 # from resolvers.community import community_follow, community_unfollow
-from orm.user import AuthorFollower
-from orm.topic import TopicFollower
-from orm.shout import ShoutReactionsFollower
 from resolvers.zine.profile import author_follow, author_unfollow
 from resolvers.zine.reactions import reactions_follow, reactions_unfollow
 from resolvers.zine.topics import topic_follow, topic_unfollow
-from services.following import Following, FollowingManager, FollowingResult
-from graphql.type import GraphQLResolveInfo
+from services.following import FollowingManager, FollowingResult
 
 
 @mutation.field("follow")
 @login_required
-async def follow(_, info, what, slug):
+async def follow(_, info, what, slug):  # noqa: C901
     auth: AuthCredentials = info.context["request"].auth
 
     try:
         if what == "AUTHOR":
             if author_follow(auth.user_id, slug):
-                result = FollowingResult("NEW", 'author', slug)
-                await FollowingManager.push('author', result)
+                result = FollowingResult("NEW", "author", slug)
+                await FollowingManager.push("author", result)
         elif what == "TOPIC":
             if topic_follow(auth.user_id, slug):
-                result = FollowingResult("NEW", 'topic', slug)
-                await FollowingManager.push('topic', result)
+                result = FollowingResult("NEW", "topic", slug)
+                await FollowingManager.push("topic", result)
         elif what == "COMMUNITY":
             if False:  # TODO: use community_follow(auth.user_id, slug):
-                result = FollowingResult("NEW", 'community', slug)
-                await FollowingManager.push('community', result)
+                result = FollowingResult("NEW", "community", slug)
+                await FollowingManager.push("community", result)
         elif what == "REACTIONS":
             if reactions_follow(auth.user_id, slug):
-                result = FollowingResult("NEW", 'shout', slug)
-                await FollowingManager.push('shout', result)
+                result = FollowingResult("NEW", "shout", slug)
+                await FollowingManager.push("shout", result)
     except Exception as e:
         print(Exception(e))
         return {"error": str(e)}
@@ -45,26 +40,26 @@ async def follow(_, info, what, slug):
 
 @mutation.field("unfollow")
 @login_required
-async def unfollow(_, info, what, slug):
+async def unfollow(_, info, what, slug):  # noqa: C901
     auth: AuthCredentials = info.context["request"].auth
 
     try:
         if what == "AUTHOR":
             if author_unfollow(auth.user_id, slug):
-                result = FollowingResult("DELETED", 'author', slug)
-                await FollowingManager.push('author', result)
+                result = FollowingResult("DELETED", "author", slug)
+                await FollowingManager.push("author", result)
         elif what == "TOPIC":
             if topic_unfollow(auth.user_id, slug):
-                result = FollowingResult("DELETED", 'topic', slug)
-                await FollowingManager.push('topic', result)
+                result = FollowingResult("DELETED", "topic", slug)
+                await FollowingManager.push("topic", result)
         elif what == "COMMUNITY":
             if False:  # TODO: use community_unfollow(auth.user_id, slug):
-                result = FollowingResult("DELETED", 'community', slug)
-                await FollowingManager.push('community', result)
+                result = FollowingResult("DELETED", "community", slug)
+                await FollowingManager.push("community", result)
         elif what == "REACTIONS":
             if reactions_unfollow(auth.user_id, slug):
-                result = FollowingResult("DELETED", 'shout', slug)
-                await FollowingManager.push('shout', result)
+                result = FollowingResult("DELETED", "shout", slug)
+                await FollowingManager.push("shout", result)
     except Exception as e:
         return {"error": str(e)}
 

resolvers/zine/load.py

@@ -1,11 +1,11 @@
 from datetime import datetime, timedelta, timezone
 
-from sqlalchemy.orm import joinedload, aliased
-from sqlalchemy.sql.expression import desc, asc, select, func, case, and_, text, nulls_last
+from sqlalchemy.orm import aliased, joinedload
+from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select
 
 from auth.authenticate import login_required
 from auth.credentials import AuthCredentials
-from base.exceptions import ObjectNotExist, OperationNotAllowed
+from base.exceptions import ObjectNotExist
 from base.orm import local_session
 from base.resolvers import query
 from orm import TopicFollower
@@ -18,37 +18,37 @@ def add_stat_columns(q):
     aliased_reaction = aliased(Reaction)
 
     q = q.outerjoin(aliased_reaction).add_columns(
-        func.sum(
-            aliased_reaction.id
-        ).label('reacted_stat'),
+        func.sum(aliased_reaction.id).label("reacted_stat"),
+        func.sum(case((aliased_reaction.kind == ReactionKind.COMMENT, 1), else_=0)).label(
+            "commented_stat"
+        ),
         func.sum(
             case(
-                (aliased_reaction.kind == ReactionKind.COMMENT, 1),
-                else_=0
+                # do not count comments' reactions
+                (aliased_reaction.replyTo.is_not(None), 0),
+                (aliased_reaction.kind == ReactionKind.AGREE, 1),
+                (aliased_reaction.kind == ReactionKind.DISAGREE, -1),
+                (aliased_reaction.kind == ReactionKind.PROOF, 1),
+                (aliased_reaction.kind == ReactionKind.DISPROOF, -1),
+                (aliased_reaction.kind == ReactionKind.ACCEPT, 1),
+                (aliased_reaction.kind == ReactionKind.REJECT, -1),
+                (aliased_reaction.kind == ReactionKind.LIKE, 1),
+                (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
+                else_=0,
             )
-        ).label('commented_stat'),
-        func.sum(case(
-            # do not count comments' reactions
-            (aliased_reaction.replyTo.is_not(None), 0),
-            (aliased_reaction.kind == ReactionKind.AGREE, 1),
-            (aliased_reaction.kind == ReactionKind.DISAGREE, -1),
-            (aliased_reaction.kind == ReactionKind.PROOF, 1),
-            (aliased_reaction.kind == ReactionKind.DISPROOF, -1),
-            (aliased_reaction.kind == ReactionKind.ACCEPT, 1),
-            (aliased_reaction.kind == ReactionKind.REJECT, -1),
-            (aliased_reaction.kind == ReactionKind.LIKE, 1),
-            (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
-            else_=0)
-        ).label('rating_stat'),
-        func.max(case(
-            (aliased_reaction.kind != ReactionKind.COMMENT, None),
-            else_=aliased_reaction.createdAt
-        )).label('last_comment'))
+        ).label("rating_stat"),
+        func.max(
+            case(
+                (aliased_reaction.kind != ReactionKind.COMMENT, None),
+                else_=aliased_reaction.createdAt,
+            )
+        ).label("last_comment"),
+    )
 
     return q
 
 
-def apply_filters(q, filters, user_id=None):
+def apply_filters(q, filters, user_id=None):  # noqa: C901
     if filters.get("reacted") and user_id:
         q.join(Reaction, Reaction.createdBy == user_id)
 
@@ -60,7 +60,7 @@ def apply_filters(q, filters, user_id=None):
 
     if filters.get("layout"):
         q = q.filter(Shout.layout == filters.get("layout"))
-    if filters.get('excludeLayout'):
+    if filters.get("excludeLayout"):
         q = q.filter(Shout.layout != filters.get("excludeLayout"))
     if filters.get("author"):
         q = q.filter(Shout.authors.any(slug=filters.get("author")))
@@ -87,27 +87,23 @@ async def load_shout(_, info, slug=None, shout_id=None):
         q = add_stat_columns(q)
 
         if slug is not None:
-            q = q.filter(
-                Shout.slug == slug
-            )
+            q = q.filter(Shout.slug == slug)
 
         if shout_id is not None:
-            q = q.filter(
-                Shout.id == shout_id
-            )
+            q = q.filter(Shout.id == shout_id)
 
-        q = q.filter(
-            Shout.deletedAt.is_(None)
-        ).group_by(Shout.id)
+        q = q.filter(Shout.deletedAt.is_(None)).group_by(Shout.id)
 
         try:
-            [shout, reacted_stat, commented_stat, rating_stat, last_comment] = session.execute(q).first()
+            [shout, reacted_stat, commented_stat, rating_stat, last_comment] = session.execute(
+                q
+            ).first()
 
             shout.stat = {
                 "viewed": shout.views,
                 "reacted": reacted_stat,
                 "commented": commented_stat,
-                "rating": rating_stat
+                "rating": rating_stat,
             }
 
             for author_caption in session.query(ShoutAuthor).join(Shout).where(Shout.slug == slug):
@@ -142,14 +138,13 @@ async def load_shouts_by(_, info, options):
     :return: Shout[]
     """
 
-    q = select(Shout).options(
-        joinedload(Shout.authors),
-        joinedload(Shout.topics),
-    ).where(
-        and_(
-            Shout.deletedAt.is_(None),
-            Shout.layout.is_not(None)
+    q = (
+        select(Shout)
+        .options(
+            joinedload(Shout.authors),
+            joinedload(Shout.topics),
         )
+        .where(and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None)))
     )
 
     q = add_stat_columns(q)
@@ -159,7 +154,7 @@ async def load_shouts_by(_, info, options):
 
     order_by = options.get("order_by", Shout.publishedAt)
 
-    query_order_by = desc(order_by) if options.get('order_by_desc', True) else asc(order_by)
+    query_order_by = desc(order_by) if options.get("order_by_desc", True) else asc(order_by)
     offset = options.get("offset", 0)
     limit = options.get("limit", 10)
 
@@ -169,13 +164,15 @@ async def load_shouts_by(_, info, options):
     with local_session() as session:
         shouts_map = {}
 
-        for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute(q).unique():
+        for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute(
+            q
+        ).unique():
             shouts.append(shout)
             shout.stat = {
                 "viewed": shout.views,
                 "reacted": reacted_stat,
                 "commented": commented_stat,
-                "rating": rating_stat
+                "rating": rating_stat,
             }
             shouts_map[shout.id] = shout
 
@@ -188,11 +185,13 @@ async def get_drafts(_, info):
     auth: AuthCredentials = info.context["request"].auth
     user_id = auth.user_id
 
-    q = select(Shout).options(
-        joinedload(Shout.authors),
-        joinedload(Shout.topics),
-    ).where(
-        and_(Shout.deletedAt.is_(None), Shout.createdBy == user_id)
+    q = (
+        select(Shout)
+        .options(
+            joinedload(Shout.authors),
+            joinedload(Shout.topics),
+        )
+        .where(and_(Shout.deletedAt.is_(None), Shout.createdBy == user_id))
     )
 
     q = q.group_by(Shout.id)
@@ -211,24 +210,22 @@ async def get_my_feed(_, info, options):
     auth: AuthCredentials = info.context["request"].auth
     user_id = auth.user_id
 
-    subquery = select(Shout.id).join(
-        ShoutAuthor
-    ).join(
-        AuthorFollower, AuthorFollower.follower == user_id
-    ).join(
-        ShoutTopic
-    ).join(
-        TopicFollower, TopicFollower.follower == user_id
+    subquery = (
+        select(Shout.id)
+        .join(ShoutAuthor)
+        .join(AuthorFollower, AuthorFollower.follower == user_id)
+        .join(ShoutTopic)
+        .join(TopicFollower, TopicFollower.follower == user_id)
     )
 
-    q = select(Shout).options(
-        joinedload(Shout.authors),
-        joinedload(Shout.topics),
-    ).where(
-        and_(
-            Shout.publishedAt.is_not(None),
-            Shout.deletedAt.is_(None),
-            Shout.id.in_(subquery)
+    q = (
+        select(Shout)
+        .options(
+            joinedload(Shout.authors),
+            joinedload(Shout.topics),
+        )
+        .where(
+            and_(Shout.publishedAt.is_not(None), Shout.deletedAt.is_(None), Shout.id.in_(subquery))
         )
     )
 
@@ -237,7 +234,7 @@ async def get_my_feed(_, info, options):
 
     order_by = options.get("order_by", Shout.publishedAt)
 
-    query_order_by = desc(order_by) if options.get('order_by_desc', True) else asc(order_by)
+    query_order_by = desc(order_by) if options.get("order_by_desc", True) else asc(order_by)
     offset = options.get("offset", 0)
     limit = options.get("limit", 10)
 
@@ -246,13 +243,15 @@ async def get_my_feed(_, info, options):
     shouts = []
     with local_session() as session:
         shouts_map = {}
-        for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute(q).unique():
+        for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute(
+            q
+        ).unique():
             shouts.append(shout)
             shout.stat = {
                 "viewed": shout.views,
                 "reacted": reacted_stat,
                 "commented": commented_stat,
-                "rating": rating_stat
+                "rating": rating_stat,
             }
             shouts_map[shout.id] = shout
 

resolvers/zine/profile.py

@@ -1,6 +1,7 @@
-from typing import List
 from datetime import datetime, timedelta, timezone
-from sqlalchemy import and_, func, distinct, select, literal
+from typing import List
+
+from sqlalchemy import and_, distinct, func, literal, select
 from sqlalchemy.orm import aliased, joinedload
 
 from auth.authenticate import login_required
@@ -21,27 +22,27 @@ def add_author_stat_columns(q):
     # user_rating_aliased = aliased(UserRating)
 
     q = q.outerjoin(shout_author_aliased).add_columns(
-        func.count(distinct(shout_author_aliased.shout)).label('shouts_stat')
+        func.count(distinct(shout_author_aliased.shout)).label("shouts_stat")
     )
     q = q.outerjoin(author_followers, author_followers.author == User.id).add_columns(
-        func.count(distinct(author_followers.follower)).label('followers_stat')
+        func.count(distinct(author_followers.follower)).label("followers_stat")
     )
 
     q = q.outerjoin(author_following, author_following.follower == User.id).add_columns(
-        func.count(distinct(author_following.author)).label('followings_stat')
+        func.count(distinct(author_following.author)).label("followings_stat")
     )
 
-    q = q.add_columns(literal(0).label('rating_stat'))
+    q = q.add_columns(literal(0).label("rating_stat"))
     # FIXME
     # q = q.outerjoin(user_rating_aliased, user_rating_aliased.user == User.id).add_columns(
     #     # TODO: check
     #     func.sum(user_rating_aliased.value).label('rating_stat')
     # )
 
-    q = q.add_columns(literal(0).label('commented_stat'))
-    # q = q.outerjoin(Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))).add_columns(
-    #     func.count(distinct(Reaction.id)).label('commented_stat')
-    # )
+    q = q.add_columns(literal(0).label("commented_stat"))
+    # q = q.outerjoin(
+    #     Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))
+    # ).add_columns(func.count(distinct(Reaction.id)).label("commented_stat"))
 
     q = q.group_by(User.id)
 
@@ -55,7 +56,7 @@ def add_stat(author, stat_columns):
         "followers": followers_stat,
         "followings": followings_stat,
         "rating": rating_stat,
-        "commented": commented_stat
+        "commented": commented_stat,
     }
 
     return author
@@ -119,10 +120,10 @@ async def user_followers(_, _info, slug) -> List[User]:
     q = add_author_stat_columns(q)
 
     aliased_user = aliased(User)
-    q = q.join(AuthorFollower, AuthorFollower.follower == User.id).join(
-        aliased_user, aliased_user.id == AuthorFollower.author
-    ).where(
-        aliased_user.slug == slug
+    q = (
+        q.join(AuthorFollower, AuthorFollower.follower == User.id)
+        .join(aliased_user, aliased_user.id == AuthorFollower.author)
+        .where(aliased_user.slug == slug)
     )
 
     return get_authors_from_query(q)
@@ -150,15 +151,10 @@ async def update_profile(_, info, profile):
     with local_session() as session:
         user = session.query(User).filter(User.id == user_id).one()
         if not user:
-            return {
-                "error": "canoot find user"
-            }
+            return {"error": "canoot find user"}
         user.update(profile)
         session.commit()
-    return {
-        "error": None,
-        "author": user
-    }
+    return {"error": None, "author": user}
 
 
 @mutation.field("rateUser")
@@ -192,7 +188,8 @@ def author_follow(user_id, slug):
             session.add(af)
             session.commit()
         return True
-    except:
+    except Exception as e:
+        print(e)
         return False
 
 
@@ -200,13 +197,10 @@ def author_follow(user_id, slug):
 def author_unfollow(user_id, slug):
     with local_session() as session:
         flw = (
-            session.query(
-                AuthorFollower
-            ).join(User, User.id == AuthorFollower.author).filter(
-                and_(
-                    AuthorFollower.follower == user_id, User.slug == slug
-                )
-            ).first()
+            session.query(AuthorFollower)
+            .join(User, User.id == AuthorFollower.author)
+            .filter(and_(AuthorFollower.follower == user_id, User.slug == slug))
+            .first()
         )
         if flw:
             session.delete(flw)
@@ -232,12 +226,11 @@ async def get_author(_, _info, slug):
     [author] = get_authors_from_query(q)
 
     with local_session() as session:
-        comments_count = session.query(Reaction).where(
-            and_(
-                Reaction.createdBy == author.id,
-                Reaction.kind == ReactionKind.COMMENT
-            )
-        ).count()
+        comments_count = (
+            session.query(Reaction)
+            .where(and_(Reaction.createdBy == author.id, Reaction.kind == ReactionKind.COMMENT))
+            .count()
+        )
         author.stat["commented"] = comments_count
 
     return author
@@ -260,9 +253,7 @@ async def load_authors_by(_, info, by, limit, offset):
         days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"])
         q = q.filter(User.createdAt > days_before)
 
-    q = q.order_by(
-        by.get("order", User.createdAt)
-    ).limit(limit).offset(offset)
+    q = q.order_by(by.get("order", User.createdAt)).limit(limit).offset(offset)
 
     return get_authors_from_query(q)
 
@@ -273,13 +264,13 @@ async def load_my_subscriptions(_, info):
     auth = info.context["request"].auth
     user_id = auth.user_id
 
-    authors_query = select(User).join(AuthorFollower, AuthorFollower.author == User.id).where(
-        AuthorFollower.follower == user_id
+    authors_query = (
+        select(User)
+        .join(AuthorFollower, AuthorFollower.author == User.id)
+        .where(AuthorFollower.follower == user_id)
     )
 
-    topics_query = select(Topic).join(TopicFollower).where(
-        TopicFollower.follower == user_id
-    )
+    topics_query = select(Topic).join(TopicFollower).where(TopicFollower.follower == user_id)
 
     topics = []
     authors = []
@@ -291,7 +282,4 @@ async def load_my_subscriptions(_, info):
         for [topic] in session.execute(topics_query):
             topics.append(topic)
 
-    return {
-        "topics": topics,
-        "authors": authors
-    }
+    return {"topics": topics, "authors": authors}

resolvers/zine/reactions.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta, timezone
-from sqlalchemy import and_, asc, desc, select, text, func, case
+
+from sqlalchemy import and_, asc, case, desc, func, select, text
 from sqlalchemy.orm import aliased
 
 from auth.authenticate import login_required
@@ -17,26 +18,22 @@ def add_reaction_stat_columns(q):
     aliased_reaction = aliased(Reaction)
 
     q = q.outerjoin(aliased_reaction, Reaction.id == aliased_reaction.replyTo).add_columns(
-        func.sum(
-            aliased_reaction.id
-        ).label('reacted_stat'),
+        func.sum(aliased_reaction.id).label("reacted_stat"),
+        func.sum(case((aliased_reaction.body.is_not(None), 1), else_=0)).label("commented_stat"),
         func.sum(
             case(
-                (aliased_reaction.body.is_not(None), 1),
-                else_=0
+                (aliased_reaction.kind == ReactionKind.AGREE, 1),
+                (aliased_reaction.kind == ReactionKind.DISAGREE, -1),
+                (aliased_reaction.kind == ReactionKind.PROOF, 1),
+                (aliased_reaction.kind == ReactionKind.DISPROOF, -1),
+                (aliased_reaction.kind == ReactionKind.ACCEPT, 1),
+                (aliased_reaction.kind == ReactionKind.REJECT, -1),
+                (aliased_reaction.kind == ReactionKind.LIKE, 1),
+                (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
+                else_=0,
             )
-        ).label('commented_stat'),
-        func.sum(case(
-            (aliased_reaction.kind == ReactionKind.AGREE, 1),
-            (aliased_reaction.kind == ReactionKind.DISAGREE, -1),
-            (aliased_reaction.kind == ReactionKind.PROOF, 1),
-            (aliased_reaction.kind == ReactionKind.DISPROOF, -1),
-            (aliased_reaction.kind == ReactionKind.ACCEPT, 1),
-            (aliased_reaction.kind == ReactionKind.REJECT, -1),
-            (aliased_reaction.kind == ReactionKind.LIKE, 1),
-            (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
-            else_=0)
-        ).label('rating_stat'))
+        ).label("rating_stat"),
+    )
 
     return q
 
@@ -47,22 +44,25 @@ def reactions_follow(user_id, shout_id: int, auto=False):
             shout = session.query(Shout).where(Shout.id == shout_id).one()
 
             following = (
-                session.query(ShoutReactionsFollower).where(and_(
-                    ShoutReactionsFollower.follower == user_id,
-                    ShoutReactionsFollower.shout == shout.id,
-                )).first()
+                session.query(ShoutReactionsFollower)
+                .where(
+                    and_(
+                        ShoutReactionsFollower.follower == user_id,
+                        ShoutReactionsFollower.shout == shout.id,
+                    )
+                )
+                .first()
             )
 
             if not following:
                 following = ShoutReactionsFollower.create(
-                    follower=user_id,
-                    shout=shout.id,
-                    auto=auto
+                    follower=user_id, shout=shout.id, auto=auto
                 )
                 session.add(following)
                 session.commit()
                 return True
-    except:
+    except Exception as e:
+        print(e)
         return False
 
 
@@ -72,46 +72,52 @@ def reactions_unfollow(user_id: int, shout_id: int):
             shout = session.query(Shout).where(Shout.id == shout_id).one()
 
             following = (
-                session.query(ShoutReactionsFollower).where(and_(
-                    ShoutReactionsFollower.follower == user_id,
-                    ShoutReactionsFollower.shout == shout.id
-                )).first()
+                session.query(ShoutReactionsFollower)
+                .where(
+                    and_(
+                        ShoutReactionsFollower.follower == user_id,
+                        ShoutReactionsFollower.shout == shout.id,
+                    )
+                )
+                .first()
             )
 
             if following:
                 session.delete(following)
                 session.commit()
                 return True
-    except:
+    except Exception as e:
+        print(e)
         pass
     return False
 
 
 def is_published_author(session, user_id):
-    ''' checks if user has at least one publication '''
-    return session.query(
-        Shout
-    ).where(
-        Shout.authors.contains(user_id)
-    ).filter(
-        and_(
-            Shout.publishedAt.is_not(None),
-            Shout.deletedAt.is_(None)
-        )
-    ).count() > 0
+    """checks if user has at least one publication"""
+    return (
+        session.query(Shout)
+        .where(Shout.authors.contains(user_id))
+        .filter(and_(Shout.publishedAt.is_not(None), Shout.deletedAt.is_(None)))
+        .count()
+        > 0
+    )
 
 
 def check_to_publish(session, user_id, reaction):
-    ''' set shout to public if publicated approvers amount > 4 '''
+    """set shout to public if publicated approvers amount > 4"""
     if not reaction.replyTo and reaction.kind in [
         ReactionKind.ACCEPT,
         ReactionKind.LIKE,
-        ReactionKind.PROOF
+        ReactionKind.PROOF,
     ]:
         if is_published_author(user_id):
             # now count how many approvers are voted already
-            approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all()
-            approvers = [user_id, ]
+            approvers_reactions = (
+                session.query(Reaction).where(Reaction.shout == reaction.shout).all()
+            )
+            approvers = [
+                user_id,
+            ]
             for ar in approvers_reactions:
                 a = ar.createdBy
                 if is_published_author(session, a):
@@ -122,21 +128,17 @@ def check_to_publish(session, user_id, reaction):
 
 
 def check_to_hide(session, user_id, reaction):
-    ''' hides any shout if 20% of reactions are negative '''
+    """hides any shout if 20% of reactions are negative"""
     if not reaction.replyTo and reaction.kind in [
         ReactionKind.REJECT,
         ReactionKind.DISLIKE,
-        ReactionKind.DISPROOF
+        ReactionKind.DISPROOF,
     ]:
         # if is_published_author(user):
         approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all()
         rejects = 0
         for r in approvers_reactions:
-            if r.kind in [
-                ReactionKind.REJECT,
-                ReactionKind.DISLIKE,
-                ReactionKind.DISPROOF
-            ]:
+            if r.kind in [ReactionKind.REJECT, ReactionKind.DISLIKE, ReactionKind.DISPROOF]:
                 rejects += 1
         if len(approvers_reactions) / rejects < 5:
             return True
@@ -146,14 +148,14 @@ def check_to_hide(session, user_id, reaction):
 def set_published(session, shout_id):
     s = session.query(Shout).where(Shout.id == shout_id).first()
     s.publishedAt = datetime.now(tz=timezone.utc)
-    s.visibility = text('public')
+    s.visibility = text("public")
     session.add(s)
     session.commit()
 
 
 def set_hidden(session, shout_id):
     s = session.query(Shout).where(Shout.id == shout_id).first()
-    s.visibility = text('community')
+    s.visibility = text("community")
     session.add(s)
     session.commit()
 
@@ -162,37 +164,46 @@ def set_hidden(session, shout_id):
 @login_required
 async def create_reaction(_, info, reaction):
     auth: AuthCredentials = info.context["request"].auth
-    reaction['createdBy'] = auth.user_id
+    reaction["createdBy"] = auth.user_id
     rdict = {}
     with local_session() as session:
         shout = session.query(Shout).where(Shout.id == reaction["shout"]).one()
         author = session.query(User).where(User.id == auth.user_id).one()
 
-        if reaction["kind"] in [
-            ReactionKind.DISLIKE.name,
-            ReactionKind.LIKE.name
-        ]:
-            existing_reaction = session.query(Reaction).where(
-                and_(
-                    Reaction.shout == reaction["shout"],
-                    Reaction.createdBy == auth.user_id,
-                    Reaction.kind == reaction["kind"],
-                    Reaction.replyTo == reaction.get("replyTo")
+        if reaction["kind"] in [ReactionKind.DISLIKE.name, ReactionKind.LIKE.name]:
+            existing_reaction = (
+                session.query(Reaction)
+                .where(
+                    and_(
+                        Reaction.shout == reaction["shout"],
+                        Reaction.createdBy == auth.user_id,
+                        Reaction.kind == reaction["kind"],
+                        Reaction.replyTo == reaction.get("replyTo"),
+                    )
                 )
-            ).first()
+                .first()
+            )
 
             if existing_reaction is not None:
                 raise OperationNotAllowed("You can't vote twice")
 
-            opposite_reaction_kind = ReactionKind.DISLIKE if reaction["kind"] == ReactionKind.LIKE.name else ReactionKind.LIKE
-            opposite_reaction = session.query(Reaction).where(
+            opposite_reaction_kind = (
+                ReactionKind.DISLIKE
+                if reaction["kind"] == ReactionKind.LIKE.name
+                else ReactionKind.LIKE
+            )
+            opposite_reaction = (
+                session.query(Reaction)
+                .where(
                     and_(
                         Reaction.shout == reaction["shout"],
                         Reaction.createdBy == auth.user_id,
                         Reaction.kind == opposite_reaction_kind,
-                        Reaction.replyTo == reaction.get("replyTo")
+                        Reaction.replyTo == reaction.get("replyTo"),
                     )
-                ).first()
+                )
+                .first()
+            )
 
             if opposite_reaction is not None:
                 session.delete(opposite_reaction)
@@ -221,8 +232,8 @@ async def create_reaction(_, info, reaction):
         await notification_service.handle_new_reaction(r.id)
 
         rdict = r.dict()
-        rdict['shout'] = shout.dict()
-        rdict['createdBy'] = author.dict()
+        rdict["shout"] = shout.dict()
+        rdict["createdBy"] = author.dict()
 
         # self-regulation mechanics
         if check_to_hide(session, auth.user_id, r):
@@ -235,11 +246,7 @@ async def create_reaction(_, info, reaction):
     except Exception as e:
         print(f"[resolvers.reactions] error on reactions autofollowing: {e}")
 
-    rdict['stat'] = {
-        "commented": 0,
-        "reacted": 0,
-        "rating": 0
-    }
+    rdict["stat"] = {"commented": 0, "reacted": 0, "rating": 0}
     return {"reaction": rdict}
 
 
@@ -269,11 +276,7 @@ async def update_reaction(_, info, id, reaction={}):
         if reaction.get("range"):
             r.range = reaction.get("range")
         session.commit()
-        r.stat = {
-            "commented": commented_stat,
-            "reacted": reacted_stat,
-            "rating": rating_stat
-        }
+        r.stat = {"commented": commented_stat, "reacted": reacted_stat, "rating": rating_stat}
 
     return {"reaction": r}
 
@@ -290,17 +293,12 @@ async def delete_reaction(_, info, id):
         if r.createdBy != auth.user_id:
             return {"error": "access denied"}
 
-        if r.kind in [
-            ReactionKind.LIKE,
-            ReactionKind.DISLIKE
-        ]:
+        if r.kind in [ReactionKind.LIKE, ReactionKind.DISLIKE]:
             session.delete(r)
         else:
             r.deletedAt = datetime.now(tz=timezone.utc)
         session.commit()
-        return {
-            "reaction": r
-        }
+        return {"reaction": r}
 
 
 @query.field("loadReactionsBy")
@@ -321,12 +319,10 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
     :return: Reaction[]
     """
 
-    q = select(
-        Reaction, User, Shout
-    ).join(
-        User, Reaction.createdBy == User.id
-    ).join(
-        Shout, Reaction.shout == Shout.id
+    q = (
+        select(Reaction, User, Shout)
+        .join(User, Reaction.createdBy == User.id)
+        .join(Shout, Reaction.shout == Shout.id)
     )
 
     if by.get("shout"):
@@ -344,7 +340,7 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
     if by.get("comment"):
         q = q.filter(func.length(Reaction.body) > 0)
 
-    if len(by.get('search', '')) > 2:
+    if len(by.get("search", "")) > 2:
         q = q.filter(Reaction.body.ilike(f'%{by["body"]}%'))
 
     if by.get("days"):
@@ -352,13 +348,9 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
         q = q.filter(Reaction.createdAt > after)
 
     order_way = asc if by.get("sort", "").startswith("-") else desc
-    order_field = by.get("sort", "").replace('-', '') or Reaction.createdAt
+    order_field = by.get("sort", "").replace("-", "") or Reaction.createdAt
 
-    q = q.group_by(
-        Reaction.id, User.id, Shout.id
-    ).order_by(
-        order_way(order_field)
-    )
+    q = q.group_by(Reaction.id, User.id, Shout.id).order_by(order_way(order_field))
 
     q = add_reaction_stat_columns(q)
 
@@ -367,13 +359,15 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
     reactions = []
 
     with local_session() as session:
-        for [reaction, user, shout, reacted_stat, commented_stat, rating_stat] in session.execute(q):
+        for [reaction, user, shout, reacted_stat, commented_stat, rating_stat] in session.execute(
+            q
+        ):
             reaction.createdBy = user
             reaction.shout = shout
             reaction.stat = {
                 "rating": rating_stat,
                 "commented": commented_stat,
-                "reacted": reacted_stat
+                "reacted": reacted_stat,
             }
 
             reaction.kind = reaction.kind.name

resolvers/zine/topics.py

@@ -1,24 +1,25 @@
-from sqlalchemy import and_, select, distinct, func
+from sqlalchemy import and_, distinct, func, select
 from sqlalchemy.orm import aliased
 
 from auth.authenticate import login_required
 from base.orm import local_session
 from base.resolvers import mutation, query
-from orm.shout import ShoutTopic, ShoutAuthor
-from orm.topic import Topic, TopicFollower
 from orm import User
+from orm.shout import ShoutAuthor, ShoutTopic
+from orm.topic import Topic, TopicFollower
 
 
 def add_topic_stat_columns(q):
     aliased_shout_author = aliased(ShoutAuthor)
     aliased_topic_follower = aliased(TopicFollower)
 
-    q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic).add_columns(
-        func.count(distinct(ShoutTopic.shout)).label('shouts_stat')
-    ).outerjoin(aliased_shout_author, ShoutTopic.shout == aliased_shout_author.shout).add_columns(
-        func.count(distinct(aliased_shout_author.user)).label('authors_stat')
-    ).outerjoin(aliased_topic_follower).add_columns(
-        func.count(distinct(aliased_topic_follower.follower)).label('followers_stat')
+    q = (
+        q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic)
+        .add_columns(func.count(distinct(ShoutTopic.shout)).label("shouts_stat"))
+        .outerjoin(aliased_shout_author, ShoutTopic.shout == aliased_shout_author.shout)
+        .add_columns(func.count(distinct(aliased_shout_author.user)).label("authors_stat"))
+        .outerjoin(aliased_topic_follower)
+        .add_columns(func.count(distinct(aliased_topic_follower.follower)).label("followers_stat"))
     )
 
     q = q.group_by(Topic.id)
@@ -28,11 +29,7 @@ def add_topic_stat_columns(q):
 
 def add_stat(topic, stat_columns):
     [shouts_stat, authors_stat, followers_stat] = stat_columns
-    topic.stat = {
-        "shouts": shouts_stat,
-        "authors": authors_stat,
-        "followers": followers_stat
-    }
+    topic.stat = {"shouts": shouts_stat, "authors": authors_stat, "followers": followers_stat}
 
     return topic
 
@@ -125,7 +122,8 @@ def topic_follow(user_id, slug):
             session.add(following)
             session.commit()
             return True
-    except:
+    except Exception as e:
+        print(e)
         return False
 
 
@@ -133,18 +131,17 @@ def topic_unfollow(user_id, slug):
     try:
         with local_session() as session:
             sub = (
-                session.query(TopicFollower).join(Topic).filter(
-                    and_(
-                        TopicFollower.follower == user_id,
-                        Topic.slug == slug
-                    )
-                ).first()
+                session.query(TopicFollower)
+                .join(Topic)
+                .filter(and_(TopicFollower.follower == user_id, Topic.slug == slug))
+                .first()
             )
             if sub:
                 session.delete(sub)
                 session.commit()
                 return True
-    except:
+    except Exception as e:
+        print(e)
         pass
     return False
 

server.py

@@ -1,8 +1,9 @@
-import sys
 import os
+import sys
+
 import uvicorn
 
-from settings import PORT, DEV_SERVER_PID_FILE_NAME
+from settings import DEV_SERVER_PID_FILE_NAME, PORT
 
 
 def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook):
@@ -10,47 +11,36 @@ def exception_handler(exception_type, exception, traceback, debug_hook=sys.excep
 
 
 log_settings = {
-    'version': 1,
-    'disable_existing_loggers': True,
-    'formatters': {
-        'default': {
-            '()': 'uvicorn.logging.DefaultFormatter',
-            'fmt': '%(levelprefix)s %(message)s',
-            'use_colors': None
+    "version": 1,
+    "disable_existing_loggers": True,
+    "formatters": {
+        "default": {
+            "()": "uvicorn.logging.DefaultFormatter",
+            "fmt": "%(levelprefix)s %(message)s",
+            "use_colors": None,
+        },
+        "access": {
+            "()": "uvicorn.logging.AccessFormatter",
+            "fmt": '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s',
         },
-        'access': {
-            '()': 'uvicorn.logging.AccessFormatter',
-            'fmt': '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s'
-        }
     },
-    'handlers': {
-        'default': {
-            'formatter': 'default',
-            'class': 'logging.StreamHandler',
-            'stream': 'ext://sys.stderr'
+    "handlers": {
+        "default": {
+            "formatter": "default",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stderr",
+        },
+        "access": {
+            "formatter": "access",
+            "class": "logging.StreamHandler",
+            "stream": "ext://sys.stdout",
         },
-        'access': {
-            'formatter': 'access',
-            'class': 'logging.StreamHandler',
-            'stream': 'ext://sys.stdout'
-        }
     },
-    'loggers': {
-        'uvicorn': {
-            'handlers': ['default'],
-            'level': 'INFO'
-        },
-        'uvicorn.error': {
-            'level': 'INFO',
-            'handlers': ['default'],
-            'propagate': True
-        },
-        'uvicorn.access': {
-            'handlers': ['access'],
-            'level': 'INFO',
-            'propagate': False
-        }
-    }
+    "loggers": {
+        "uvicorn": {"handlers": ["default"], "level": "INFO"},
+        "uvicorn.error": {"level": "INFO", "handlers": ["default"], "propagate": True},
+        "uvicorn.access": {"handlers": ["access"], "level": "INFO", "propagate": False},
+    },
 }
 
 local_headers = [
@@ -58,7 +48,8 @@ local_headers = [
     ("Access-Control-Allow-Origin", "https://localhost:3000"),
     (
         "Access-Control-Allow-Headers",
-        "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization",
+        "DNT,User-Agent,X-Requested-With,If-Modified-Since,"
+        + " Cache-Control,Content-Type,Range,Authorization",
     ),
     ("Access-Control-Expose-Headers", "Content-Length,Content-Range"),
     ("Access-Control-Allow-Credentials", "true"),
@@ -86,24 +77,20 @@ if __name__ == "__main__":
             # log_config=log_settings,
             log_level=None,
             access_log=True,
-            reload=want_reload
+            reload=want_reload,
         )  # , ssl_keyfile="discours.key", ssl_certfile="discours.crt")
     elif x == "migrate":
         from migration import process
+
         print("MODE: MIGRATE")
 
         process()
     elif x == "bson":
         from migration.bson2json import json_tables
+
         print("MODE: BSON")
 
         json_tables()
     else:
         sys.excepthook = exception_handler
-        uvicorn.run(
-            "main:app",
-            host="0.0.0.0",
-            port=PORT,
-            proxy_headers=True,
-            server_header=True
-        )
+        uvicorn.run("main:app", host="0.0.0.0", port=PORT, proxy_headers=True, server_header=True)

services/following.py

@@ -18,12 +18,7 @@ class Following:
 
 class FollowingManager:
     lock = asyncio.Lock()
-    data = {
-        'author': [],
-        'topic': [],
-        'shout': [],
-        'chat': []
-    }
+    data = {"author": [], "topic": [], "shout": [], "chat": []}
 
     @staticmethod
     async def register(kind, uid):
@@ -39,13 +34,13 @@ class FollowingManager:
     async def push(kind, payload):
         try:
             async with FollowingManager.lock:
-                if kind == 'chat':
-                    for chat in FollowingManager['chat']:
+                if kind == "chat":
+                    for chat in FollowingManager["chat"]:
                         if payload.message["chatId"] == chat.uid:
                             chat.queue.put_nowait(payload)
                 else:
                     for entity in FollowingManager[kind]:
-                        if payload.shout['createdBy'] == entity.uid:
+                        if payload.shout["createdBy"] == entity.uid:
                             entity.queue.put_nowait(payload)
         except Exception as e:
             print(Exception(e))

services/main.py

@@ -1,13 +1,13 @@
+from base.orm import local_session
 from services.search import SearchService
 from services.stat.viewed import ViewedStorage
-from base.orm import local_session
 
 
 async def storages_init():
     with local_session() as session:
-        print('[main] initialize SearchService')
+        print("[main] initialize SearchService")
         await SearchService.init(session)
-        print('[main] SearchService initialized')
-        print('[main] initialize storages')
+        print("[main] SearchService initialized")
+        print("[main] initialize storages")
         await ViewedStorage.init()
-        print('[main] storages initialized')
+        print("[main] storages initialized")

services/notifications/notification_service.py

@@ -5,32 +5,24 @@ from datetime import datetime, timezone
 from sqlalchemy import and_
 
 from base.orm import local_session
-from orm import Reaction, Shout, Notification, User
+from orm import Notification, Reaction, Shout, User
 from orm.notification import NotificationType
 from orm.reaction import ReactionKind
 from services.notifications.sse import connection_manager
 
 
 def shout_to_shout_data(shout):
-    return {
-        "title": shout.title,
-        "slug": shout.slug
-    }
+    return {"title": shout.title, "slug": shout.slug}
 
 
 def user_to_user_data(user):
-    return {
-        "id": user.id,
-        "name": user.name,
-        "slug": user.slug,
-        "userpic": user.userpic
-    }
+    return {"id": user.id, "name": user.name, "slug": user.slug, "userpic": user.userpic}
 
 
 def update_prev_notification(notification, user, reaction):
     notification_data = json.loads(notification.data)
 
-    notification_data["users"] = [u for u in notification_data["users"] if u['id'] != user.id]
+    notification_data["users"] = [u for u in notification_data["users"] if u["id"] != user.id]
     notification_data["users"].append(user_to_user_data(user))
 
     if notification_data["reactionIds"] is None:
@@ -57,34 +49,45 @@ class NewReactionNotificator:
             if reaction.kind == ReactionKind.COMMENT:
                 parent_reaction = None
                 if reaction.replyTo:
-                    parent_reaction = session.query(Reaction).where(Reaction.id == reaction.replyTo).one()
+                    parent_reaction = (
+                        session.query(Reaction).where(Reaction.id == reaction.replyTo).one()
+                    )
                     if parent_reaction.createdBy != reaction.createdBy:
-                        prev_new_reply_notification = session.query(Notification).where(
-                            and_(
-                                Notification.user == shout.createdBy,
-                                Notification.type == NotificationType.NEW_REPLY,
-                                Notification.shout == shout.id,
-                                Notification.reaction == parent_reaction.id,
-                                Notification.seen == False
+                        prev_new_reply_notification = (
+                            session.query(Notification)
+                            .where(
+                                and_(
+                                    Notification.user == shout.createdBy,
+                                    Notification.type == NotificationType.NEW_REPLY,
+                                    Notification.shout == shout.id,
+                                    Notification.reaction == parent_reaction.id,
+                                    Notification.seen == False,  # noqa: E712
+                                )
                             )
-                        ).first()
+                            .first()
+                        )
 
                         if prev_new_reply_notification:
                             update_prev_notification(prev_new_reply_notification, user, reaction)
                         else:
-                            reply_notification_data = json.dumps({
-                                "shout": shout_to_shout_data(shout),
-                                "users": [user_to_user_data(user)],
-                                "reactionIds": [reaction.id]
-                            }, ensure_ascii=False)
+                            reply_notification_data = json.dumps(
+                                {
+                                    "shout": shout_to_shout_data(shout),
+                                    "users": [user_to_user_data(user)],
+                                    "reactionIds": [reaction.id],
+                                },
+                                ensure_ascii=False,
+                            )
 
-                            reply_notification = Notification.create(**{
-                                "user": parent_reaction.createdBy,
-                                "type": NotificationType.NEW_REPLY,
-                                "shout": shout.id,
-                                "reaction": parent_reaction.id,
-                                "data": reply_notification_data
-                            })
+                            reply_notification = Notification.create(
+                                **{
+                                    "user": parent_reaction.createdBy,
+                                    "type": NotificationType.NEW_REPLY,
+                                    "shout": shout.id,
+                                    "reaction": parent_reaction.id,
+                                    "data": reply_notification_data,
+                                }
+                            )
 
                             session.add(reply_notification)
 
@@ -93,30 +96,39 @@ class NewReactionNotificator:
                 if reaction.createdBy != shout.createdBy and (
                     parent_reaction is None or parent_reaction.createdBy != shout.createdBy
                 ):
-                    prev_new_comment_notification = session.query(Notification).where(
-                        and_(
-                            Notification.user == shout.createdBy,
-                            Notification.type == NotificationType.NEW_COMMENT,
-                            Notification.shout == shout.id,
-                            Notification.seen == False
+                    prev_new_comment_notification = (
+                        session.query(Notification)
+                        .where(
+                            and_(
+                                Notification.user == shout.createdBy,
+                                Notification.type == NotificationType.NEW_COMMENT,
+                                Notification.shout == shout.id,
+                                Notification.seen == False,  # noqa: E712
+                            )
                         )
-                    ).first()
+                        .first()
+                    )
 
                     if prev_new_comment_notification:
                         update_prev_notification(prev_new_comment_notification, user, reaction)
                     else:
-                        notification_data_string = json.dumps({
-                            "shout": shout_to_shout_data(shout),
-                            "users": [user_to_user_data(user)],
-                            "reactionIds": [reaction.id]
-                        }, ensure_ascii=False)
+                        notification_data_string = json.dumps(
+                            {
+                                "shout": shout_to_shout_data(shout),
+                                "users": [user_to_user_data(user)],
+                                "reactionIds": [reaction.id],
+                            },
+                            ensure_ascii=False,
+                        )
 
-                        author_notification = Notification.create(**{
-                            "user": shout.createdBy,
-                            "type": NotificationType.NEW_COMMENT,
-                            "shout": shout.id,
-                            "data": notification_data_string
-                        })
+                        author_notification = Notification.create(
+                            **{
+                                "user": shout.createdBy,
+                                "type": NotificationType.NEW_COMMENT,
+                                "shout": shout.id,
+                                "data": notification_data_string,
+                            }
+                        )
 
                         session.add(author_notification)
 
@@ -142,7 +154,7 @@ class NotificationService:
             try:
                 await notificator.run()
             except Exception as e:
-                print(f'[NotificationService.worker] error: {str(e)}')
+                print(f"[NotificationService.worker] error: {str(e)}")
 
 
 notification_service = NotificationService()

services/notifications/sse.py

@@ -1,8 +1,8 @@
+import asyncio
 import json
 
 from sse_starlette.sse import EventSourceResponse
 from starlette.requests import Request
-import asyncio
 
 
 class ConnectionManager:
@@ -28,9 +28,7 @@ class ConnectionManager:
             return
 
         for connection in self.connections_by_user_id[user_id]:
-            data = {
-                "type": "newNotifications"
-            }
+            data = {"type": "newNotifications"}
             data_string = json.dumps(data, ensure_ascii=False)
             await connection.put(data_string)
 

services/search.py

@@ -1,5 +1,7 @@
 import asyncio
 import json
+from typing import List
+
 from base.redis import redis
 from orm.shout import Shout
 from resolvers.zine.load import load_shouts_by
@@ -7,25 +9,20 @@ from resolvers.zine.load import load_shouts_by
 
 class SearchService:
     lock = asyncio.Lock()
-    cache = {}
+    # cache = {}
 
     @staticmethod
     async def init(session):
         async with SearchService.lock:
-            print('[search.service] did nothing')
-            SearchService.cache = {}
+            print("[search.service] did nothing")
+            # SearchService.cache = {}
 
     @staticmethod
-    async def search(text, limit, offset) -> [Shout]:
+    async def search(text, limit, offset) -> List[Shout]:
         cached = await redis.execute("GET", text)
         if not cached:
             async with SearchService.lock:
-                options = {
-                    "title": text,
-                    "body": text,
-                    "limit": limit,
-                    "offset": offset
-                }
+                options = {"title": text, "body": text, "limit": limit, "offset": offset}
                 payload = await load_shouts_by(None, None, options)
                 await redis.execute("SET", text, json.dumps(payload))
                 return payload

services/stat/viewed.py

@@ -1,18 +1,18 @@
 import asyncio
 import time
-from datetime import timedelta, timezone, datetime
+from datetime import datetime, timedelta, timezone
 from os import environ, path
 from ssl import create_default_context
 
 from gql import Client, gql
 from gql.transport.aiohttp import AIOHTTPTransport
-from sqlalchemy import func
 
 from base.orm import local_session
-from orm import User, Topic
-from orm.shout import ShoutTopic, Shout
+from orm import Topic
+from orm.shout import Shout, ShoutTopic
 
-load_facts = gql("""
+load_facts = gql(
+    """
 query getDomains {
     domains {
         id
@@ -25,9 +25,11 @@ query getDomains {
         }
     }
 }
-""")
+"""
+)
 
-load_pages = gql("""
+load_pages = gql(
+    """
 query getDomains {
     domains {
     title
@@ -41,8 +43,9 @@ query getDomains {
         }
     }
 }
-""")
-schema_str = open(path.dirname(__file__) + '/ackee.graphql').read()
+"""
+)
+schema_str = open(path.dirname(__file__) + "/ackee.graphql").read()
 token = environ.get("ACKEE_TOKEN", "")
 
 
@@ -50,10 +53,8 @@ def create_client(headers=None, schema=None):
     return Client(
         schema=schema,
         transport=AIOHTTPTransport(
-            url="https://ackee.discours.io/api",
-            ssl=create_default_context(),
-            headers=headers
-        )
+            url="https://ackee.discours.io/api", ssl=create_default_context(), headers=headers
+        ),
     )
 
 
@@ -71,13 +72,13 @@ class ViewedStorage:
 
     @staticmethod
     async def init():
-        """ graphql client connection using permanent token """
+        """graphql client connection using permanent token"""
         self = ViewedStorage
         async with self.lock:
             if token:
-                self.client = create_client({
-                    "Authorization": "Bearer %s" % str(token)
-                }, schema=schema_str)
+                self.client = create_client(
+                    {"Authorization": "Bearer %s" % str(token)}, schema=schema_str
+                )
                 print("[stat.viewed] * authorized permanentely by ackee.discours.io: %s" % token)
             else:
                 print("[stat.viewed] * please set ACKEE_TOKEN")
@@ -85,7 +86,7 @@ class ViewedStorage:
 
     @staticmethod
     async def update_pages():
-        """ query all the pages from ackee sorted by views count """
+        """query all the pages from ackee sorted by views count"""
         print("[stat.viewed] ⎧ updating ackee pages data ---")
         start = time.time()
         self = ViewedStorage
@@ -96,7 +97,7 @@ class ViewedStorage:
             try:
                 for page in self.pages:
                     p = page["value"].split("?")[0]
-                    slug = p.split('discours.io/')[-1]
+                    slug = p.split("discours.io/")[-1]
                     shouts[slug] = page["count"]
                 for slug in shouts.keys():
                     await ViewedStorage.increment(slug, shouts[slug])
@@ -118,7 +119,7 @@ class ViewedStorage:
     # unused yet
     @staticmethod
     async def get_shout(shout_slug):
-        """ getting shout views metric by slug """
+        """getting shout views metric by slug"""
         self = ViewedStorage
         async with self.lock:
             shout_views = self.by_shouts.get(shout_slug)
@@ -136,7 +137,7 @@ class ViewedStorage:
 
     @staticmethod
     async def get_topic(topic_slug):
-        """ getting topic views value summed """
+        """getting topic views value summed"""
         self = ViewedStorage
         topic_views = 0
         async with self.lock:
@@ -146,24 +147,28 @@ class ViewedStorage:
 
     @staticmethod
     def update_topics(session, shout_slug):
-        """ updates topics counters by shout slug """
+        """updates topics counters by shout slug"""
         self = ViewedStorage
-        for [shout_topic, topic] in session.query(ShoutTopic, Topic).join(Topic).join(Shout).where(
-            Shout.slug == shout_slug
-        ).all():
+        for [shout_topic, topic] in (
+            session.query(ShoutTopic, Topic)
+            .join(Topic)
+            .join(Shout)
+            .where(Shout.slug == shout_slug)
+            .all()
+        ):
             if not self.by_topics.get(topic.slug):
                 self.by_topics[topic.slug] = {}
             self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug]
 
     @staticmethod
-    async def increment(shout_slug, amount=1, viewer='ackee'):
-        """ the only way to change views counter """
+    async def increment(shout_slug, amount=1, viewer="ackee"):
+        """the only way to change views counter"""
         self = ViewedStorage
         async with self.lock:
             # TODO optimize, currenty we execute 1 DB transaction per shout
             with local_session() as session:
                 shout = session.query(Shout).where(Shout.slug == shout_slug).one()
-                if viewer == 'old-discours':
+                if viewer == "old-discours":
                     # this is needed for old db migration
                     if shout.viewsOld == amount:
                         print(f"viewsOld amount: {amount}")
@@ -185,7 +190,7 @@ class ViewedStorage:
 
     @staticmethod
     async def worker():
-        """ async task worker """
+        """async task worker"""
         failed = 0
         self = ViewedStorage
         if self.disabled:
@@ -205,9 +210,10 @@ class ViewedStorage:
             if failed == 0:
                 when = datetime.now(timezone.utc) + timedelta(seconds=self.period)
                 t = format(when.astimezone().isoformat())
-                print("[stat.viewed] ⎩ next update: %s" % (
-                    t.split("T")[0] + " " + t.split("T")[1].split(".")[0]
-                ))
+                print(
+                    "[stat.viewed] ⎩ next update: %s"
+                    % (t.split("T")[0] + " " + t.split("T")[1].split(".")[0])
+                )
                 await asyncio.sleep(self.period)
             else:
                 await asyncio.sleep(10)

settings.py

@@ -3,8 +3,9 @@ from os import environ
 PORT = 8080
 
 DB_URL = (
-    environ.get("DATABASE_URL") or environ.get("DB_URL") or
-    "postgresql://postgres@localhost:5432/discoursio"
+    environ.get("DATABASE_URL")
+    or environ.get("DB_URL")
+    or "postgresql://postgres@localhost:5432/discoursio"
 )
 JWT_ALGORITHM = "HS256"
 JWT_SECRET_KEY = environ.get("JWT_SECRET_KEY") or "8f1bd7696ffb482d8486dfbc6e7d16dd-secret-key"
@@ -30,4 +31,4 @@ SENTRY_DSN = environ.get("SENTRY_DSN")
 SESSION_SECRET_KEY = environ.get("SESSION_SECRET_KEY") or "!secret"
 
 # for local development
-DEV_SERVER_PID_FILE_NAME = 'dev-server.pid'
+DEV_SERVER_PID_FILE_NAME = "dev-server.pid"

setup.cfg

@@ -1,23 +1,13 @@
 [isort]
 # https://github.com/PyCQA/isort
-line_length = 120
-multi_line_output = 3
-include_trailing_comma = true
-force_grid_wrap = 0
-use_parentheses = true
-force_alphabetical_sort = false
-
-[tool:brunette]
-# https://github.com/odwyersoftware/brunette
-line-length = 120
-single-quotes = false
+profile = black
 
 [flake8]
 # https://github.com/PyCQA/flake8
-exclude = .git,__pycache__,.mypy_cache,.vercel
-max-line-length = 120
-max-complexity = 15
-select = B,C,E,F,W,T4,B9
+exclude = .git,.mypy_cache,schema_types.py
+max-line-length = 100
+max-complexity = 10
+# select = B,C,E,F,W,T4,B9
 # E203: Whitespace before ':'
 # E266: Too many leading '#' for block comment
 # E501: Line too long (82 > 79 characters)
@@ -25,15 +15,12 @@ select = B,C,E,F,W,T4,B9
 # W503: Line break occurred before a binary operator
 # F403: 'from module import *' used; unable to detect undefined names
 # C901: Function is too complex
-ignore = E203,E266,E501,E722,W503,F403,C901
+# ignore = E203,E266,E501,E722,W503,F403,C901
+extend-ignore = E203
 
 [mypy]
 # https://github.com/python/mypy
-ignore_missing_imports = true
-warn_return_any = false
-warn_unused_configs = true
-disallow_untyped_calls = true
-disallow_untyped_defs = true
-disallow_incomplete_defs = true
-[mypy-api.*]
-ignore_errors = true
+exclude = schema_types.py
+explicit_package_bases = true
+check_untyped_defs = true
+plugins = sqlmypy

validations/auth.py

@@ -1,4 +1,5 @@
 from typing import Optional, Text
+
 from pydantic import BaseModel
 
 

validations/inbox.py

@@ -1,4 +1,5 @@
-from typing import Optional, Text, List
+from typing import List, Optional, Text
+
 from pydantic import BaseModel
 
 
@@ -20,6 +21,7 @@ class Member(BaseModel):
 
 
 class Chat(BaseModel):
+    id: int
     createdAt: int
     createdBy: int
     users: List[int]