This commit is contained in:
Igor Lobanov 2023-10-26 19:56:42 +02:00
parent 44bd146bdf
commit 2c524279f6
65 changed files with 802 additions and 1049 deletions

View File

@ -6,7 +6,7 @@ exclude: |
) )
default_language_version: default_language_version:
python: python3.8 python: python3.10
repos: repos:
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
@ -23,20 +23,20 @@ repos:
- id: trailing-whitespace - id: trailing-whitespace
- repo: https://github.com/timothycrosley/isort - repo: https://github.com/timothycrosley/isort
rev: 5.5.3 rev: 5.12.0
hooks: hooks:
- id: isort - id: isort
- repo: https://github.com/ambv/black - repo: https://github.com/ambv/black
rev: 20.8b1 rev: 23.9.1
hooks: hooks:
- id: black - id: black
args: args:
- --line-length=100 - --line-length=100
- --skip-string-normalization - --skip-string-normalization
- repo: https://gitlab.com/pycqa/flake8 - repo: https://github.com/PyCQA/flake8
rev: 3.8.3 rev: 6.1.0
hooks: hooks:
- id: flake8 - id: flake8
args: args:

View File

@ -1,2 +1 @@
web: python server.py web: python server.py

View File

@ -35,6 +35,13 @@ pip install -r requirements.txt
python3 server.py dev python3 server.py dev
``` ```
# pre-commit hook
```
pip install -r requirements-dev.txt
pre-commit install
```
# How to do an authorized request # How to do an authorized request
Put the header 'Authorization' with token from signIn query or registerUser mutation. Put the header 'Authorization' with token from signIn query or registerUser mutation.
@ -42,4 +49,3 @@ Put the header 'Authorization' with token from signIn query or registerUser muta
# How to debug Ackee # How to debug Ackee
Set ACKEE_TOKEN var Set ACKEE_TOKEN var

View File

@ -1,9 +1,10 @@
import re import re
from string import punctuation
import nltk import nltk
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from nltk.corpus import stopwords from nltk.corpus import stopwords
from pymystem3 import Mystem from pymystem3 import Mystem
from string import punctuation
from transformers import BertTokenizer from transformers import BertTokenizer
nltk.download("stopwords") nltk.download("stopwords")
@ -30,9 +31,11 @@ def get_clear_text(text):
russian_stopwords = stopwords.words("russian") russian_stopwords = stopwords.words("russian")
tokens = mystem.lemmatize(clear_text) tokens = mystem.lemmatize(clear_text)
tokens = [token for token in tokens if token not in russian_stopwords \ tokens = [
and token != " " \ token
and token.strip() not in punctuation] for token in tokens
if token not in russian_stopwords and token != " " and token.strip() not in punctuation
]
clear_text = " ".join(tokens) clear_text = " ".join(tokens)

View File

@ -1,10 +1,8 @@
from logging.config import fileConfig from logging.config import fileConfig
from sqlalchemy import engine_from_config from sqlalchemy import engine_from_config, pool
from sqlalchemy import pool
from alembic import context from alembic import context
from settings import DB_URL from settings import DB_URL
# this is the Alembic Config object, which provides # this is the Alembic Config object, which provides
@ -20,6 +18,7 @@ if config.config_file_name is not None:
fileConfig(config.config_file_name) fileConfig(config.config_file_name)
from base.orm import Base from base.orm import Base
target_metadata = [Base.metadata] target_metadata = [Base.metadata]
# other values from the config, defined by the needs of env.py, # other values from the config, defined by the needs of env.py,
@ -66,9 +65,7 @@ def run_migrations_online() -> None:
) )
with connectable.connect() as connection: with connectable.connect() as connection:
context.configure( context.configure(connection=connection, target_metadata=target_metadata)
connection=connection, target_metadata=target_metadata
)
with context.begin_transaction(): with context.begin_transaction():
context.run_migrations() context.run_migrations()

View File

@ -7,9 +7,9 @@ Create Date: 2023-08-19 01:37:57.031933
""" """
from typing import Sequence, Union from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision: str = 'fe943b098418' revision: str = 'fe943b098418'

View File

@ -2,24 +2,22 @@ from functools import wraps
from typing import Optional, Tuple from typing import Optional, Tuple
from graphql.type import GraphQLResolveInfo from graphql.type import GraphQLResolveInfo
from sqlalchemy.orm import joinedload, exc from sqlalchemy.orm import exc, joinedload
from starlette.authentication import AuthenticationBackend from starlette.authentication import AuthenticationBackend
from starlette.requests import HTTPConnection from starlette.requests import HTTPConnection
from auth.credentials import AuthCredentials, AuthUser from auth.credentials import AuthCredentials, AuthUser
from base.orm import local_session
from orm.user import User, Role
from settings import SESSION_TOKEN_HEADER
from auth.tokenstorage import SessionToken from auth.tokenstorage import SessionToken
from base.exceptions import OperationNotAllowed from base.exceptions import OperationNotAllowed
from base.orm import local_session
from orm.user import Role, User
from settings import SESSION_TOKEN_HEADER
class JWTAuthenticate(AuthenticationBackend): class JWTAuthenticate(AuthenticationBackend):
async def authenticate( async def authenticate(
self, request: HTTPConnection self, request: HTTPConnection
) -> Optional[Tuple[AuthCredentials, AuthUser]]: ) -> Optional[Tuple[AuthCredentials, AuthUser]]:
if SESSION_TOKEN_HEADER not in request.headers: if SESSION_TOKEN_HEADER not in request.headers:
return AuthCredentials(scopes={}), AuthUser(user_id=None, username='') return AuthCredentials(scopes={}), AuthUser(user_id=None, username='')
@ -36,28 +34,27 @@ class JWTAuthenticate(AuthenticationBackend):
with local_session() as session: with local_session() as session:
try: try:
user = ( user = (
session.query(User).options( session.query(User)
.options(
joinedload(User.roles).options(joinedload(Role.permissions)), joinedload(User.roles).options(joinedload(Role.permissions)),
joinedload(User.ratings) joinedload(User.ratings),
).filter( )
User.id == payload.user_id .filter(User.id == payload.user_id)
).one() .one()
) )
scopes = {} # TODO: integrate await user.get_permission() scopes = {} # TODO: integrate await user.get_permission()
return ( return (
AuthCredentials( AuthCredentials(user_id=payload.user_id, scopes=scopes, logged_in=True),
user_id=payload.user_id,
scopes=scopes,
logged_in=True
),
AuthUser(user_id=user.id, username=''), AuthUser(user_id=user.id, username=''),
) )
except exc.NoResultFound: except exc.NoResultFound:
pass pass
return AuthCredentials(scopes={}, error_message=str('Invalid token')), AuthUser(user_id=None, username='') return AuthCredentials(scopes={}, error_message=str('Invalid token')), AuthUser(
user_id=None, username=''
)
def login_required(func): def login_required(func):
@ -68,9 +65,7 @@ def login_required(func):
# print(auth) # print(auth)
if not auth or not auth.logged_in: if not auth or not auth.logged_in:
# raise Unauthorized(auth.error_message or "Please login") # raise Unauthorized(auth.error_message or "Please login")
return { return {"error": "Please login first"}
"error": "Please login first"
}
return await func(parent, info, *args, **kwargs) return await func(parent, info, *args, **kwargs)
return wrap return wrap
@ -79,7 +74,9 @@ def login_required(func):
def permission_required(resource, operation, func): def permission_required(resource, operation, func):
@wraps(func) @wraps(func)
async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs): async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs):
print('[auth.authenticate] permission_required for %r with info %r' % (func, info)) # debug only print(
'[auth.authenticate] permission_required for %r with info %r' % (func, info)
) # debug only
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
if not auth.logged_in: if not auth.logged_in:
raise OperationNotAllowed(auth.error_message or "Please login") raise OperationNotAllowed(auth.error_message or "Please login")

View File

@ -23,9 +23,7 @@ class AuthCredentials(BaseModel):
async def permissions(self) -> List[Permission]: async def permissions(self) -> List[Permission]:
if self.user_id is None: if self.user_id is None:
# raise Unauthorized("Please login first") # raise Unauthorized("Please login first")
return { return {"error": "Please login first"}
"error": "Please login first"
}
else: else:
# TODO: implement permissions logix # TODO: implement permissions logix
print(self.user_id) print(self.user_id)

View File

@ -4,10 +4,7 @@ from settings import MAILGUN_API_KEY, MAILGUN_DOMAIN
api_url = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN or 'discours.io') api_url = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN or 'discours.io')
noreply = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN or 'discours.io') noreply = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN or 'discours.io')
lang_subject = { lang_subject = {"ru": "Подтверждение почты", "en": "Confirm email"}
"ru": "Подтверждение почты",
"en": "Confirm email"
}
async def send_auth_email(user, token, lang="ru", template="email_confirmation"): async def send_auth_email(user, token, lang="ru", template="email_confirmation"):
@ -22,16 +19,12 @@ async def send_auth_email(user, token, lang="ru", template="email_confirmation")
"to": to, "to": to,
"subject": subject, "subject": subject,
"template": template, "template": template,
"h:X-Mailgun-Variables": "{ \"token\": \"%s\" }" % token "h:X-Mailgun-Variables": "{ \"token\": \"%s\" }" % token,
} }
print('[auth.email] payload: %r' % payload) print('[auth.email] payload: %r' % payload)
# debug # debug
# print('http://localhost:3000/?modal=auth&mode=confirm-email&token=%s' % token) # print('http://localhost:3000/?modal=auth&mode=confirm-email&token=%s' % token)
response = requests.post( response = requests.post(api_url, auth=("api", MAILGUN_API_KEY), data=payload)
api_url,
auth=("api", MAILGUN_API_KEY),
data=payload
)
response.raise_for_status() response.raise_for_status()
except Exception as e: except Exception as e:
print(e) print(e)

View File

@ -7,6 +7,7 @@ from sqlalchemy import or_
from auth.jwtcodec import JWTCodec from auth.jwtcodec import JWTCodec
from auth.tokenstorage import TokenStorage from auth.tokenstorage import TokenStorage
# from base.exceptions import InvalidPassword, InvalidToken # from base.exceptions import InvalidPassword, InvalidToken
from base.orm import local_session from base.orm import local_session
from orm import User from orm import User
@ -57,14 +58,10 @@ class Identity:
user = User(**orm_user.dict()) user = User(**orm_user.dict())
if not user.password: if not user.password:
# raise InvalidPassword("User password is empty") # raise InvalidPassword("User password is empty")
return { return {"error": "User password is empty"}
"error": "User password is empty"
}
if not Password.verify(password, user.password): if not Password.verify(password, user.password):
# raise InvalidPassword("Wrong user password") # raise InvalidPassword("Wrong user password")
return { return {"error": "Wrong user password"}
"error": "Wrong user password"
}
return user return user
@staticmethod @staticmethod
@ -91,26 +88,18 @@ class Identity:
payload = JWTCodec.decode(token) payload = JWTCodec.decode(token)
if not await TokenStorage.exist(f"{payload.user_id}-{payload.username}-{token}"): if not await TokenStorage.exist(f"{payload.user_id}-{payload.username}-{token}"):
# raise InvalidToken("Login token has expired, please login again") # raise InvalidToken("Login token has expired, please login again")
return { return {"error": "Token has expired"}
"error": "Token has expired"
}
except ExpiredSignatureError: except ExpiredSignatureError:
# raise InvalidToken("Login token has expired, please try again") # raise InvalidToken("Login token has expired, please try again")
return { return {"error": "Token has expired"}
"error": "Token has expired"
}
except DecodeError: except DecodeError:
# raise InvalidToken("token format error") from e # raise InvalidToken("token format error") from e
return { return {"error": "Token format error"}
"error": "Token format error"
}
with local_session() as session: with local_session() as session:
user = session.query(User).filter_by(id=payload.user_id).first() user = session.query(User).filter_by(id=payload.user_id).first()
if not user: if not user:
# raise Exception("user not exist") # raise Exception("user not exist")
return { return {"error": "User does not exist"}
"error": "User does not exist"
}
if not user.emailConfirmed: if not user.emailConfirmed:
user.emailConfirmed = True user.emailConfirmed = True
session.commit() session.commit()

View File

@ -1,8 +1,10 @@
from datetime import datetime, timezone from datetime import datetime, timezone
import jwt import jwt
from base.exceptions import ExpiredToken, InvalidToken from base.exceptions import ExpiredToken, InvalidToken
from validations.auth import TokenPayload, AuthInput
from settings import JWT_ALGORITHM, JWT_SECRET_KEY from settings import JWT_ALGORITHM, JWT_SECRET_KEY
from validations.auth import AuthInput, TokenPayload
class JWTCodec: class JWTCodec:
@ -13,7 +15,7 @@ class JWTCodec:
"username": user.email or user.phone, "username": user.email or user.phone,
"exp": exp, "exp": exp,
"iat": datetime.now(tz=timezone.utc), "iat": datetime.now(tz=timezone.utc),
"iss": "discours" "iss": "discours",
} }
try: try:
return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM) return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM)
@ -33,7 +35,7 @@ class JWTCodec:
# "verify_signature": False # "verify_signature": False
}, },
algorithms=[JWT_ALGORITHM], algorithms=[JWT_ALGORITHM],
issuer="discours" issuer="discours",
) )
r = TokenPayload(**payload) r = TokenPayload(**payload)
# print('[auth.jwtcodec] debug token %r' % r) # print('[auth.jwtcodec] debug token %r' % r)

View File

@ -1,8 +1,9 @@
from authlib.integrations.starlette_client import OAuth from authlib.integrations.starlette_client import OAuth
from starlette.responses import RedirectResponse from starlette.responses import RedirectResponse
from auth.identity import Identity from auth.identity import Identity
from auth.tokenstorage import TokenStorage from auth.tokenstorage import TokenStorage
from settings import OAUTH_CLIENTS, FRONTEND_URL from settings import FRONTEND_URL, OAUTH_CLIENTS
oauth = OAuth() oauth = OAuth()

View File

@ -1,9 +1,9 @@
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from auth.jwtcodec import JWTCodec from auth.jwtcodec import JWTCodec
from validations.auth import AuthInput
from base.redis import redis from base.redis import redis
from settings import SESSION_TOKEN_LIFE_SPAN, ONETIME_TOKEN_LIFE_SPAN from settings import ONETIME_TOKEN_LIFE_SPAN, SESSION_TOKEN_LIFE_SPAN
from validations.auth import AuthInput
async def save(token_key, life_span, auto_delete=True): async def save(token_key, life_span, auto_delete=True):

View File

@ -1,8 +1,8 @@
from graphql.error import GraphQLError from graphql.error import GraphQLError
# TODO: remove traceback from logs for defined exceptions # TODO: remove traceback from logs for defined exceptions
class BaseHttpException(GraphQLError): class BaseHttpException(GraphQLError):
code = 500 code = 500
message = "500 Server error" message = "500 Server error"

View File

@ -1,15 +1,13 @@
from typing import TypeVar, Any, Dict, Generic, Callable from typing import Any, Callable, Dict, Generic, TypeVar
from sqlalchemy import create_engine, Column, Integer from sqlalchemy import Column, Integer, create_engine
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from sqlalchemy.sql.schema import Table from sqlalchemy.sql.schema import Table
from settings import DB_URL from settings import DB_URL
engine = create_engine( engine = create_engine(DB_URL, echo=False, pool_size=10, max_overflow=20)
DB_URL, echo=False, pool_size=10, max_overflow=20
)
T = TypeVar("T") T = TypeVar("T")
@ -47,7 +45,7 @@ class Base(declarative_base()):
def update(self, input): def update(self, input):
column_names = self.__table__.columns.keys() column_names = self.__table__.columns.keys()
for (name, value) in input.items(): for name, value in input.items():
if name in column_names: if name in column_names:
setattr(self, name, value) setattr(self, name, value)

View File

@ -1,5 +1,7 @@
from aioredis import from_url
from asyncio import sleep from asyncio import sleep
from aioredis import from_url
from settings import REDIS_URL from settings import REDIS_URL

18
main.py
View File

@ -2,6 +2,7 @@ import asyncio
import os import os
from importlib import import_module from importlib import import_module
from os.path import exists from os.path import exists
from ariadne import load_schema_from_path, make_executable_schema from ariadne import load_schema_from_path, make_executable_schema
from ariadne.asgi import GraphQL from ariadne.asgi import GraphQL
from starlette.applications import Starlette from starlette.applications import Starlette
@ -9,20 +10,21 @@ from starlette.middleware import Middleware
from starlette.middleware.authentication import AuthenticationMiddleware from starlette.middleware.authentication import AuthenticationMiddleware
from starlette.middleware.sessions import SessionMiddleware from starlette.middleware.sessions import SessionMiddleware
from starlette.routing import Route from starlette.routing import Route
from orm import init_tables
from auth.authenticate import JWTAuthenticate from auth.authenticate import JWTAuthenticate
from auth.oauth import oauth_login, oauth_authorize from auth.oauth import oauth_authorize, oauth_login
from base.redis import redis from base.redis import redis
from base.resolvers import resolvers from base.resolvers import resolvers
from orm import init_tables
from resolvers.auth import confirm_email_handler from resolvers.auth import confirm_email_handler
from resolvers.upload import upload_handler from resolvers.upload import upload_handler
from services.main import storages_init from services.main import storages_init
from services.notifications.notification_service import notification_service from services.notifications.notification_service import notification_service
from services.notifications.sse import sse_subscribe_handler
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
# from services.zine.gittask import GitTask # from services.zine.gittask import GitTask
from settings import DEV_SERVER_PID_FILE_NAME, SENTRY_DSN, SESSION_SECRET_KEY from settings import DEV_SERVER_PID_FILE_NAME, SENTRY_DSN, SESSION_SECRET_KEY
from services.notifications.sse import sse_subscribe_handler
import_module("resolvers") import_module("resolvers")
schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) # type: ignore schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) # type: ignore
@ -46,6 +48,7 @@ async def start_up():
try: try:
import sentry_sdk import sentry_sdk
sentry_sdk.init(SENTRY_DSN) sentry_sdk.init(SENTRY_DSN)
except Exception as e: except Exception as e:
print('[sentry] init error') print('[sentry] init error')
@ -82,9 +85,7 @@ app = Starlette(
middleware=middleware, middleware=middleware,
routes=routes, routes=routes,
) )
app.mount("/", GraphQL( app.mount("/", GraphQL(schema))
schema
))
dev_app = Starlette( dev_app = Starlette(
debug=True, debug=True,
@ -93,7 +94,4 @@ dev_app = Starlette(
middleware=middleware, middleware=middleware,
routes=routes, routes=routes,
) )
dev_app.mount("/", GraphQL( dev_app.mount("/", GraphQL(schema, debug=True))
schema,
debug=True
))

View File

@ -16,4 +16,3 @@ echo "Start migration"
python3 server.py migrate python3 server.py migrate
if [ $? -ne 0 ]; then { echo "Migration failed, aborting." ; exit 1; } fi if [ $? -ne 0 ]; then { echo "Migration failed, aborting." ; exit 1; } fi
echo 'Done!' echo 'Done!'

View File

@ -14,8 +14,9 @@ from migration.tables.content_items import get_shout_slug
from migration.tables.content_items import migrate as migrateShout from migration.tables.content_items import migrate as migrateShout
from migration.tables.remarks import migrate as migrateRemark from migration.tables.remarks import migrate as migrateRemark
from migration.tables.topics import migrate as migrateTopic from migration.tables.topics import migrate as migrateTopic
from migration.tables.users import migrate as migrateUser, post_migrate as users_post_migrate from migration.tables.users import migrate as migrateUser
from migration.tables.users import migrate_2stage as migrateUser_2stage from migration.tables.users import migrate_2stage as migrateUser_2stage
from migration.tables.users import post_migrate as users_post_migrate
from orm import init_tables from orm import init_tables
from orm.reaction import Reaction from orm.reaction import Reaction
@ -63,16 +64,8 @@ async def topics_handle(storage):
del storage["topics"]["by_slug"][oldslug] del storage["topics"]["by_slug"][oldslug]
storage["topics"]["by_oid"][oid] = storage["topics"]["by_slug"][newslug] storage["topics"]["by_oid"][oid] = storage["topics"]["by_slug"][newslug]
print("[migration] " + str(counter) + " topics migrated") print("[migration] " + str(counter) + " topics migrated")
print( print("[migration] " + str(len(storage["topics"]["by_oid"].values())) + " topics by oid")
"[migration] " print("[migration] " + str(len(storage["topics"]["by_slug"].values())) + " topics by slug")
+ str(len(storage["topics"]["by_oid"].values()))
+ " topics by oid"
)
print(
"[migration] "
+ str(len(storage["topics"]["by_slug"].values()))
+ " topics by slug"
)
async def shouts_handle(storage, args): async def shouts_handle(storage, args):
@ -117,9 +110,10 @@ async def shouts_handle(storage, args):
# print main counter # print main counter
counter += 1 counter += 1
print('[migration] shouts_handle %d: %s @%s' % ( print(
(counter + 1), shout_dict["slug"], author["slug"] '[migration] shouts_handle %d: %s @%s'
)) % ((counter + 1), shout_dict["slug"], author["slug"])
)
b = bs4.BeautifulSoup(shout_dict["body"], "html.parser") b = bs4.BeautifulSoup(shout_dict["body"], "html.parser")
texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")]
@ -214,9 +208,7 @@ def data_load():
tags_data = json.loads(open("migration/data/tags.json").read()) tags_data = json.loads(open("migration/data/tags.json").read())
storage["topics"]["tags"] = tags_data storage["topics"]["tags"] = tags_data
print("[migration.load] " + str(len(tags_data)) + " tags ") print("[migration.load] " + str(len(tags_data)) + " tags ")
cats_data = json.loads( cats_data = json.loads(open("migration/data/content_item_categories.json").read())
open("migration/data/content_item_categories.json").read()
)
storage["topics"]["cats"] = cats_data storage["topics"]["cats"] = cats_data
print("[migration.load] " + str(len(cats_data)) + " cats ") print("[migration.load] " + str(len(cats_data)) + " cats ")
comments_data = json.loads(open("migration/data/comments.json").read()) comments_data = json.loads(open("migration/data/comments.json").read())
@ -235,11 +227,7 @@ def data_load():
storage["users"]["by_oid"][x["_id"]] = x storage["users"]["by_oid"][x["_id"]] = x
# storage['users']['by_slug'][x['slug']] = x # storage['users']['by_slug'][x['slug']] = x
# no user.slug yet # no user.slug yet
print( print("[migration.load] " + str(len(storage["users"]["by_oid"].keys())) + " users by oid")
"[migration.load] "
+ str(len(storage["users"]["by_oid"].keys()))
+ " users by oid"
)
for x in tags_data: for x in tags_data:
storage["topics"]["by_oid"][x["_id"]] = x storage["topics"]["by_oid"][x["_id"]] = x
storage["topics"]["by_slug"][x["slug"]] = x storage["topics"]["by_slug"][x["slug"]] = x
@ -247,9 +235,7 @@ def data_load():
storage["topics"]["by_oid"][x["_id"]] = x storage["topics"]["by_oid"][x["_id"]] = x
storage["topics"]["by_slug"][x["slug"]] = x storage["topics"]["by_slug"][x["slug"]] = x
print( print(
"[migration.load] " "[migration.load] " + str(len(storage["topics"]["by_slug"].keys())) + " topics by slug"
+ str(len(storage["topics"]["by_slug"].keys()))
+ " topics by slug"
) )
for item in content_data: for item in content_data:
slug = get_shout_slug(item) slug = get_shout_slug(item)

View File

@ -1,8 +1,9 @@
import gc
import json import json
import os import os
import bson import bson
import gc
from .utils import DateTimeEncoder from .utils import DateTimeEncoder
@ -15,7 +16,7 @@ def json_tables():
"email_subscriptions": [], "email_subscriptions": [],
"users": [], "users": [],
"comments": [], "comments": [],
"remarks": [] "remarks": [],
} }
for table in data.keys(): for table in data.keys():
print('[migration] bson2json for ' + table) print('[migration] bson2json for ' + table)

View File

@ -71,47 +71,29 @@ def export_slug(slug, storage):
def export_email_subscriptions(): def export_email_subscriptions():
email_subscriptions_data = json.loads( email_subscriptions_data = json.loads(open("migration/data/email_subscriptions.json").read())
open("migration/data/email_subscriptions.json").read()
)
for data in email_subscriptions_data: for data in email_subscriptions_data:
# TODO: migrate to mailgun list manually # TODO: migrate to mailgun list manually
# migrate_email_subscription(data) # migrate_email_subscription(data)
pass pass
print( print("[migration] " + str(len(email_subscriptions_data)) + " email subscriptions exported")
"[migration] "
+ str(len(email_subscriptions_data))
+ " email subscriptions exported"
)
def export_shouts(storage): def export_shouts(storage):
# update what was just migrated or load json again # update what was just migrated or load json again
if len(storage["users"]["by_slugs"].keys()) == 0: if len(storage["users"]["by_slugs"].keys()) == 0:
storage["users"]["by_slugs"] = json.loads( storage["users"]["by_slugs"] = json.loads(open(EXPORT_DEST + "authors.json").read())
open(EXPORT_DEST + "authors.json").read() print("[migration] " + str(len(storage["users"]["by_slugs"].keys())) + " exported authors ")
)
print(
"[migration] "
+ str(len(storage["users"]["by_slugs"].keys()))
+ " exported authors "
)
if len(storage["shouts"]["by_slugs"].keys()) == 0: if len(storage["shouts"]["by_slugs"].keys()) == 0:
storage["shouts"]["by_slugs"] = json.loads( storage["shouts"]["by_slugs"] = json.loads(open(EXPORT_DEST + "articles.json").read())
open(EXPORT_DEST + "articles.json").read()
)
print( print(
"[migration] " "[migration] " + str(len(storage["shouts"]["by_slugs"].keys())) + " exported articles "
+ str(len(storage["shouts"]["by_slugs"].keys()))
+ " exported articles "
) )
for slug in storage["shouts"]["by_slugs"].keys(): for slug in storage["shouts"]["by_slugs"].keys():
export_slug(slug, storage) export_slug(slug, storage)
def export_json( def export_json(export_articles={}, export_authors={}, export_topics={}, export_comments={}):
export_articles={}, export_authors={}, export_topics={}, export_comments={}
):
open(EXPORT_DEST + "authors.json", "w").write( open(EXPORT_DEST + "authors.json", "w").write(
json.dumps( json.dumps(
export_authors, export_authors,
@ -152,8 +134,4 @@ def export_json(
ensure_ascii=False, ensure_ascii=False,
) )
) )
print( print("[migration] " + str(len(export_comments.items())) + " exported articles with comments")
"[migration] "
+ str(len(export_comments.items()))
+ " exported articles with comments"
)

View File

@ -5,7 +5,6 @@ import uuid
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)" TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)"
contentDir = os.path.join( contentDir = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "..", "..", "discoursio-web", "content" os.path.dirname(os.path.realpath(__file__)), "..", "..", "discoursio-web", "content"
@ -27,7 +26,6 @@ def replace_tooltips(body):
return newbody return newbody
def extract_footnotes(body, shout_dict): def extract_footnotes(body, shout_dict):
parts = body.split("&&&") parts = body.split("&&&")
lll = len(parts) lll = len(parts)
@ -47,12 +45,16 @@ def extract_footnotes(body, shout_dict):
extracted_body = part.split(fn, 1)[1].split('>', 1)[1].split('</a>', 1)[0] extracted_body = part.split(fn, 1)[1].split('>', 1)[1].split('</a>', 1)[0]
print("[extract] footnote link: " + extracted_link) print("[extract] footnote link: " + extracted_link)
with local_session() as session: with local_session() as session:
Reaction.create({ Reaction.create(
{
"shout": shout_dict['id'], "shout": shout_dict['id'],
"kind": ReactionKind.FOOTNOTE, "kind": ReactionKind.FOOTNOTE,
"body": extracted_body, "body": extracted_body,
"range": str(body.index(fn + link) - len('<')) + ':' + str(body.index(extracted_body) + len('</a>')) "range": str(body.index(fn + link) - len('<'))
}) + ':'
+ str(body.index(extracted_body) + len('</a>')),
}
)
newparts[i] = "<a href='#'></a>" newparts[i] = "<a href='#'></a>"
else: else:
newparts[i] = part newparts[i] = part
@ -76,9 +78,7 @@ def place_tooltips(body):
print("[extract] footnote: " + part) print("[extract] footnote: " + part)
fn = 'a class="footnote-url" href="' fn = 'a class="footnote-url" href="'
link = part.split(fn, 1)[1].split('"', 1)[0] link = part.split(fn, 1)[1].split('"', 1)[0]
extracted_part = ( extracted_part = part.split(fn, 1)[0] + " " + part.split("/", 1)[-1]
part.split(fn, 1)[0] + " " + part.split("/", 1)[-1]
)
newparts[i] = ( newparts[i] = (
"<Tooltip" "<Tooltip"
+ (' link="' + link + '" ' if link else "") + (' link="' + link + '" ' if link else "")
@ -96,7 +96,9 @@ def place_tooltips(body):
return ("".join(newparts), placed) return ("".join(newparts), placed)
IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,((?:[A-Za-z\d+\/]{4})*(?:[A-Za-z\d+\/]{3}=" IMG_REGEX = (
r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,((?:[A-Za-z\d+\/]{4})*(?:[A-Za-z\d+\/]{3}="
)
IMG_REGEX += r"|[A-Za-z\d+\/]{2}==)))\)" IMG_REGEX += r"|[A-Za-z\d+\/]{2}==)))\)"
parentDir = "/".join(os.getcwd().split("/")[:-1]) parentDir = "/".join(os.getcwd().split("/")[:-1])
@ -159,11 +161,7 @@ def extract_imageparts(bodyparts, prefix):
try: try:
content = base64.b64decode(b64encoded + "==") content = base64.b64decode(b64encoded + "==")
open(public + link, "wb").write(content) open(public + link, "wb").write(content)
print( print("[extract] " + str(len(content)) + " image bytes been written")
"[extract] "
+ str(len(content))
+ " image bytes been written"
)
cache[b64encoded] = name cache[b64encoded] = name
except Exception: except Exception:
raise Exception raise Exception
@ -172,18 +170,11 @@ def extract_imageparts(bodyparts, prefix):
print("[extract] cached link " + cache[b64encoded]) print("[extract] cached link " + cache[b64encoded])
name = cache[b64encoded] name = cache[b64encoded]
link = cdn + "/upload/image-" + name + "." + ext link = cdn + "/upload/image-" + name + "." + ext
newparts[i] = ( newparts[i] = current[: -len(mime)] + current[-len(mime) :] + link + next[-b64end:]
current[: -len(mime)]
+ current[-len(mime) :]
+ link
+ next[-b64end:]
)
newparts[i + 1] = next[:-b64end] newparts[i + 1] = next[:-b64end]
break break
return ( return (
extract_imageparts( extract_imageparts(newparts[i] + newparts[i + 1] + b64.join(bodyparts[(i + 2) :]), prefix)
newparts[i] + newparts[i + 1] + b64.join(bodyparts[(i + 2) :]), prefix
)
if len(bodyparts) > (i + 1) if len(bodyparts) > (i + 1)
else "".join(newparts) else "".join(newparts)
) )
@ -279,7 +270,6 @@ def extract_md(body, shout_dict = None):
raise Exception("cleanup error") raise Exception("cleanup error")
if shout_dict: if shout_dict:
uid = shout_dict['id'] or uuid.uuid4() uid = shout_dict['id'] or uuid.uuid4()
newbody = extract_md_images(newbody, uid) newbody = extract_md_images(newbody, uid)
if not newbody: if not newbody:
@ -323,12 +313,7 @@ def extract_media(entry):
url = "https://vimeo.com/" + m["vimeoId"] url = "https://vimeo.com/" + m["vimeoId"]
# body # body
body = m.get("body") or m.get("literatureBody") or "" body = m.get("body") or m.get("literatureBody") or ""
media.append({ media.append({"url": url, "pic": pic, "title": title, "body": body})
"url": url,
"pic": pic,
"title": title,
"body": body
})
return media return media
@ -398,9 +383,7 @@ def cleanup_html(body: str) -> str:
r"<h4>\s*</h4>", r"<h4>\s*</h4>",
r"<div>\s*</div>", r"<div>\s*</div>",
] ]
regex_replace = { regex_replace = {r"<br>\s*</p>": "</p>"}
r"<br>\s*</p>": "</p>"
}
changed = True changed = True
while changed: while changed:
# we need several iterations to clean nested tags this way # we need several iterations to clean nested tags this way
@ -414,6 +397,7 @@ def cleanup_html(body: str) -> str:
changed = True changed = True
return new_body return new_body
def extract_html(entry, shout_id=None, cleanup=False): def extract_html(entry, shout_id=None, cleanup=False):
body_orig = (entry.get("body") or "").replace('\(', '(').replace('\)', ')') body_orig = (entry.get("body") or "").replace('\(', '(').replace('\)', ')')
if cleanup: if cleanup:

View File

@ -119,9 +119,7 @@ class HTML2Text(html.parser.HTMLParser):
self.lastWasList = False self.lastWasList = False
self.style = 0 self.style = 0
self.style_def = {} # type: Dict[str, Dict[str, str]] self.style_def = {} # type: Dict[str, Dict[str, str]]
self.tag_stack = ( self.tag_stack = [] # type: List[Tuple[str, Dict[str, Optional[str]], Dict[str, str]]]
[]
) # type: List[Tuple[str, Dict[str, Optional[str]], Dict[str, str]]]
self.emphasis = 0 self.emphasis = 0
self.drop_white_space = 0 self.drop_white_space = 0
self.inheader = False self.inheader = False
@ -300,9 +298,7 @@ class HTML2Text(html.parser.HTMLParser):
if strikethrough: if strikethrough:
self.quiet -= 1 self.quiet -= 1
def handle_tag( def handle_tag(self, tag: str, attrs: Dict[str, Optional[str]], start: bool) -> None:
self, tag: str, attrs: Dict[str, Optional[str]], start: bool
) -> None:
self.current_tag = tag self.current_tag = tag
if self.tag_callback is not None: if self.tag_callback is not None:
@ -333,9 +329,7 @@ class HTML2Text(html.parser.HTMLParser):
tag_style = element_style(attrs, self.style_def, parent_style) tag_style = element_style(attrs, self.style_def, parent_style)
self.tag_stack.append((tag, attrs, tag_style)) self.tag_stack.append((tag, attrs, tag_style))
else: else:
dummy, attrs, tag_style = ( dummy, attrs, tag_style = self.tag_stack.pop() if self.tag_stack else (None, {}, {})
self.tag_stack.pop() if self.tag_stack else (None, {}, {})
)
if self.tag_stack: if self.tag_stack:
parent_style = self.tag_stack[-1][2] parent_style = self.tag_stack[-1][2]
@ -385,11 +379,7 @@ class HTML2Text(html.parser.HTMLParser):
): ):
self.o("`") # NOTE: same as <code> self.o("`") # NOTE: same as <code>
self.span_highlight = True self.span_highlight = True
elif ( elif self.current_class == "lead" and not self.inheader and not self.span_highlight:
self.current_class == "lead"
and not self.inheader
and not self.span_highlight
):
# self.o("==") # NOTE: CriticMarkup {== # self.o("==") # NOTE: CriticMarkup {==
self.span_lead = True self.span_lead = True
else: else:
@ -479,11 +469,7 @@ class HTML2Text(html.parser.HTMLParser):
and not self.span_lead and not self.span_lead
and not self.span_highlight and not self.span_highlight
): ):
if ( if start and self.preceding_data and self.preceding_data[-1] == self.strong_mark[0]:
start
and self.preceding_data
and self.preceding_data[-1] == self.strong_mark[0]
):
strong = " " + self.strong_mark strong = " " + self.strong_mark
self.preceding_data += " " self.preceding_data += " "
else: else:
@ -548,13 +534,8 @@ class HTML2Text(html.parser.HTMLParser):
"href" in attrs "href" in attrs
and not attrs["href"].startswith("#_ftn") and not attrs["href"].startswith("#_ftn")
and attrs["href"] is not None and attrs["href"] is not None
and not ( and not (self.skip_internal_links and attrs["href"].startswith("#"))
self.skip_internal_links and attrs["href"].startswith("#") and not (self.ignore_mailto_links and attrs["href"].startswith("mailto:"))
)
and not (
self.ignore_mailto_links
and attrs["href"].startswith("mailto:")
)
): ):
self.astack.append(attrs) self.astack.append(attrs)
self.maybe_automatic_link = attrs["href"] self.maybe_automatic_link = attrs["href"]
@ -638,9 +619,7 @@ class HTML2Text(html.parser.HTMLParser):
self.o("![" + escape_md(alt) + "]") self.o("![" + escape_md(alt) + "]")
if self.inline_links: if self.inline_links:
href = attrs.get("href") or "" href = attrs.get("href") or ""
self.o( self.o("(" + escape_md(urlparse.urljoin(self.baseurl, href)) + ")")
"(" + escape_md(urlparse.urljoin(self.baseurl, href)) + ")"
)
else: else:
i = self.previousIndex(attrs) i = self.previousIndex(attrs)
if i is not None: if i is not None:
@ -696,9 +675,7 @@ class HTML2Text(html.parser.HTMLParser):
# WARNING: does not line up <ol><li>s > 9 correctly. # WARNING: does not line up <ol><li>s > 9 correctly.
parent_list = None parent_list = None
for list in self.list: for list in self.list:
self.o( self.o(" " if parent_list == "ol" and list.name == "ul" else " ")
" " if parent_list == "ol" and list.name == "ul" else " "
)
parent_list = list.name parent_list = list.name
if li.name == "ul": if li.name == "ul":
@ -787,9 +764,7 @@ class HTML2Text(html.parser.HTMLParser):
self.pbr() self.pbr()
self.br_toggle = " " self.br_toggle = " "
def o( def o(self, data: str, puredata: bool = False, force: Union[bool, str] = False) -> None:
self, data: str, puredata: bool = False, force: Union[bool, str] = False
) -> None:
""" """
Deal with indentation and whitespace Deal with indentation and whitespace
""" """
@ -864,9 +839,7 @@ class HTML2Text(html.parser.HTMLParser):
self.out(" ") self.out(" ")
self.space = False self.space = False
if self.a and ( if self.a and ((self.p_p == 2 and self.links_each_paragraph) or force == "end"):
(self.p_p == 2 and self.links_each_paragraph) or force == "end"
):
if force == "end": if force == "end":
self.out("\n") self.out("\n")
@ -925,11 +898,7 @@ class HTML2Text(html.parser.HTMLParser):
if self.maybe_automatic_link is not None: if self.maybe_automatic_link is not None:
href = self.maybe_automatic_link href = self.maybe_automatic_link
if ( if href == data and self.absolute_url_matcher.match(href) and self.use_automatic_links:
href == data
and self.absolute_url_matcher.match(href)
and self.use_automatic_links
):
self.o("<" + data + ">") self.o("<" + data + ">")
self.empty_link = False self.empty_link = False
return return
@ -1000,9 +969,7 @@ class HTML2Text(html.parser.HTMLParser):
self.inline_links = False self.inline_links = False
for para in text.split("\n"): for para in text.split("\n"):
if len(para) > 0: if len(para) > 0:
if not skipwrap( if not skipwrap(para, self.wrap_links, self.wrap_list_items, self.wrap_tables):
para, self.wrap_links, self.wrap_list_items, self.wrap_tables
):
indent = "" indent = ""
if para.startswith(" " + self.ul_item_mark): if para.startswith(" " + self.ul_item_mark):
# list item continuation: add a double indent to the # list item continuation: add a double indent to the
@ -1043,9 +1010,7 @@ class HTML2Text(html.parser.HTMLParser):
return result return result
def html2text( def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = config.BODY_WIDTH) -> str:
html: str, baseurl: str = "", bodywidth: Optional[int] = config.BODY_WIDTH
) -> str:
h = html.strip() or "" h = html.strip() or ""
if h: if h:
h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth) h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)

View File

@ -117,10 +117,7 @@ def main() -> None:
dest="images_with_size", dest="images_with_size",
action="store_true", action="store_true",
default=config.IMAGES_WITH_SIZE, default=config.IMAGES_WITH_SIZE,
help=( help=("Write image tags with height and width attrs as raw html to retain " "dimensions"),
"Write image tags with height and width attrs as raw html to retain "
"dimensions"
),
) )
p.add_argument( p.add_argument(
"-g", "-g",
@ -260,9 +257,7 @@ def main() -> None:
default=config.CLOSE_QUOTE, default=config.CLOSE_QUOTE,
help="The character used to close quotes", help="The character used to close quotes",
) )
p.add_argument( p.add_argument("--version", action="version", version=".".join(map(str, __version__)))
"--version", action="version", version=".".join(map(str, __version__))
)
p.add_argument("filename", nargs="?") p.add_argument("filename", nargs="?")
p.add_argument("encoding", nargs="?", default="utf-8") p.add_argument("encoding", nargs="?", default="utf-8")
args = p.parse_args() args = p.parse_args()

View File

@ -4,9 +4,7 @@ from typing import Dict, List, Optional
from . import config from . import config
unifiable_n = { unifiable_n = {
html.entities.name2codepoint[k]: v html.entities.name2codepoint[k]: v for k, v in config.UNIFIABLE.items() if k != "nbsp"
for k, v in config.UNIFIABLE.items()
if k != "nbsp"
} }
@ -156,9 +154,7 @@ def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
return 0 return 0
def skipwrap( def skipwrap(para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool) -> bool:
para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool
) -> bool:
# If it appears to contain a link # If it appears to contain a link
# don't wrap # don't wrap
if not wrap_links and config.RE_LINK.search(para): if not wrap_links and config.RE_LINK.search(para):
@ -236,9 +232,7 @@ def reformat_table(lines: List[str], right_margin: int) -> List[str]:
max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]] max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
max_cols = num_cols max_cols = num_cols
max_width = [ max_width = [max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)]
max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
]
# reformat # reformat
new_lines = [] new_lines = []
@ -247,15 +241,13 @@ def reformat_table(lines: List[str], right_margin: int) -> List[str]:
if set(line.strip()) == set("-|"): if set(line.strip()) == set("-|"):
filler = "-" filler = "-"
new_cols = [ new_cols = [
x.rstrip() + (filler * (M - len(x.rstrip()))) x.rstrip() + (filler * (M - len(x.rstrip()))) for x, M in zip(cols, max_width)
for x, M in zip(cols, max_width)
] ]
new_lines.append("|-" + "|".join(new_cols) + "|") new_lines.append("|-" + "|".join(new_cols) + "|")
else: else:
filler = " " filler = " "
new_cols = [ new_cols = [
x.rstrip() + (filler * (M - len(x.rstrip()))) x.rstrip() + (filler * (M - len(x.rstrip()))) for x, M in zip(cols, max_width)
for x, M in zip(cols, max_width)
] ]
new_lines.append("| " + "|".join(new_cols) + "|") new_lines.append("| " + "|".join(new_cols) + "|")
return new_lines return new_lines

View File

@ -5,61 +5,48 @@ from dateutil.parser import parse as date_parse
from base.orm import local_session from base.orm import local_session
from migration.html2text import html2text from migration.html2text import html2text
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import ShoutReactionsFollower from orm.shout import Shout, ShoutReactionsFollower
from orm.topic import TopicFollower from orm.topic import TopicFollower
from orm.user import User from orm.user import User
from orm.shout import Shout
ts = datetime.now(tz=timezone.utc) ts = datetime.now(tz=timezone.utc)
def auto_followers(session, topics, reaction_dict): def auto_followers(session, topics, reaction_dict):
# creating shout's reactions following for reaction author # creating shout's reactions following for reaction author
following1 = session.query( following1 = (
ShoutReactionsFollower session.query(ShoutReactionsFollower)
).where( .where(ShoutReactionsFollower.follower == reaction_dict["createdBy"])
ShoutReactionsFollower.follower == reaction_dict["createdBy"] .filter(ShoutReactionsFollower.shout == reaction_dict["shout"])
).filter( .first()
ShoutReactionsFollower.shout == reaction_dict["shout"] )
).first()
if not following1: if not following1:
following1 = ShoutReactionsFollower.create( following1 = ShoutReactionsFollower.create(
follower=reaction_dict["createdBy"], follower=reaction_dict["createdBy"], shout=reaction_dict["shout"], auto=True
shout=reaction_dict["shout"],
auto=True
) )
session.add(following1) session.add(following1)
# creating topics followings for reaction author # creating topics followings for reaction author
for t in topics: for t in topics:
tf = session.query( tf = (
TopicFollower session.query(TopicFollower)
).where( .where(TopicFollower.follower == reaction_dict["createdBy"])
TopicFollower.follower == reaction_dict["createdBy"] .filter(TopicFollower.topic == t['id'])
).filter( .first()
TopicFollower.topic == t['id'] )
).first()
if not tf: if not tf:
topic_following = TopicFollower.create( topic_following = TopicFollower.create(
follower=reaction_dict["createdBy"], follower=reaction_dict["createdBy"], topic=t['id'], auto=True
topic=t['id'],
auto=True
) )
session.add(topic_following) session.add(topic_following)
def migrate_ratings(session, entry, reaction_dict): def migrate_ratings(session, entry, reaction_dict):
for comment_rating_old in entry.get("ratings", []): for comment_rating_old in entry.get("ratings", []):
rater = ( rater = session.query(User).filter(User.oid == comment_rating_old["createdBy"]).first()
session.query(User)
.filter(User.oid == comment_rating_old["createdBy"])
.first()
)
re_reaction_dict = { re_reaction_dict = {
"shout": reaction_dict["shout"], "shout": reaction_dict["shout"],
"replyTo": reaction_dict["id"], "replyTo": reaction_dict["id"],
"kind": ReactionKind.LIKE "kind": ReactionKind.LIKE if comment_rating_old["value"] > 0 else ReactionKind.DISLIKE,
if comment_rating_old["value"] > 0
else ReactionKind.DISLIKE,
"createdBy": rater.id if rater else 1, "createdBy": rater.id if rater else 1,
} }
cts = comment_rating_old.get("createdAt") cts = comment_rating_old.get("createdAt")
@ -68,18 +55,15 @@ def migrate_ratings(session, entry, reaction_dict):
try: try:
# creating reaction from old rating # creating reaction from old rating
rr = Reaction.create(**re_reaction_dict) rr = Reaction.create(**re_reaction_dict)
following2 = session.query( following2 = (
ShoutReactionsFollower session.query(ShoutReactionsFollower)
).where( .where(ShoutReactionsFollower.follower == re_reaction_dict['createdBy'])
ShoutReactionsFollower.follower == re_reaction_dict['createdBy'] .filter(ShoutReactionsFollower.shout == rr.shout)
).filter( .first()
ShoutReactionsFollower.shout == rr.shout )
).first()
if not following2: if not following2:
following2 = ShoutReactionsFollower.create( following2 = ShoutReactionsFollower.create(
follower=re_reaction_dict['createdBy'], follower=re_reaction_dict['createdBy'], shout=rr.shout, auto=True
shout=rr.shout,
auto=True
) )
session.add(following2) session.add(following2)
session.add(rr) session.add(rr)
@ -150,9 +134,7 @@ async def migrate(entry, storage):
else: else:
stage = "author and old id found" stage = "author and old id found"
try: try:
shout = session.query( shout = session.query(Shout).where(Shout.slug == old_shout["slug"]).one()
Shout
).where(Shout.slug == old_shout["slug"]).one()
if shout: if shout:
reaction_dict["shout"] = shout.id reaction_dict["shout"] = shout.id
reaction_dict["createdBy"] = author.id if author else 1 reaction_dict["createdBy"] = author.id if author else 1
@ -190,17 +172,20 @@ def migrate_2stage(old_comment, idmap):
comment = session.query(Reaction).where(Reaction.id == new_id).first() comment = session.query(Reaction).where(Reaction.id == new_id).first()
try: try:
if new_replyto_id: if new_replyto_id:
new_reply = session.query(Reaction).where(Reaction.id == new_replyto_id).first() new_reply = (
session.query(Reaction).where(Reaction.id == new_replyto_id).first()
)
if not new_reply: if not new_reply:
print(new_replyto_id) print(new_replyto_id)
raise Exception("cannot find reply by id!") raise Exception("cannot find reply by id!")
comment.replyTo = new_reply.id comment.replyTo = new_reply.id
session.add(comment) session.add(comment)
srf = session.query(ShoutReactionsFollower).where( srf = (
ShoutReactionsFollower.shout == comment.shout session.query(ShoutReactionsFollower)
).filter( .where(ShoutReactionsFollower.shout == comment.shout)
ShoutReactionsFollower.follower == comment.createdBy .filter(ShoutReactionsFollower.follower == comment.createdBy)
).first() .first()
)
if not srf: if not srf:
srf = ShoutReactionsFollower.create( srf = ShoutReactionsFollower.create(
shout=comment.shout, follower=comment.createdBy, auto=True shout=comment.shout, follower=comment.createdBy, auto=True

View File

@ -1,16 +1,18 @@
from datetime import datetime, timezone
import json import json
import re
from datetime import datetime, timezone
from dateutil.parser import parse as date_parse from dateutil.parser import parse as date_parse
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from transliterate import translit from transliterate import translit
from base.orm import local_session from base.orm import local_session
from migration.extract import extract_html, extract_media from migration.extract import extract_html, extract_media
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.shout import Shout, ShoutReactionsFollower, ShoutTopic
from orm.topic import Topic, TopicFollower
from orm.user import User from orm.user import User
from orm.topic import TopicFollower, Topic
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
import re
OLD_DATE = "2016-03-05 22:22:00.350000" OLD_DATE = "2016-03-05 22:22:00.350000"
ts = datetime.now(tz=timezone.utc) ts = datetime.now(tz=timezone.utc)
@ -91,11 +93,12 @@ async def create_shout(shout_dict):
s = Shout.create(**shout_dict) s = Shout.create(**shout_dict)
author = s.authors[0] author = s.authors[0]
with local_session() as session: with local_session() as session:
srf = session.query(ShoutReactionsFollower).where( srf = (
ShoutReactionsFollower.shout == s.id session.query(ShoutReactionsFollower)
).filter( .where(ShoutReactionsFollower.shout == s.id)
ShoutReactionsFollower.follower == author.id .filter(ShoutReactionsFollower.follower == author.id)
).first() .first()
)
if not srf: if not srf:
srf = ShoutReactionsFollower.create(shout=s.id, follower=author.id, auto=True) srf = ShoutReactionsFollower.create(shout=s.id, follower=author.id, auto=True)
session.add(srf) session.add(srf)
@ -137,11 +140,14 @@ async def migrate(entry, storage):
r = { r = {
"layout": type2layout[entry["type"]], "layout": type2layout[entry["type"]],
"title": entry["title"], "title": entry["title"],
"authors": [author, ], "authors": [
author,
],
"slug": get_shout_slug(entry), "slug": get_shout_slug(entry),
"cover": ( "cover": (
"https://images.discours.io/unsafe/" + "https://images.discours.io/unsafe/" + entry["thumborId"]
entry["thumborId"] if entry.get("thumborId") else entry.get("image", {}).get("url") if entry.get("thumborId")
else entry.get("image", {}).get("url")
), ),
"visibility": "public" if entry.get("published") else "community", "visibility": "public" if entry.get("published") else "community",
"publishedAt": date_parse(entry.get("publishedAt")) if entry.get("published") else None, "publishedAt": date_parse(entry.get("publishedAt")) if entry.get("published") else None,
@ -150,7 +156,7 @@ async def migrate(entry, storage):
"updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts, "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts,
"createdBy": author.id, "createdBy": author.id,
"topics": await add_topics_follower(entry, storage, author), "topics": await add_topics_follower(entry, storage, author),
"body": extract_html(entry, cleanup=True) "body": extract_html(entry, cleanup=True),
} }
# main topic patch # main topic patch
@ -184,7 +190,9 @@ async def migrate(entry, storage):
# udpate data # udpate data
shout_dict = shout.dict() shout_dict = shout.dict()
shout_dict["authors"] = [author.dict(), ] shout_dict["authors"] = [
author.dict(),
]
# shout topics aftermath # shout topics aftermath
shout_dict["topics"] = await topics_aftermath(r, storage) shout_dict["topics"] = await topics_aftermath(r, storage)
@ -193,7 +201,9 @@ async def migrate(entry, storage):
await content_ratings_to_reactions(entry, shout_dict["slug"]) await content_ratings_to_reactions(entry, shout_dict["slug"])
# shout views # shout views
await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1), viewer='old-discours') await ViewedStorage.increment(
shout_dict["slug"], amount=entry.get("views", 1), viewer='old-discours'
)
# del shout_dict['ratings'] # del shout_dict['ratings']
storage["shouts"]["by_oid"][entry["_id"]] = shout_dict storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
@ -205,7 +215,9 @@ async def add_topics_follower(entry, storage, user):
topics = set([]) topics = set([])
category = entry.get("category") category = entry.get("category")
topics_by_oid = storage["topics"]["by_oid"] topics_by_oid = storage["topics"]["by_oid"]
oids = [category, ] + entry.get("tags", []) oids = [
category,
] + entry.get("tags", [])
for toid in oids: for toid in oids:
tslug = topics_by_oid.get(toid, {}).get("slug") tslug = topics_by_oid.get(toid, {}).get("slug")
if tslug: if tslug:
@ -217,19 +229,14 @@ async def add_topics_follower(entry, storage, user):
try: try:
tpc = session.query(Topic).where(Topic.slug == tpcslug).first() tpc = session.query(Topic).where(Topic.slug == tpcslug).first()
if tpc: if tpc:
tf = session.query( tf = (
TopicFollower session.query(TopicFollower)
).where( .where(TopicFollower.follower == user.id)
TopicFollower.follower == user.id .filter(TopicFollower.topic == tpc.id)
).filter( .first()
TopicFollower.topic == tpc.id
).first()
if not tf:
tf = TopicFollower.create(
topic=tpc.id,
follower=user.id,
auto=True
) )
if not tf:
tf = TopicFollower.create(topic=tpc.id, follower=user.id, auto=True)
session.add(tf) session.add(tf)
session.commit() session.commit()
except IntegrityError: except IntegrityError:
@ -295,10 +302,7 @@ async def resolve_create_shout(shout_dict):
for key in shout_dict: for key in shout_dict:
if key in s.__dict__: if key in s.__dict__:
if s.__dict__[key] != shout_dict[key]: if s.__dict__[key] != shout_dict[key]:
print( print("[migration] shout already exists, but differs in %s" % key)
"[migration] shout already exists, but differs in %s"
% key
)
bump = True bump = True
else: else:
print("[migration] shout already exists, but lacks %s" % key) print("[migration] shout already exists, but lacks %s" % key)
@ -344,9 +348,7 @@ async def topics_aftermath(entry, storage):
) )
if not shout_topic_new: if not shout_topic_new:
try: try:
ShoutTopic.create( ShoutTopic.create(**{"shout": shout.id, "topic": new_topic.id})
**{"shout": shout.id, "topic": new_topic.id}
)
except Exception: except Exception:
print("[migration] shout topic error: " + newslug) print("[migration] shout topic error: " + newslug)
session.commit() session.commit()
@ -363,9 +365,7 @@ async def content_ratings_to_reactions(entry, slug):
with local_session() as session: with local_session() as session:
for content_rating in entry.get("ratings", []): for content_rating in entry.get("ratings", []):
rater = ( rater = (
session.query(User) session.query(User).filter(User.oid == content_rating["createdBy"]).first()
.filter(User.oid == content_rating["createdBy"])
.first()
) or User.default_user ) or User.default_user
shout = session.query(Shout).where(Shout.slug == slug).first() shout = session.query(Shout).where(Shout.slug == slug).first()
cts = content_rating.get("createdAt") cts = content_rating.get("createdAt")
@ -375,7 +375,7 @@ async def content_ratings_to_reactions(entry, slug):
if content_rating["value"] > 0 if content_rating["value"] > 0
else ReactionKind.DISLIKE, else ReactionKind.DISLIKE,
"createdBy": rater.id, "createdBy": rater.id,
"shout": shout.id "shout": shout.id,
} }
reaction = ( reaction = (
session.query(Reaction) session.query(Reaction)

View File

@ -12,25 +12,17 @@ def migrate(entry, storage):
print(shout_dict['body']) print(shout_dict['body'])
remark = { remark = {
"shout": shout_dict['id'], "shout": shout_dict['id'],
"body": extract_md( "body": extract_md(html2text(entry['body']), shout_dict),
html2text(entry['body']), "kind": ReactionKind.REMARK,
shout_dict
),
"kind": ReactionKind.REMARK
} }
if entry.get('textBefore'): if entry.get('textBefore'):
remark['range'] = str( remark['range'] = (
shout_dict['body'] str(shout_dict['body'].index(entry['textBefore'] or ''))
.index( + ':'
entry['textBefore'] or '' + str(
) shout_dict['body'].index(entry['textAfter'] or '')
) + ':' + str( + len(entry['textAfter'] or '')
shout_dict['body']
.index(
entry['textAfter'] or ''
) + len(
entry['textAfter'] or ''
) )
) )

View File

@ -10,7 +10,7 @@ def migrate(entry):
"slug": entry["slug"], "slug": entry["slug"],
"oid": entry["_id"], "oid": entry["_id"],
"title": entry["title"].replace("&nbsp;", " "), "title": entry["title"].replace("&nbsp;", " "),
"body": extract_md(html2text(body_orig)) "body": extract_md(html2text(body_orig)),
} }
with local_session() as session: with local_session() as session:

View File

@ -23,7 +23,7 @@ def migrate(entry):
"muted": False, # amnesty "muted": False, # amnesty
"links": [], "links": [],
"name": "anonymous", "name": "anonymous",
"password": entry["services"]["password"].get("bcrypt") "password": entry["services"]["password"].get("bcrypt"),
} }
if "updatedAt" in entry: if "updatedAt" in entry:
@ -35,7 +35,11 @@ def migrate(entry):
slug = entry["profile"].get("path").lower() slug = entry["profile"].get("path").lower()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip() slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip()
user_dict["slug"] = slug user_dict["slug"] = slug
bio = (entry.get("profile", {"bio": ""}).get("bio") or "").replace('\(', '(').replace('\)', ')') bio = (
(entry.get("profile", {"bio": ""}).get("bio") or "")
.replace('\(', '(')
.replace('\)', ')')
)
bio_text = BeautifulSoup(bio, features="lxml").text bio_text = BeautifulSoup(bio, features="lxml").text
if len(bio_text) > 120: if len(bio_text) > 120:
@ -46,8 +50,7 @@ def migrate(entry):
# userpic # userpic
try: try:
user_dict["userpic"] = ( user_dict["userpic"] = (
"https://images.discours.io/unsafe/" "https://images.discours.io/unsafe/" + entry["profile"]["thumborId"]
+ entry["profile"]["thumborId"]
) )
except KeyError: except KeyError:
try: try:
@ -62,11 +65,7 @@ def migrate(entry):
name = (name + " " + ln) if ln else name name = (name + " " + ln) if ln else name
if not name: if not name:
name = slug if slug else "anonymous" name = slug if slug else "anonymous"
name = ( name = entry["profile"]["path"].lower().strip().replace(" ", "-") if len(name) < 2 else name
entry["profile"]["path"].lower().strip().replace(" ", "-")
if len(name) < 2
else name
)
user_dict["name"] = name user_dict["name"] = name
# links # links
@ -95,9 +94,7 @@ def migrate(entry):
except IntegrityError: except IntegrityError:
print("[migration] cannot create user " + user_dict["slug"]) print("[migration] cannot create user " + user_dict["slug"])
with local_session() as session: with local_session() as session:
old_user = ( old_user = session.query(User).filter(User.slug == user_dict["slug"]).first()
session.query(User).filter(User.slug == user_dict["slug"]).first()
)
old_user.oid = oid old_user.oid = oid
old_user.password = user_dict["password"] old_user.password = user_dict["password"]
session.commit() session.commit()
@ -114,7 +111,7 @@ def post_migrate():
"slug": "old-discours", "slug": "old-discours",
"username": "old-discours", "username": "old-discours",
"email": "old@discours.io", "email": "old@discours.io",
"name": "Просмотры на старой версии сайта" "name": "Просмотры на старой версии сайта",
} }
with local_session() as session: with local_session() as session:
@ -148,11 +145,7 @@ def migrate_2stage(entry, id_map):
user_rating = UserRating.create(**user_rating_dict) user_rating = UserRating.create(**user_rating_dict)
if user_rating_dict['value'] > 0: if user_rating_dict['value'] > 0:
af = AuthorFollower.create( af = AuthorFollower.create(author=user.id, follower=rater.id, auto=True)
author=user.id,
follower=rater.id,
auto=True
)
session.add(af) session.add(af)
session.add(user_rating) session.add(user_rating)
session.commit() session.commit()

View File

@ -1,7 +1,7 @@
from base.orm import Base, engine from base.orm import Base, engine
from orm.community import Community from orm.community import Community
from orm.notification import Notification from orm.notification import Notification
from orm.rbac import Operation, Resource, Permission, Role from orm.rbac import Operation, Permission, Resource, Role
from orm.reaction import Reaction from orm.reaction import Reaction
from orm.shout import Shout from orm.shout import Shout
from orm.topic import Topic, TopicFollower from orm.topic import Topic, TopicFollower
@ -32,5 +32,5 @@ __all__ = [
"Notification", "Notification",
"Reaction", "Reaction",
"UserRating", "UserRating",
"init_tables" "init_tables",
] ]

View File

@ -1,6 +1,7 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime from sqlalchemy import Column, DateTime, ForeignKey, String
from base.orm import Base, local_session from base.orm import Base, local_session
@ -10,9 +11,7 @@ class CommunityFollower(Base):
id = None # type: ignore id = None # type: ignore
follower = Column(ForeignKey("user.id"), primary_key=True) follower = Column(ForeignKey("user.id"), primary_key=True)
community = Column(ForeignKey("community.id"), primary_key=True) community = Column(ForeignKey("community.id"), primary_key=True)
joinedAt = Column( joinedAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
DateTime, nullable=False, default=datetime.now, comment="Created at"
)
# role = Column(ForeignKey(Role.id), nullable=False, comment="Role for member") # role = Column(ForeignKey(Role.id), nullable=False, comment="Role for member")
@ -23,16 +22,12 @@ class Community(Base):
slug = Column(String, nullable=False, unique=True, comment="Slug") slug = Column(String, nullable=False, unique=True, comment="Slug")
desc = Column(String, nullable=False, default="") desc = Column(String, nullable=False, default="")
pic = Column(String, nullable=False, default="") pic = Column(String, nullable=False, default="")
createdAt = Column( createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
DateTime, nullable=False, default=datetime.now, comment="Created at"
)
@staticmethod @staticmethod
def init_table(): def init_table():
with local_session() as session: with local_session() as session:
d = ( d = session.query(Community).filter(Community.slug == "discours").first()
session.query(Community).filter(Community.slug == "discours").first()
)
if not d: if not d:
d = Community.create(name="Дискурс", slug="discours") d = Community.create(name="Дискурс", slug="discours")
session.add(d) session.add(d)

View File

@ -1,9 +1,10 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Column, Enum, ForeignKey, DateTime, Boolean, Integer from enum import Enum as Enumeration
from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer
from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.dialects.postgresql import JSONB
from base.orm import Base from base.orm import Base
from enum import Enum as Enumeration
class NotificationType(Enumeration): class NotificationType(Enumeration):

View File

@ -1,9 +1,9 @@
import warnings import warnings
from sqlalchemy import String, Column, ForeignKey, UniqueConstraint, TypeDecorator from sqlalchemy import Column, ForeignKey, String, TypeDecorator, UniqueConstraint
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from base.orm import Base, REGISTRY, engine, local_session from base.orm import REGISTRY, Base, engine, local_session
# Role Based Access Control # # Role Based Access Control #
@ -121,16 +121,23 @@ class Operation(Base):
class Resource(Base): class Resource(Base):
__tablename__ = "resource" __tablename__ = "resource"
resourceClass = Column( resourceClass = Column(String, nullable=False, unique=True, comment="Resource class")
String, nullable=False, unique=True, comment="Resource class"
)
name = Column(String, nullable=False, unique=True, comment="Resource name") name = Column(String, nullable=False, unique=True, comment="Resource name")
# TODO: community = Column(ForeignKey()) # TODO: community = Column(ForeignKey())
@staticmethod @staticmethod
def init_table(): def init_table():
with local_session() as session: with local_session() as session:
for res in ["shout", "topic", "reaction", "chat", "message", "invite", "community", "user"]: for res in [
"shout",
"topic",
"reaction",
"chat",
"message",
"invite",
"community",
"user",
]:
r = session.query(Resource).filter(Resource.name == res).first() r = session.query(Resource).filter(Resource.name == res).first()
if not r: if not r:
r = Resource.create(name=res, resourceClass=res) r = Resource.create(name=res, resourceClass=res)
@ -145,9 +152,7 @@ class Permission(Base):
{"extend_existing": True}, {"extend_existing": True},
) )
role = Column( role = Column(ForeignKey("role.id", ondelete="CASCADE"), nullable=False, comment="Role")
ForeignKey("role.id", ondelete="CASCADE"), nullable=False, comment="Role"
)
operation = Column( operation = Column(
ForeignKey("operation.id", ondelete="CASCADE"), ForeignKey("operation.id", ondelete="CASCADE"),
nullable=False, nullable=False,

View File

@ -27,18 +27,14 @@ class ReactionKind(Enumeration):
class Reaction(Base): class Reaction(Base):
__tablename__ = "reaction" __tablename__ = "reaction"
body = Column(String, nullable=True, comment="Reaction Body") body = Column(String, nullable=True, comment="Reaction Body")
createdAt = Column( createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
DateTime, nullable=False, default=datetime.now, comment="Created at"
)
createdBy = Column(ForeignKey("user.id"), nullable=False, index=True, comment="Sender") createdBy = Column(ForeignKey("user.id"), nullable=False, index=True, comment="Sender")
updatedAt = Column(DateTime, nullable=True, comment="Updated at") updatedAt = Column(DateTime, nullable=True, comment="Updated at")
updatedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor") updatedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor")
deletedAt = Column(DateTime, nullable=True, comment="Deleted at") deletedAt = Column(DateTime, nullable=True, comment="Deleted at")
deletedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by") deletedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by")
shout = Column(ForeignKey("shout.id"), nullable=False, index=True) shout = Column(ForeignKey("shout.id"), nullable=False, index=True)
replyTo = Column( replyTo = Column(ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID")
ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID"
)
range = Column(String, nullable=True, comment="Range in format <start index>:<end>") range = Column(String, nullable=True, comment="Range in format <start index>:<end>")
kind = Column(Enum(ReactionKind), nullable=False, comment="Reaction kind") kind = Column(Enum(ReactionKind), nullable=False, comment="Reaction kind")
oid = Column(String, nullable=True, comment="Old ID") oid = Column(String, nullable=True, comment="Old ID")

View File

@ -1,6 +1,6 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, JSON from sqlalchemy import JSON, Boolean, Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.orm import column_property, relationship from sqlalchemy.orm import column_property, relationship
from base.orm import Base, local_session from base.orm import Base, local_session
@ -24,9 +24,7 @@ class ShoutReactionsFollower(Base):
follower = Column(ForeignKey("user.id"), primary_key=True, index=True) follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
shout = Column(ForeignKey("shout.id"), primary_key=True, index=True) shout = Column(ForeignKey("shout.id"), primary_key=True, index=True)
auto = Column(Boolean, nullable=False, default=False) auto = Column(Boolean, nullable=False, default=False)
createdAt = Column( createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
DateTime, nullable=False, default=datetime.now, comment="Created at"
)
deletedAt = Column(DateTime, nullable=True) deletedAt = Column(DateTime, nullable=True)
@ -83,12 +81,7 @@ class Shout(Base):
with local_session() as session: with local_session() as session:
s = session.query(Shout).first() s = session.query(Shout).first()
if not s: if not s:
entry = { entry = {"slug": "genesis-block", "body": "", "title": "Ничего", "lang": "ru"}
"slug": "genesis-block",
"body": "",
"title": "Ничего",
"lang": "ru"
}
s = Shout.create(**entry) s = Shout.create(**entry)
session.add(s) session.add(s)
session.commit() session.commit()

View File

@ -11,9 +11,7 @@ class TopicFollower(Base):
id = None # type: ignore id = None # type: ignore
follower = Column(ForeignKey("user.id"), primary_key=True, index=True) follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
topic = Column(ForeignKey("topic.id"), primary_key=True, index=True) topic = Column(ForeignKey("topic.id"), primary_key=True, index=True)
createdAt = Column( createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
DateTime, nullable=False, default=datetime.now, comment="Created at"
)
auto = Column(Boolean, nullable=False, default=False) auto = Column(Boolean, nullable=False, default=False)
@ -24,7 +22,5 @@ class Topic(Base):
title = Column(String, nullable=False, comment="Title") title = Column(String, nullable=False, comment="Title")
body = Column(String, nullable=True, comment="Body") body = Column(String, nullable=True, comment="Body")
pic = Column(String, nullable=True, comment="Picture") pic = Column(String, nullable=True, comment="Picture")
community = Column( community = Column(ForeignKey("community.id"), default=1, comment="Community")
ForeignKey("community.id"), default=1, comment="Community"
)
oid = Column(String, nullable=True, comment="Old ID") oid = Column(String, nullable=True, comment="Old ID")

View File

@ -3,6 +3,7 @@ from datetime import datetime
from sqlalchemy import JSON as JSONType from sqlalchemy import JSON as JSONType
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from base.orm import Base, local_session from base.orm import Base, local_session
from orm.rbac import Role from orm.rbac import Role
@ -34,9 +35,7 @@ class AuthorFollower(Base):
id = None # type: ignore id = None # type: ignore
follower = Column(ForeignKey("user.id"), primary_key=True, index=True) follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
author = Column(ForeignKey("user.id"), primary_key=True, index=True) author = Column(ForeignKey("user.id"), primary_key=True, index=True)
createdAt = Column( createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
DateTime, nullable=False, default=datetime.now, comment="Created at"
)
auto = Column(Boolean, nullable=False, default=False) auto = Column(Boolean, nullable=False, default=False)
@ -54,12 +53,8 @@ class User(Base):
slug = Column(String, unique=True, comment="User's slug") slug = Column(String, unique=True, comment="User's slug")
muted = Column(Boolean, default=False) muted = Column(Boolean, default=False)
emailConfirmed = Column(Boolean, default=False) emailConfirmed = Column(Boolean, default=False)
createdAt = Column( createdAt = Column(DateTime, nullable=False, default=datetime.now, comment="Created at")
DateTime, nullable=False, default=datetime.now, comment="Created at" lastSeen = Column(DateTime, nullable=False, default=datetime.now, comment="Was online at")
)
lastSeen = Column(
DateTime, nullable=False, default=datetime.now, comment="Was online at"
)
deletedAt = Column(DateTime, nullable=True, comment="Deleted at") deletedAt = Column(DateTime, nullable=True, comment="Deleted at")
links = Column(JSONType, nullable=True, comment="Links") links = Column(JSONType, nullable=True, comment="Links")
oauth = Column(String, nullable=True) oauth = Column(String, nullable=True)

View File

@ -2,3 +2,4 @@ isort
brunette brunette
flake8 flake8
mypy mypy
pre-commit

View File

@ -53,4 +53,3 @@ echo "Start migration"
python3 server.py migrate python3 server.py migrate
if [ $? -ne 0 ]; then { echo "Migration failed, aborting." ; exit 1; } fi if [ $? -ne 0 ]; then { echo "Migration failed, aborting." ; exit 1; } fi
echo 'Done!' echo 'Done!'

View File

@ -1,67 +1,35 @@
from resolvers.auth import ( from resolvers.auth import (
login,
sign_out,
is_email_used,
register_by_email,
confirm_email,
auth_send_link, auth_send_link,
confirm_email,
get_current_user, get_current_user,
is_email_used,
login,
register_by_email,
sign_out,
) )
from resolvers.create.migrate import markdown_body
from resolvers.create.editor import create_shout, delete_shout, update_shout from resolvers.create.editor import create_shout, delete_shout, update_shout
from resolvers.create.migrate import markdown_body
from resolvers.zine.profile import ( from resolvers.inbox.chats import create_chat, delete_chat, update_chat
load_authors_by, from resolvers.inbox.load import load_chats, load_messages_by, load_recipients
rate_user, from resolvers.inbox.messages import create_message, delete_message, mark_as_read, update_message
update_profile, from resolvers.inbox.search import search_recipients
get_authors_all from resolvers.notifications import load_notifications
) from resolvers.zine.following import follow, unfollow
from resolvers.zine.load import load_shout, load_shouts_by
from resolvers.zine.profile import get_authors_all, load_authors_by, rate_user, update_profile
from resolvers.zine.reactions import ( from resolvers.zine.reactions import (
create_reaction, create_reaction,
delete_reaction, delete_reaction,
update_reaction, load_reactions_by,
reactions_unfollow,
reactions_follow, reactions_follow,
load_reactions_by reactions_unfollow,
update_reaction,
) )
from resolvers.zine.topics import ( from resolvers.zine.topics import (
get_topic,
topic_follow, topic_follow,
topic_unfollow, topic_unfollow,
topics_all,
topics_by_author, topics_by_author,
topics_by_community, topics_by_community,
topics_all,
get_topic
) )
from resolvers.zine.following import (
follow,
unfollow
)
from resolvers.zine.load import (
load_shout,
load_shouts_by
)
from resolvers.inbox.chats import (
create_chat,
delete_chat,
update_chat
)
from resolvers.inbox.messages import (
create_message,
delete_message,
update_message,
mark_as_read
)
from resolvers.inbox.load import (
load_chats,
load_messages_by,
load_recipients
)
from resolvers.inbox.search import search_recipients
from resolvers.notifications import load_notifications

View File

@ -1,24 +1,30 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import re
from datetime import datetime, timezone from datetime import datetime, timezone
from urllib.parse import quote_plus from urllib.parse import quote_plus
from graphql.type import GraphQLResolveInfo from graphql.type import GraphQLResolveInfo
from starlette.responses import RedirectResponse from starlette.responses import RedirectResponse
from transliterate import translit from transliterate import translit
import re
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from auth.email import send_auth_email from auth.email import send_auth_email
from auth.identity import Identity, Password from auth.identity import Identity, Password
from auth.jwtcodec import JWTCodec from auth.jwtcodec import JWTCodec
from auth.tokenstorage import TokenStorage from auth.tokenstorage import TokenStorage
from base.exceptions import (BaseHttpException, InvalidPassword, InvalidToken, from base.exceptions import (
ObjectNotExist, Unauthorized) BaseHttpException,
InvalidPassword,
InvalidToken,
ObjectNotExist,
Unauthorized,
)
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from orm import Role, User from orm import Role, User
from settings import SESSION_TOKEN_HEADER, FRONTEND_URL from settings import FRONTEND_URL, SESSION_TOKEN_HEADER
@mutation.field("getSession") @mutation.field("getSession")
@ -32,10 +38,7 @@ async def get_current_user(_, info):
user.lastSeen = datetime.now(tz=timezone.utc) user.lastSeen = datetime.now(tz=timezone.utc)
session.commit() session.commit()
return { return {"token": token, "user": user}
"token": token,
"user": user
}
@mutation.field("confirmEmail") @mutation.field("confirmEmail")
@ -53,10 +56,7 @@ async def confirm_email(_, info, token):
user.lastSeen = datetime.now(tz=timezone.utc) user.lastSeen = datetime.now(tz=timezone.utc)
session.add(user) session.add(user)
session.commit() session.commit()
return { return {"token": session_token, "user": user}
"token": session_token,
"user": user
}
except InvalidToken as e: except InvalidToken as e:
raise InvalidToken(e.message) raise InvalidToken(e.message)
except Exception as e: except Exception as e:
@ -122,7 +122,7 @@ async def register_by_email(_, _info, email: str, password: str = "", name: str
"email": email, "email": email,
"username": email, # will be used to store phone number or some messenger network id "username": email, # will be used to store phone number or some messenger network id
"name": name, "name": name,
"slug": slug "slug": slug,
} }
if password: if password:
user_dict["password"] = Password.encode(password) user_dict["password"] = Password.encode(password)
@ -172,10 +172,7 @@ async def login(_, info, email: str, password: str = "", lang: str = "ru"):
user = Identity.password(orm_user, password) user = Identity.password(orm_user, password)
session_token = await TokenStorage.create_session(user) session_token = await TokenStorage.create_session(user)
print(f"[auth] user {email} authorized") print(f"[auth] user {email} authorized")
return { return {"token": session_token, "user": user}
"token": session_token,
"user": user
}
except InvalidPassword: except InvalidPassword:
print(f"[auth] {email}: invalid password") print(f"[auth] {email}: invalid password")
raise InvalidPassword("invalid password") # contains webserver status raise InvalidPassword("invalid password") # contains webserver status

View File

@ -20,7 +20,8 @@ async def create_shout(_, info, inp):
with local_session() as session: with local_session() as session:
topics = session.query(Topic).filter(Topic.slug.in_(inp.get('topics', []))).all() topics = session.query(Topic).filter(Topic.slug.in_(inp.get('topics', []))).all()
new_shout = Shout.create(**{ new_shout = Shout.create(
**{
"title": inp.get("title"), "title": inp.get("title"),
"subtitle": inp.get('subtitle'), "subtitle": inp.get('subtitle'),
"lead": inp.get('lead'), "lead": inp.get('lead'),
@ -31,8 +32,9 @@ async def create_shout(_, info, inp):
"slug": inp.get("slug"), "slug": inp.get("slug"),
"mainTopic": inp.get("mainTopic"), "mainTopic": inp.get("mainTopic"),
"visibility": "owner", "visibility": "owner",
"createdBy": auth.user_id "createdBy": auth.user_id,
}) }
)
for topic in topics: for topic in topics:
t = ShoutTopic.create(topic=topic.id, shout=new_shout.id) t = ShoutTopic.create(topic=topic.id, shout=new_shout.id)
@ -64,10 +66,15 @@ async def update_shout(_, info, shout_id, shout_input=None, publish=False):
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
with local_session() as session: with local_session() as session:
shout = session.query(Shout).options( shout = (
session.query(Shout)
.options(
joinedload(Shout.authors), joinedload(Shout.authors),
joinedload(Shout.topics), joinedload(Shout.topics),
).filter(Shout.id == shout_id).first() )
.filter(Shout.id == shout_id)
.first()
)
if not shout: if not shout:
return {"error": "shout not found"} return {"error": "shout not found"}
@ -94,25 +101,34 @@ async def update_shout(_, info, shout_id, shout_input=None, publish=False):
session.commit() session.commit()
for new_topic_to_link in new_topics_to_link: for new_topic_to_link in new_topics_to_link:
created_unlinked_topic = ShoutTopic.create(shout=shout.id, topic=new_topic_to_link.id) created_unlinked_topic = ShoutTopic.create(
shout=shout.id, topic=new_topic_to_link.id
)
session.add(created_unlinked_topic) session.add(created_unlinked_topic)
existing_topics_input = [topic_input for topic_input in topics_input if topic_input.get("id", 0) > 0] existing_topics_input = [
existing_topic_to_link_ids = [existing_topic_input["id"] for existing_topic_input in existing_topics_input topic_input for topic_input in topics_input if topic_input.get("id", 0) > 0
if existing_topic_input["id"] not in [topic.id for topic in shout.topics]] ]
existing_topic_to_link_ids = [
existing_topic_input["id"]
for existing_topic_input in existing_topics_input
if existing_topic_input["id"] not in [topic.id for topic in shout.topics]
]
for existing_topic_to_link_id in existing_topic_to_link_ids: for existing_topic_to_link_id in existing_topic_to_link_ids:
created_unlinked_topic = ShoutTopic.create(shout=shout.id, topic=existing_topic_to_link_id) created_unlinked_topic = ShoutTopic.create(
shout=shout.id, topic=existing_topic_to_link_id
)
session.add(created_unlinked_topic) session.add(created_unlinked_topic)
topic_to_unlink_ids = [topic.id for topic in shout.topics topic_to_unlink_ids = [
if topic.id not in [topic_input["id"] for topic_input in existing_topics_input]] topic.id
for topic in shout.topics
if topic.id not in [topic_input["id"] for topic_input in existing_topics_input]
]
shout_topics_to_remove = session.query(ShoutTopic).filter( shout_topics_to_remove = session.query(ShoutTopic).filter(
and_( and_(ShoutTopic.shout == shout.id, ShoutTopic.topic.in_(topic_to_unlink_ids))
ShoutTopic.shout == shout.id,
ShoutTopic.topic.in_(topic_to_unlink_ids)
)
) )
for shout_topic_to_remove in shout_topics_to_remove: for shout_topic_to_remove in shout_topics_to_remove:

View File

@ -1,7 +1,6 @@
from base.resolvers import query from base.resolvers import query
from resolvers.auth import login_required
from migration.extract import extract_md from migration.extract import extract_md
from resolvers.auth import login_required
@login_required @login_required

View File

@ -24,27 +24,24 @@ async def update_chat(_, info, chat_new: Chat):
chat_id = chat_new["id"] chat_id = chat_new["id"]
chat = await redis.execute("GET", f"chats/{chat_id}") chat = await redis.execute("GET", f"chats/{chat_id}")
if not chat: if not chat:
return { return {"error": "chat not exist"}
"error": "chat not exist"
}
chat = dict(json.loads(chat)) chat = dict(json.loads(chat))
# TODO # TODO
if auth.user_id in chat["admins"]: if auth.user_id in chat["admins"]:
chat.update({ chat.update(
{
"title": chat_new.get("title", chat["title"]), "title": chat_new.get("title", chat["title"]),
"description": chat_new.get("description", chat["description"]), "description": chat_new.get("description", chat["description"]),
"updatedAt": int(datetime.now(tz=timezone.utc).timestamp()), "updatedAt": int(datetime.now(tz=timezone.utc).timestamp()),
"admins": chat_new.get("admins", chat.get("admins") or []), "admins": chat_new.get("admins", chat.get("admins") or []),
"users": chat_new.get("users", chat["users"]) "users": chat_new.get("users", chat["users"]),
}) }
)
await redis.execute("SET", f"chats/{chat.id}", json.dumps(chat)) await redis.execute("SET", f"chats/{chat.id}", json.dumps(chat))
await redis.execute("COMMIT") await redis.execute("COMMIT")
return { return {"error": None, "chat": chat}
"error": None,
"chat": chat
}
@mutation.field("createChat") @mutation.field("createChat")
@ -79,10 +76,7 @@ async def create_chat(_, info, title="", members=[]):
print(chat) print(chat)
break break
if chat: if chat:
return { return {"chat": chat, "error": "existed"}
"chat": chat,
"error": "existed"
}
chat_id = str(uuid.uuid4()) chat_id = str(uuid.uuid4())
chat = { chat = {
@ -92,7 +86,7 @@ async def create_chat(_, info, title="", members=[]):
"createdBy": auth.user_id, "createdBy": auth.user_id,
"createdAt": int(datetime.now(tz=timezone.utc).timestamp()), "createdAt": int(datetime.now(tz=timezone.utc).timestamp()),
"updatedAt": int(datetime.now(tz=timezone.utc).timestamp()), "updatedAt": int(datetime.now(tz=timezone.utc).timestamp()),
"admins": members if (len(members) == 2 and title == "") else [] "admins": members if (len(members) == 2 and title == "") else [],
} }
for m in members: for m in members:
@ -100,10 +94,7 @@ async def create_chat(_, info, title="", members=[]):
await redis.execute("SET", f"chats/{chat_id}", json.dumps(chat)) await redis.execute("SET", f"chats/{chat_id}", json.dumps(chat))
await redis.execute("SET", f"chats/{chat_id}/next_message_id", str(0)) await redis.execute("SET", f"chats/{chat_id}/next_message_id", str(0))
await redis.execute("COMMIT") await redis.execute("COMMIT")
return { return {"error": None, "chat": chat}
"error": None,
"chat": chat
}
@mutation.field("deleteChat") @mutation.field("deleteChat")
@ -119,6 +110,4 @@ async def delete_chat(_, info, chat_id: str):
await redis.execute("SREM", "chats_by_user/" + str(auth.user_id), chat_id) await redis.execute("SREM", "chats_by_user/" + str(auth.user_id), chat_id)
await redis.execute("COMMIT") await redis.execute("COMMIT")
else: else:
return { return {"error": "chat not exist"}
"error": "chat not exist"
}

View File

@ -1,15 +1,17 @@
import json import json
# from datetime import datetime, timedelta, timezone
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.redis import redis
from base.orm import local_session from base.orm import local_session
from base.redis import redis
from base.resolvers import query from base.resolvers import query
from orm.user import User from orm.user import User
from resolvers.zine.profile import followed_authors from resolvers.zine.profile import followed_authors
from .unread import get_unread_counter from .unread import get_unread_counter
# from datetime import datetime, timedelta, timezone
async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]): async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]):
'''load :limit messages for :chat_id with :offset''' '''load :limit messages for :chat_id with :offset'''
@ -19,10 +21,7 @@ async def load_messages(chat_id: str, limit: int = 5, offset: int = 0, ids=[]):
message_ids += ids message_ids += ids
try: try:
if limit: if limit:
mids = await redis.lrange(f"chats/{chat_id}/message_ids", mids = await redis.lrange(f"chats/{chat_id}/message_ids", offset, offset + limit)
offset,
offset + limit
)
mids = [mid.decode("utf-8") for mid in mids] mids = [mid.decode("utf-8") for mid in mids]
message_ids += mids message_ids += mids
except Exception as e: except Exception as e:
@ -71,19 +70,18 @@ async def load_chats(_, info, limit: int = 50, offset: int = 0):
for uid in c["users"]: for uid in c["users"]:
a = session.query(User).where(User.id == uid).first() a = session.query(User).where(User.id == uid).first()
if a: if a:
c['members'].append({ c['members'].append(
{
"id": a.id, "id": a.id,
"slug": a.slug, "slug": a.slug,
"userpic": a.userpic, "userpic": a.userpic,
"name": a.name, "name": a.name,
"lastSeen": a.lastSeen, "lastSeen": a.lastSeen,
"online": a.id in onliners "online": a.id in onliners,
})
chats.append(c)
return {
"chats": chats,
"error": None
} }
)
chats.append(c)
return {"chats": chats, "error": None}
@query.field("loadMessagesBy") @query.field("loadMessagesBy")
@ -103,23 +101,12 @@ async def load_messages_by(_, info, by, limit: int = 10, offset: int = 0):
chat = await redis.execute("GET", f"chats/{by_chat}") chat = await redis.execute("GET", f"chats/{by_chat}")
# print(chat) # print(chat)
if not chat: if not chat:
return { return {"messages": [], "error": "chat not exist"}
"messages": [],
"error": "chat not exist"
}
# everyone's messages in filtered chat # everyone's messages in filtered chat
messages = await load_messages(by_chat, limit, offset) messages = await load_messages(by_chat, limit, offset)
return { return {"messages": sorted(list(messages), key=lambda m: m['createdAt']), "error": None}
"messages": sorted(
list(messages),
key=lambda m: m['createdAt']
),
"error": None
}
else: else:
return { return {"error": "Cannot access messages of this chat"}
"error": "Cannot access messages of this chat"
}
@query.field("loadRecipients") @query.field("loadRecipients")
@ -138,15 +125,14 @@ async def load_recipients(_, info, limit=50, offset=0):
chat_users += session.query(User).where(User.emailConfirmed).limit(limit).offset(offset) chat_users += session.query(User).where(User.emailConfirmed).limit(limit).offset(offset)
members = [] members = []
for a in chat_users: for a in chat_users:
members.append({ members.append(
{
"id": a.id, "id": a.id,
"slug": a.slug, "slug": a.slug,
"userpic": a.userpic, "userpic": a.userpic,
"name": a.name, "name": a.name,
"lastSeen": a.lastSeen, "lastSeen": a.lastSeen,
"online": a.id in onliners "online": a.id in onliners,
})
return {
"members": members,
"error": None
} }
)
return {"members": members, "error": None}

View File

@ -1,13 +1,15 @@
import asyncio import asyncio
import json import json
from typing import Any
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Any
from graphql.type import GraphQLResolveInfo from graphql.type import GraphQLResolveInfo
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.redis import redis from base.redis import redis
from base.resolvers import mutation from base.resolvers import mutation
from services.following import FollowingManager, FollowingResult, Following from services.following import Following, FollowingManager, FollowingResult
from validations.inbox import Message from validations.inbox import Message
@ -19,9 +21,7 @@ async def create_message(_, info, chat: str, body: str, replyTo=None):
chat = await redis.execute("GET", f"chats/{chat}") chat = await redis.execute("GET", f"chats/{chat}")
if not chat: if not chat:
return { return {"error": "chat is not exist"}
"error": "chat is not exist"
}
else: else:
chat = dict(json.loads(chat)) chat = dict(json.loads(chat))
message_id = await redis.execute("GET", f"chats/{chat['id']}/next_message_id") message_id = await redis.execute("GET", f"chats/{chat['id']}/next_message_id")
@ -31,7 +31,7 @@ async def create_message(_, info, chat: str, body: str, replyTo=None):
"id": message_id, "id": message_id,
"author": auth.user_id, "author": auth.user_id,
"body": body, "body": body,
"createdAt": int(datetime.now(tz=timezone.utc).timestamp()) "createdAt": int(datetime.now(tz=timezone.utc).timestamp()),
} }
if replyTo: if replyTo:
new_message['replyTo'] = replyTo new_message['replyTo'] = replyTo
@ -46,17 +46,12 @@ async def create_message(_, info, chat: str, body: str, replyTo=None):
users = chat["users"] users = chat["users"]
for user_slug in users: for user_slug in users:
await redis.execute( await redis.execute("LPUSH", f"chats/{chat['id']}/unread/{user_slug}", str(message_id))
"LPUSH", f"chats/{chat['id']}/unread/{user_slug}", str(message_id)
)
result = FollowingResult("NEW", 'chat', new_message) result = FollowingResult("NEW", 'chat', new_message)
await FollowingManager.push('chat', result) await FollowingManager.push('chat', result)
return { return {"message": new_message, "error": None}
"message": new_message,
"error": None
}
@mutation.field("updateMessage") @mutation.field("updateMessage")
@ -84,10 +79,7 @@ async def update_message(_, info, chat_id: str, message_id: int, body: str):
result = FollowingResult("UPDATED", 'chat', message) result = FollowingResult("UPDATED", 'chat', message)
await FollowingManager.push('chat', result) await FollowingManager.push('chat', result)
return { return {"message": message, "error": None}
"message": message,
"error": None
}
@mutation.field("deleteMessage") @mutation.field("deleteMessage")
@ -137,6 +129,4 @@ async def mark_as_read(_, info, chat_id: str, messages: [int]):
for message_id in messages: for message_id in messages:
await redis.execute("LREM", f"chats/{chat_id}/unread/{auth.user_id}", 0, str(message_id)) await redis.execute("LREM", f"chats/{chat_id}/unread/{auth.user_id}", 0, str(message_id))
return { return {"error": None}
"error": None
}

View File

@ -1,10 +1,11 @@
import json import json
from datetime import datetime, timezone, timedelta from datetime import datetime, timedelta, timezone
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.orm import local_session
from base.redis import redis from base.redis import redis
from base.resolvers import query from base.resolvers import query
from base.orm import local_session
from orm.user import AuthorFollower, User from orm.user import AuthorFollower, User
from resolvers.inbox.load import load_messages from resolvers.inbox.load import load_messages
@ -31,23 +32,24 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int =
with local_session() as session: with local_session() as session:
# followings # followings
result += session.query(AuthorFollower.author).join( result += (
User, User.id == AuthorFollower.follower session.query(AuthorFollower.author)
).where( .join(User, User.id == AuthorFollower.follower)
User.slug.startswith(query) .where(User.slug.startswith(query))
).offset(offset + len(result)).limit(more_amount) .offset(offset + len(result))
.limit(more_amount)
)
more_amount = limit more_amount = limit
# followers # followers
result += session.query(AuthorFollower.follower).join( result += (
User, User.id == AuthorFollower.author session.query(AuthorFollower.follower)
).where( .join(User, User.id == AuthorFollower.author)
User.slug.startswith(query) .where(User.slug.startswith(query))
).offset(offset + len(result)).limit(offset + len(result) + limit) .offset(offset + len(result))
return { .limit(offset + len(result) + limit)
"members": list(result), )
"error": None return {"members": list(result), "error": None}
}
@query.field("searchMessages") @query.field("searchMessages")
@ -83,13 +85,12 @@ async def search_user_chats(by, messages, user_id: int, limit, offset):
days = by.get("days") days = by.get("days")
if days: if days:
messages.extend(filter( messages.extend(
filter(
list(messages), list(messages),
key=lambda m: ( key=lambda m: (
datetime.now(tz=timezone.utc) - int(m["createdAt"]) < timedelta(days=by["days"]) datetime.now(tz=timezone.utc) - int(m["createdAt"]) < timedelta(days=by["days"])
),
) )
)) )
return { return {"messages": messages, "error": None}
"messages": messages,
"error": None
}

View File

@ -1,9 +1,9 @@
from sqlalchemy import select, desc, and_, update from sqlalchemy import and_, desc, select, update
from auth.credentials import AuthCredentials
from base.resolvers import query, mutation
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query
from orm import Notification from orm import Notification
@ -19,22 +19,23 @@ async def load_notifications(_, info, params=None):
limit = params.get('limit', 50) limit = params.get('limit', 50)
offset = params.get('offset', 0) offset = params.get('offset', 0)
q = select(Notification).where( q = (
Notification.user == user_id select(Notification)
).order_by(desc(Notification.createdAt)).limit(limit).offset(offset) .where(Notification.user == user_id)
.order_by(desc(Notification.createdAt))
.limit(limit)
.offset(offset)
)
notifications = [] notifications = []
with local_session() as session: with local_session() as session:
total_count = session.query(Notification).where( total_count = session.query(Notification).where(Notification.user == user_id).count()
Notification.user == user_id
).count()
total_unread_count = session.query(Notification).where( total_unread_count = (
and_( session.query(Notification)
Notification.user == user_id, .where(and_(Notification.user == user_id, Notification.seen == False))
Notification.seen == False .count()
) )
).count()
for [notification] in session.execute(q): for [notification] in session.execute(q):
notification.type = notification.type.name notification.type = notification.type.name
@ -43,7 +44,7 @@ async def load_notifications(_, info, params=None):
return { return {
"notifications": notifications, "notifications": notifications,
"totalCount": total_count, "totalCount": total_count,
"totalUnreadCount": total_unread_count "totalUnreadCount": total_unread_count,
} }
@ -54,9 +55,11 @@ async def mark_notification_as_read(_, info, notification_id: int):
user_id = auth.user_id user_id = auth.user_id
with local_session() as session: with local_session() as session:
notification = session.query(Notification).where( notification = (
and_(Notification.id == notification_id, Notification.user == user_id) session.query(Notification)
).one() .where(and_(Notification.id == notification_id, Notification.user == user_id))
.one()
)
notification.seen = True notification.seen = True
session.commit() session.commit()
@ -69,12 +72,11 @@ async def mark_all_notifications_as_read(_, info):
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
user_id = auth.user_id user_id = auth.user_id
statement = update(Notification).where( statement = (
and_( update(Notification)
Notification.user == user_id, .where(and_(Notification.user == user_id, Notification.seen == False))
Notification.seen == False .values(seen=True)
) )
).values(seen=True)
with local_session() as session: with local_session() as session:
try: try:

View File

@ -2,6 +2,7 @@ import os
import shutil import shutil
import tempfile import tempfile
import uuid import uuid
import boto3 import boto3
from botocore.exceptions import BotoCoreError, ClientError from botocore.exceptions import BotoCoreError, ClientError
from starlette.responses import JSONResponse from starlette.responses import JSONResponse
@ -25,10 +26,12 @@ async def upload_handler(request):
key = 'files/' + str(uuid.uuid4()) + file_extension key = 'files/' + str(uuid.uuid4()) + file_extension
# Create an S3 client with Storj configuration # Create an S3 client with Storj configuration
s3 = boto3.client('s3', s3 = boto3.client(
's3',
aws_access_key_id=STORJ_ACCESS_KEY, aws_access_key_id=STORJ_ACCESS_KEY,
aws_secret_access_key=STORJ_SECRET_KEY, aws_secret_access_key=STORJ_SECRET_KEY,
endpoint_url=STORJ_END_POINT) endpoint_url=STORJ_END_POINT,
)
try: try:
# Save the uploaded file to a temporary file # Save the uploaded file to a temporary file
@ -39,9 +42,7 @@ async def upload_handler(request):
Filename=tmp_file.name, Filename=tmp_file.name,
Bucket=STORJ_BUCKET_NAME, Bucket=STORJ_BUCKET_NAME,
Key=key, Key=key,
ExtraArgs={ ExtraArgs={"ContentType": file.content_type},
"ContentType": file.content_type
}
) )
url = 'https://' + CDN_DOMAIN + '/' + key url = 'https://' + CDN_DOMAIN + '/' + key
@ -51,6 +52,3 @@ async def upload_handler(request):
except (BotoCoreError, ClientError) as e: except (BotoCoreError, ClientError) as e:
print(e) print(e)
return JSONResponse({'error': 'Failed to upload file'}, status_code=500) return JSONResponse({'error': 'Failed to upload file'}, status_code=500)

View File

@ -1,17 +1,20 @@
import asyncio import asyncio
from base.orm import local_session
from base.resolvers import mutation from graphql.type import GraphQLResolveInfo
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
from base.orm import local_session
from base.resolvers import mutation
from orm.shout import ShoutReactionsFollower
from orm.topic import TopicFollower
# from resolvers.community import community_follow, community_unfollow # from resolvers.community import community_follow, community_unfollow
from orm.user import AuthorFollower from orm.user import AuthorFollower
from orm.topic import TopicFollower
from orm.shout import ShoutReactionsFollower
from resolvers.zine.profile import author_follow, author_unfollow from resolvers.zine.profile import author_follow, author_unfollow
from resolvers.zine.reactions import reactions_follow, reactions_unfollow from resolvers.zine.reactions import reactions_follow, reactions_unfollow
from resolvers.zine.topics import topic_follow, topic_unfollow from resolvers.zine.topics import topic_follow, topic_unfollow
from services.following import Following, FollowingManager, FollowingResult from services.following import Following, FollowingManager, FollowingResult
from graphql.type import GraphQLResolveInfo
@mutation.field("follow") @mutation.field("follow")

View File

@ -1,7 +1,7 @@
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from sqlalchemy.orm import joinedload, aliased from sqlalchemy.orm import aliased, joinedload
from sqlalchemy.sql.expression import desc, asc, select, func, case, and_, text, nulls_last from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select, text
from auth.authenticate import login_required from auth.authenticate import login_required
from auth.credentials import AuthCredentials from auth.credentials import AuthCredentials
@ -18,16 +18,12 @@ def add_stat_columns(q):
aliased_reaction = aliased(Reaction) aliased_reaction = aliased(Reaction)
q = q.outerjoin(aliased_reaction).add_columns( q = q.outerjoin(aliased_reaction).add_columns(
func.sum( func.sum(aliased_reaction.id).label('reacted_stat'),
aliased_reaction.id func.sum(case((aliased_reaction.kind == ReactionKind.COMMENT, 1), else_=0)).label(
).label('reacted_stat'), 'commented_stat'
),
func.sum( func.sum(
case( case(
(aliased_reaction.kind == ReactionKind.COMMENT, 1),
else_=0
)
).label('commented_stat'),
func.sum(case(
# do not count comments' reactions # do not count comments' reactions
(aliased_reaction.replyTo.is_not(None), 0), (aliased_reaction.replyTo.is_not(None), 0),
(aliased_reaction.kind == ReactionKind.AGREE, 1), (aliased_reaction.kind == ReactionKind.AGREE, 1),
@ -38,12 +34,16 @@ def add_stat_columns(q):
(aliased_reaction.kind == ReactionKind.REJECT, -1), (aliased_reaction.kind == ReactionKind.REJECT, -1),
(aliased_reaction.kind == ReactionKind.LIKE, 1), (aliased_reaction.kind == ReactionKind.LIKE, 1),
(aliased_reaction.kind == ReactionKind.DISLIKE, -1), (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
else_=0) else_=0,
)
).label('rating_stat'), ).label('rating_stat'),
func.max(case( func.max(
case(
(aliased_reaction.kind != ReactionKind.COMMENT, None), (aliased_reaction.kind != ReactionKind.COMMENT, None),
else_=aliased_reaction.createdAt else_=aliased_reaction.createdAt,
)).label('last_comment')) )
).label('last_comment'),
)
return q return q
@ -87,27 +87,23 @@ async def load_shout(_, info, slug=None, shout_id=None):
q = add_stat_columns(q) q = add_stat_columns(q)
if slug is not None: if slug is not None:
q = q.filter( q = q.filter(Shout.slug == slug)
Shout.slug == slug
)
if shout_id is not None: if shout_id is not None:
q = q.filter( q = q.filter(Shout.id == shout_id)
Shout.id == shout_id
)
q = q.filter( q = q.filter(Shout.deletedAt.is_(None)).group_by(Shout.id)
Shout.deletedAt.is_(None)
).group_by(Shout.id)
try: try:
[shout, reacted_stat, commented_stat, rating_stat, last_comment] = session.execute(q).first() [shout, reacted_stat, commented_stat, rating_stat, last_comment] = session.execute(
q
).first()
shout.stat = { shout.stat = {
"viewed": shout.views, "viewed": shout.views,
"reacted": reacted_stat, "reacted": reacted_stat,
"commented": commented_stat, "commented": commented_stat,
"rating": rating_stat "rating": rating_stat,
} }
for author_caption in session.query(ShoutAuthor).join(Shout).where(Shout.slug == slug): for author_caption in session.query(ShoutAuthor).join(Shout).where(Shout.slug == slug):
@ -142,14 +138,13 @@ async def load_shouts_by(_, info, options):
:return: Shout[] :return: Shout[]
""" """
q = select(Shout).options( q = (
select(Shout)
.options(
joinedload(Shout.authors), joinedload(Shout.authors),
joinedload(Shout.topics), joinedload(Shout.topics),
).where(
and_(
Shout.deletedAt.is_(None),
Shout.layout.is_not(None)
) )
.where(and_(Shout.deletedAt.is_(None), Shout.layout.is_not(None)))
) )
q = add_stat_columns(q) q = add_stat_columns(q)
@ -169,13 +164,15 @@ async def load_shouts_by(_, info, options):
with local_session() as session: with local_session() as session:
shouts_map = {} shouts_map = {}
for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute(q).unique(): for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute(
q
).unique():
shouts.append(shout) shouts.append(shout)
shout.stat = { shout.stat = {
"viewed": shout.views, "viewed": shout.views,
"reacted": reacted_stat, "reacted": reacted_stat,
"commented": commented_stat, "commented": commented_stat,
"rating": rating_stat "rating": rating_stat,
} }
shouts_map[shout.id] = shout shouts_map[shout.id] = shout
@ -188,11 +185,13 @@ async def get_drafts(_, info):
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
user_id = auth.user_id user_id = auth.user_id
q = select(Shout).options( q = (
select(Shout)
.options(
joinedload(Shout.authors), joinedload(Shout.authors),
joinedload(Shout.topics), joinedload(Shout.topics),
).where( )
and_(Shout.deletedAt.is_(None), Shout.createdBy == user_id) .where(and_(Shout.deletedAt.is_(None), Shout.createdBy == user_id))
) )
q = q.group_by(Shout.id) q = q.group_by(Shout.id)
@ -211,24 +210,22 @@ async def get_my_feed(_, info, options):
auth: AuthCredentials = info.context["request"].auth auth: AuthCredentials = info.context["request"].auth
user_id = auth.user_id user_id = auth.user_id
subquery = select(Shout.id).join( subquery = (
ShoutAuthor select(Shout.id)
).join( .join(ShoutAuthor)
AuthorFollower, AuthorFollower.follower == user_id .join(AuthorFollower, AuthorFollower.follower == user_id)
).join( .join(ShoutTopic)
ShoutTopic .join(TopicFollower, TopicFollower.follower == user_id)
).join(
TopicFollower, TopicFollower.follower == user_id
) )
q = select(Shout).options( q = (
select(Shout)
.options(
joinedload(Shout.authors), joinedload(Shout.authors),
joinedload(Shout.topics), joinedload(Shout.topics),
).where( )
and_( .where(
Shout.publishedAt.is_not(None), and_(Shout.publishedAt.is_not(None), Shout.deletedAt.is_(None), Shout.id.in_(subquery))
Shout.deletedAt.is_(None),
Shout.id.in_(subquery)
) )
) )
@ -246,13 +243,15 @@ async def get_my_feed(_, info, options):
shouts = [] shouts = []
with local_session() as session: with local_session() as session:
shouts_map = {} shouts_map = {}
for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute(q).unique(): for [shout, reacted_stat, commented_stat, rating_stat, last_comment] in session.execute(
q
).unique():
shouts.append(shout) shouts.append(shout)
shout.stat = { shout.stat = {
"viewed": shout.views, "viewed": shout.views,
"reacted": reacted_stat, "reacted": reacted_stat,
"commented": commented_stat, "commented": commented_stat,
"rating": rating_stat "rating": rating_stat,
} }
shouts_map[shout.id] = shout shouts_map[shout.id] = shout

View File

@ -1,6 +1,7 @@
from typing import List
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, func, distinct, select, literal from typing import List
from sqlalchemy import and_, distinct, func, literal, select
from sqlalchemy.orm import aliased, joinedload from sqlalchemy.orm import aliased, joinedload
from auth.authenticate import login_required from auth.authenticate import login_required
@ -55,7 +56,7 @@ def add_stat(author, stat_columns):
"followers": followers_stat, "followers": followers_stat,
"followings": followings_stat, "followings": followings_stat,
"rating": rating_stat, "rating": rating_stat,
"commented": commented_stat "commented": commented_stat,
} }
return author return author
@ -119,10 +120,10 @@ async def user_followers(_, _info, slug) -> List[User]:
q = add_author_stat_columns(q) q = add_author_stat_columns(q)
aliased_user = aliased(User) aliased_user = aliased(User)
q = q.join(AuthorFollower, AuthorFollower.follower == User.id).join( q = (
aliased_user, aliased_user.id == AuthorFollower.author q.join(AuthorFollower, AuthorFollower.follower == User.id)
).where( .join(aliased_user, aliased_user.id == AuthorFollower.author)
aliased_user.slug == slug .where(aliased_user.slug == slug)
) )
return get_authors_from_query(q) return get_authors_from_query(q)
@ -150,15 +151,10 @@ async def update_profile(_, info, profile):
with local_session() as session: with local_session() as session:
user = session.query(User).filter(User.id == user_id).one() user = session.query(User).filter(User.id == user_id).one()
if not user: if not user:
return { return {"error": "canoot find user"}
"error": "canoot find user"
}
user.update(profile) user.update(profile)
session.commit() session.commit()
return { return {"error": None, "author": user}
"error": None,
"author": user
}
@mutation.field("rateUser") @mutation.field("rateUser")
@ -200,13 +196,10 @@ def author_follow(user_id, slug):
def author_unfollow(user_id, slug): def author_unfollow(user_id, slug):
with local_session() as session: with local_session() as session:
flw = ( flw = (
session.query( session.query(AuthorFollower)
AuthorFollower .join(User, User.id == AuthorFollower.author)
).join(User, User.id == AuthorFollower.author).filter( .filter(and_(AuthorFollower.follower == user_id, User.slug == slug))
and_( .first()
AuthorFollower.follower == user_id, User.slug == slug
)
).first()
) )
if flw: if flw:
session.delete(flw) session.delete(flw)
@ -232,12 +225,11 @@ async def get_author(_, _info, slug):
[author] = get_authors_from_query(q) [author] = get_authors_from_query(q)
with local_session() as session: with local_session() as session:
comments_count = session.query(Reaction).where( comments_count = (
and_( session.query(Reaction)
Reaction.createdBy == author.id, .where(and_(Reaction.createdBy == author.id, Reaction.kind == ReactionKind.COMMENT))
Reaction.kind == ReactionKind.COMMENT .count()
) )
).count()
author.stat["commented"] = comments_count author.stat["commented"] = comments_count
return author return author
@ -260,9 +252,7 @@ async def load_authors_by(_, info, by, limit, offset):
days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"]) days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"])
q = q.filter(User.createdAt > days_before) q = q.filter(User.createdAt > days_before)
q = q.order_by( q = q.order_by(by.get("order", User.createdAt)).limit(limit).offset(offset)
by.get("order", User.createdAt)
).limit(limit).offset(offset)
return get_authors_from_query(q) return get_authors_from_query(q)
@ -273,13 +263,13 @@ async def load_my_subscriptions(_, info):
auth = info.context["request"].auth auth = info.context["request"].auth
user_id = auth.user_id user_id = auth.user_id
authors_query = select(User).join(AuthorFollower, AuthorFollower.author == User.id).where( authors_query = (
AuthorFollower.follower == user_id select(User)
.join(AuthorFollower, AuthorFollower.author == User.id)
.where(AuthorFollower.follower == user_id)
) )
topics_query = select(Topic).join(TopicFollower).where( topics_query = select(Topic).join(TopicFollower).where(TopicFollower.follower == user_id)
TopicFollower.follower == user_id
)
topics = [] topics = []
authors = [] authors = []
@ -291,7 +281,4 @@ async def load_my_subscriptions(_, info):
for [topic] in session.execute(topics_query): for [topic] in session.execute(topics_query):
topics.append(topic) topics.append(topic)
return { return {"topics": topics, "authors": authors}
"topics": topics,
"authors": authors
}

View File

@ -1,5 +1,6 @@
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, asc, desc, select, text, func, case
from sqlalchemy import and_, asc, case, desc, func, select, text
from sqlalchemy.orm import aliased from sqlalchemy.orm import aliased
from auth.authenticate import login_required from auth.authenticate import login_required
@ -17,16 +18,10 @@ def add_reaction_stat_columns(q):
aliased_reaction = aliased(Reaction) aliased_reaction = aliased(Reaction)
q = q.outerjoin(aliased_reaction, Reaction.id == aliased_reaction.replyTo).add_columns( q = q.outerjoin(aliased_reaction, Reaction.id == aliased_reaction.replyTo).add_columns(
func.sum( func.sum(aliased_reaction.id).label('reacted_stat'),
aliased_reaction.id func.sum(case((aliased_reaction.body.is_not(None), 1), else_=0)).label('commented_stat'),
).label('reacted_stat'),
func.sum( func.sum(
case( case(
(aliased_reaction.body.is_not(None), 1),
else_=0
)
).label('commented_stat'),
func.sum(case(
(aliased_reaction.kind == ReactionKind.AGREE, 1), (aliased_reaction.kind == ReactionKind.AGREE, 1),
(aliased_reaction.kind == ReactionKind.DISAGREE, -1), (aliased_reaction.kind == ReactionKind.DISAGREE, -1),
(aliased_reaction.kind == ReactionKind.PROOF, 1), (aliased_reaction.kind == ReactionKind.PROOF, 1),
@ -35,8 +30,10 @@ def add_reaction_stat_columns(q):
(aliased_reaction.kind == ReactionKind.REJECT, -1), (aliased_reaction.kind == ReactionKind.REJECT, -1),
(aliased_reaction.kind == ReactionKind.LIKE, 1), (aliased_reaction.kind == ReactionKind.LIKE, 1),
(aliased_reaction.kind == ReactionKind.DISLIKE, -1), (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
else_=0) else_=0,
).label('rating_stat')) )
).label('rating_stat'),
)
return q return q
@ -47,17 +44,19 @@ def reactions_follow(user_id, shout_id: int, auto=False):
shout = session.query(Shout).where(Shout.id == shout_id).one() shout = session.query(Shout).where(Shout.id == shout_id).one()
following = ( following = (
session.query(ShoutReactionsFollower).where(and_( session.query(ShoutReactionsFollower)
.where(
and_(
ShoutReactionsFollower.follower == user_id, ShoutReactionsFollower.follower == user_id,
ShoutReactionsFollower.shout == shout.id, ShoutReactionsFollower.shout == shout.id,
)).first() )
)
.first()
) )
if not following: if not following:
following = ShoutReactionsFollower.create( following = ShoutReactionsFollower.create(
follower=user_id, follower=user_id, shout=shout.id, auto=auto
shout=shout.id,
auto=auto
) )
session.add(following) session.add(following)
session.commit() session.commit()
@ -72,10 +71,14 @@ def reactions_unfollow(user_id: int, shout_id: int):
shout = session.query(Shout).where(Shout.id == shout_id).one() shout = session.query(Shout).where(Shout.id == shout_id).one()
following = ( following = (
session.query(ShoutReactionsFollower).where(and_( session.query(ShoutReactionsFollower)
.where(
and_(
ShoutReactionsFollower.follower == user_id, ShoutReactionsFollower.follower == user_id,
ShoutReactionsFollower.shout == shout.id ShoutReactionsFollower.shout == shout.id,
)).first() )
)
.first()
) )
if following: if following:
@ -89,16 +92,13 @@ def reactions_unfollow(user_id: int, shout_id: int):
def is_published_author(session, user_id): def is_published_author(session, user_id):
'''checks if user has at least one publication''' '''checks if user has at least one publication'''
return session.query( return (
Shout session.query(Shout)
).where( .where(Shout.authors.contains(user_id))
Shout.authors.contains(user_id) .filter(and_(Shout.publishedAt.is_not(None), Shout.deletedAt.is_(None)))
).filter( .count()
and_( > 0
Shout.publishedAt.is_not(None),
Shout.deletedAt.is_(None)
) )
).count() > 0
def check_to_publish(session, user_id, reaction): def check_to_publish(session, user_id, reaction):
@ -106,12 +106,16 @@ def check_to_publish(session, user_id, reaction):
if not reaction.replyTo and reaction.kind in [ if not reaction.replyTo and reaction.kind in [
ReactionKind.ACCEPT, ReactionKind.ACCEPT,
ReactionKind.LIKE, ReactionKind.LIKE,
ReactionKind.PROOF ReactionKind.PROOF,
]: ]:
if is_published_author(user_id): if is_published_author(user_id):
# now count how many approvers are voted already # now count how many approvers are voted already
approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all() approvers_reactions = (
approvers = [user_id, ] session.query(Reaction).where(Reaction.shout == reaction.shout).all()
)
approvers = [
user_id,
]
for ar in approvers_reactions: for ar in approvers_reactions:
a = ar.createdBy a = ar.createdBy
if is_published_author(session, a): if is_published_author(session, a):
@ -126,17 +130,13 @@ def check_to_hide(session, user_id, reaction):
if not reaction.replyTo and reaction.kind in [ if not reaction.replyTo and reaction.kind in [
ReactionKind.REJECT, ReactionKind.REJECT,
ReactionKind.DISLIKE, ReactionKind.DISLIKE,
ReactionKind.DISPROOF ReactionKind.DISPROOF,
]: ]:
# if is_published_author(user): # if is_published_author(user):
approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all() approvers_reactions = session.query(Reaction).where(Reaction.shout == reaction.shout).all()
rejects = 0 rejects = 0
for r in approvers_reactions: for r in approvers_reactions:
if r.kind in [ if r.kind in [ReactionKind.REJECT, ReactionKind.DISLIKE, ReactionKind.DISPROOF]:
ReactionKind.REJECT,
ReactionKind.DISLIKE,
ReactionKind.DISPROOF
]:
rejects += 1 rejects += 1
if len(approvers_reactions) / rejects < 5: if len(approvers_reactions) / rejects < 5:
return True return True
@ -168,31 +168,40 @@ async def create_reaction(_, info, reaction):
shout = session.query(Shout).where(Shout.id == reaction["shout"]).one() shout = session.query(Shout).where(Shout.id == reaction["shout"]).one()
author = session.query(User).where(User.id == auth.user_id).one() author = session.query(User).where(User.id == auth.user_id).one()
if reaction["kind"] in [ if reaction["kind"] in [ReactionKind.DISLIKE.name, ReactionKind.LIKE.name]:
ReactionKind.DISLIKE.name, existing_reaction = (
ReactionKind.LIKE.name session.query(Reaction)
]: .where(
existing_reaction = session.query(Reaction).where(
and_( and_(
Reaction.shout == reaction["shout"], Reaction.shout == reaction["shout"],
Reaction.createdBy == auth.user_id, Reaction.createdBy == auth.user_id,
Reaction.kind == reaction["kind"], Reaction.kind == reaction["kind"],
Reaction.replyTo == reaction.get("replyTo") Reaction.replyTo == reaction.get("replyTo"),
)
)
.first()
) )
).first()
if existing_reaction is not None: if existing_reaction is not None:
raise OperationNotAllowed("You can't vote twice") raise OperationNotAllowed("You can't vote twice")
opposite_reaction_kind = ReactionKind.DISLIKE if reaction["kind"] == ReactionKind.LIKE.name else ReactionKind.LIKE opposite_reaction_kind = (
opposite_reaction = session.query(Reaction).where( ReactionKind.DISLIKE
if reaction["kind"] == ReactionKind.LIKE.name
else ReactionKind.LIKE
)
opposite_reaction = (
session.query(Reaction)
.where(
and_( and_(
Reaction.shout == reaction["shout"], Reaction.shout == reaction["shout"],
Reaction.createdBy == auth.user_id, Reaction.createdBy == auth.user_id,
Reaction.kind == opposite_reaction_kind, Reaction.kind == opposite_reaction_kind,
Reaction.replyTo == reaction.get("replyTo") Reaction.replyTo == reaction.get("replyTo"),
)
)
.first()
) )
).first()
if opposite_reaction is not None: if opposite_reaction is not None:
session.delete(opposite_reaction) session.delete(opposite_reaction)
@ -235,11 +244,7 @@ async def create_reaction(_, info, reaction):
except Exception as e: except Exception as e:
print(f"[resolvers.reactions] error on reactions autofollowing: {e}") print(f"[resolvers.reactions] error on reactions autofollowing: {e}")
rdict['stat'] = { rdict['stat'] = {"commented": 0, "reacted": 0, "rating": 0}
"commented": 0,
"reacted": 0,
"rating": 0
}
return {"reaction": rdict} return {"reaction": rdict}
@ -269,11 +274,7 @@ async def update_reaction(_, info, id, reaction={}):
if reaction.get("range"): if reaction.get("range"):
r.range = reaction.get("range") r.range = reaction.get("range")
session.commit() session.commit()
r.stat = { r.stat = {"commented": commented_stat, "reacted": reacted_stat, "rating": rating_stat}
"commented": commented_stat,
"reacted": reacted_stat,
"rating": rating_stat
}
return {"reaction": r} return {"reaction": r}
@ -290,17 +291,12 @@ async def delete_reaction(_, info, id):
if r.createdBy != auth.user_id: if r.createdBy != auth.user_id:
return {"error": "access denied"} return {"error": "access denied"}
if r.kind in [ if r.kind in [ReactionKind.LIKE, ReactionKind.DISLIKE]:
ReactionKind.LIKE,
ReactionKind.DISLIKE
]:
session.delete(r) session.delete(r)
else: else:
r.deletedAt = datetime.now(tz=timezone.utc) r.deletedAt = datetime.now(tz=timezone.utc)
session.commit() session.commit()
return { return {"reaction": r}
"reaction": r
}
@query.field("loadReactionsBy") @query.field("loadReactionsBy")
@ -321,12 +317,10 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
:return: Reaction[] :return: Reaction[]
""" """
q = select( q = (
Reaction, User, Shout select(Reaction, User, Shout)
).join( .join(User, Reaction.createdBy == User.id)
User, Reaction.createdBy == User.id .join(Shout, Reaction.shout == Shout.id)
).join(
Shout, Reaction.shout == Shout.id
) )
if by.get("shout"): if by.get("shout"):
@ -354,11 +348,7 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
order_way = asc if by.get("sort", "").startswith("-") else desc order_way = asc if by.get("sort", "").startswith("-") else desc
order_field = by.get("sort", "").replace('-', '') or Reaction.createdAt order_field = by.get("sort", "").replace('-', '') or Reaction.createdAt
q = q.group_by( q = q.group_by(Reaction.id, User.id, Shout.id).order_by(order_way(order_field))
Reaction.id, User.id, Shout.id
).order_by(
order_way(order_field)
)
q = add_reaction_stat_columns(q) q = add_reaction_stat_columns(q)
@ -367,13 +357,15 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
reactions = [] reactions = []
with local_session() as session: with local_session() as session:
for [reaction, user, shout, reacted_stat, commented_stat, rating_stat] in session.execute(q): for [reaction, user, shout, reacted_stat, commented_stat, rating_stat] in session.execute(
q
):
reaction.createdBy = user reaction.createdBy = user
reaction.shout = shout reaction.shout = shout
reaction.stat = { reaction.stat = {
"rating": rating_stat, "rating": rating_stat,
"commented": commented_stat, "commented": commented_stat,
"reacted": reacted_stat "reacted": reacted_stat,
} }
reaction.kind = reaction.kind.name reaction.kind = reaction.kind.name

View File

@ -1,24 +1,25 @@
from sqlalchemy import and_, select, distinct, func from sqlalchemy import and_, distinct, func, select
from sqlalchemy.orm import aliased from sqlalchemy.orm import aliased
from auth.authenticate import login_required from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from orm.shout import ShoutTopic, ShoutAuthor
from orm.topic import Topic, TopicFollower
from orm import User from orm import User
from orm.shout import ShoutAuthor, ShoutTopic
from orm.topic import Topic, TopicFollower
def add_topic_stat_columns(q): def add_topic_stat_columns(q):
aliased_shout_author = aliased(ShoutAuthor) aliased_shout_author = aliased(ShoutAuthor)
aliased_topic_follower = aliased(TopicFollower) aliased_topic_follower = aliased(TopicFollower)
q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic).add_columns( q = (
func.count(distinct(ShoutTopic.shout)).label('shouts_stat') q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic)
).outerjoin(aliased_shout_author, ShoutTopic.shout == aliased_shout_author.shout).add_columns( .add_columns(func.count(distinct(ShoutTopic.shout)).label('shouts_stat'))
func.count(distinct(aliased_shout_author.user)).label('authors_stat') .outerjoin(aliased_shout_author, ShoutTopic.shout == aliased_shout_author.shout)
).outerjoin(aliased_topic_follower).add_columns( .add_columns(func.count(distinct(aliased_shout_author.user)).label('authors_stat'))
func.count(distinct(aliased_topic_follower.follower)).label('followers_stat') .outerjoin(aliased_topic_follower)
.add_columns(func.count(distinct(aliased_topic_follower.follower)).label('followers_stat'))
) )
q = q.group_by(Topic.id) q = q.group_by(Topic.id)
@ -28,11 +29,7 @@ def add_topic_stat_columns(q):
def add_stat(topic, stat_columns): def add_stat(topic, stat_columns):
[shouts_stat, authors_stat, followers_stat] = stat_columns [shouts_stat, authors_stat, followers_stat] = stat_columns
topic.stat = { topic.stat = {"shouts": shouts_stat, "authors": authors_stat, "followers": followers_stat}
"shouts": shouts_stat,
"authors": authors_stat,
"followers": followers_stat
}
return topic return topic
@ -133,12 +130,10 @@ def topic_unfollow(user_id, slug):
try: try:
with local_session() as session: with local_session() as session:
sub = ( sub = (
session.query(TopicFollower).join(Topic).filter( session.query(TopicFollower)
and_( .join(Topic)
TopicFollower.follower == user_id, .filter(and_(TopicFollower.follower == user_id, Topic.slug == slug))
Topic.slug == slug .first()
)
).first()
) )
if sub: if sub:
session.delete(sub) session.delete(sub)

View File

@ -1,8 +1,9 @@
import sys
import os import os
import sys
import uvicorn import uvicorn
from settings import PORT, DEV_SERVER_PID_FILE_NAME from settings import DEV_SERVER_PID_FILE_NAME, PORT
def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook): def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook):
@ -16,41 +17,30 @@ log_settings = {
'default': { 'default': {
'()': 'uvicorn.logging.DefaultFormatter', '()': 'uvicorn.logging.DefaultFormatter',
'fmt': '%(levelprefix)s %(message)s', 'fmt': '%(levelprefix)s %(message)s',
'use_colors': None 'use_colors': None,
}, },
'access': { 'access': {
'()': 'uvicorn.logging.AccessFormatter', '()': 'uvicorn.logging.AccessFormatter',
'fmt': '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s' 'fmt': '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s',
} },
}, },
'handlers': { 'handlers': {
'default': { 'default': {
'formatter': 'default', 'formatter': 'default',
'class': 'logging.StreamHandler', 'class': 'logging.StreamHandler',
'stream': 'ext://sys.stderr' 'stream': 'ext://sys.stderr',
}, },
'access': { 'access': {
'formatter': 'access', 'formatter': 'access',
'class': 'logging.StreamHandler', 'class': 'logging.StreamHandler',
'stream': 'ext://sys.stdout' 'stream': 'ext://sys.stdout',
} },
}, },
'loggers': { 'loggers': {
'uvicorn': { 'uvicorn': {'handlers': ['default'], 'level': 'INFO'},
'handlers': ['default'], 'uvicorn.error': {'level': 'INFO', 'handlers': ['default'], 'propagate': True},
'level': 'INFO' 'uvicorn.access': {'handlers': ['access'], 'level': 'INFO', 'propagate': False},
}, },
'uvicorn.error': {
'level': 'INFO',
'handlers': ['default'],
'propagate': True
},
'uvicorn.access': {
'handlers': ['access'],
'level': 'INFO',
'propagate': False
}
}
} }
local_headers = [ local_headers = [
@ -86,24 +76,20 @@ if __name__ == "__main__":
# log_config=log_settings, # log_config=log_settings,
log_level=None, log_level=None,
access_log=True, access_log=True,
reload=want_reload reload=want_reload,
) # , ssl_keyfile="discours.key", ssl_certfile="discours.crt") ) # , ssl_keyfile="discours.key", ssl_certfile="discours.crt")
elif x == "migrate": elif x == "migrate":
from migration import process from migration import process
print("MODE: MIGRATE") print("MODE: MIGRATE")
process() process()
elif x == "bson": elif x == "bson":
from migration.bson2json import json_tables from migration.bson2json import json_tables
print("MODE: BSON") print("MODE: BSON")
json_tables() json_tables()
else: else:
sys.excepthook = exception_handler sys.excepthook = exception_handler
uvicorn.run( uvicorn.run("main:app", host="0.0.0.0", port=PORT, proxy_headers=True, server_header=True)
"main:app",
host="0.0.0.0",
port=PORT,
proxy_headers=True,
server_header=True
)

View File

@ -18,12 +18,7 @@ class Following:
class FollowingManager: class FollowingManager:
lock = asyncio.Lock() lock = asyncio.Lock()
data = { data = {'author': [], 'topic': [], 'shout': [], 'chat': []}
'author': [],
'topic': [],
'shout': [],
'chat': []
}
@staticmethod @staticmethod
async def register(kind, uid): async def register(kind, uid):

View File

@ -1,6 +1,6 @@
from base.orm import local_session
from services.search import SearchService from services.search import SearchService
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
from base.orm import local_session
async def storages_init(): async def storages_init():

View File

@ -5,26 +5,18 @@ from datetime import datetime, timezone
from sqlalchemy import and_ from sqlalchemy import and_
from base.orm import local_session from base.orm import local_session
from orm import Reaction, Shout, Notification, User from orm import Notification, Reaction, Shout, User
from orm.notification import NotificationType from orm.notification import NotificationType
from orm.reaction import ReactionKind from orm.reaction import ReactionKind
from services.notifications.sse import connection_manager from services.notifications.sse import connection_manager
def shout_to_shout_data(shout): def shout_to_shout_data(shout):
return { return {"title": shout.title, "slug": shout.slug}
"title": shout.title,
"slug": shout.slug
}
def user_to_user_data(user): def user_to_user_data(user):
return { return {"id": user.id, "name": user.name, "slug": user.slug, "userpic": user.userpic}
"id": user.id,
"name": user.name,
"slug": user.slug,
"userpic": user.userpic
}
def update_prev_notification(notification, user, reaction): def update_prev_notification(notification, user, reaction):
@ -57,34 +49,45 @@ class NewReactionNotificator:
if reaction.kind == ReactionKind.COMMENT: if reaction.kind == ReactionKind.COMMENT:
parent_reaction = None parent_reaction = None
if reaction.replyTo: if reaction.replyTo:
parent_reaction = session.query(Reaction).where(Reaction.id == reaction.replyTo).one() parent_reaction = (
session.query(Reaction).where(Reaction.id == reaction.replyTo).one()
)
if parent_reaction.createdBy != reaction.createdBy: if parent_reaction.createdBy != reaction.createdBy:
prev_new_reply_notification = session.query(Notification).where( prev_new_reply_notification = (
session.query(Notification)
.where(
and_( and_(
Notification.user == shout.createdBy, Notification.user == shout.createdBy,
Notification.type == NotificationType.NEW_REPLY, Notification.type == NotificationType.NEW_REPLY,
Notification.shout == shout.id, Notification.shout == shout.id,
Notification.reaction == parent_reaction.id, Notification.reaction == parent_reaction.id,
Notification.seen == False Notification.seen == False,
)
)
.first()
) )
).first()
if prev_new_reply_notification: if prev_new_reply_notification:
update_prev_notification(prev_new_reply_notification, user, reaction) update_prev_notification(prev_new_reply_notification, user, reaction)
else: else:
reply_notification_data = json.dumps({ reply_notification_data = json.dumps(
{
"shout": shout_to_shout_data(shout), "shout": shout_to_shout_data(shout),
"users": [user_to_user_data(user)], "users": [user_to_user_data(user)],
"reactionIds": [reaction.id] "reactionIds": [reaction.id],
}, ensure_ascii=False) },
ensure_ascii=False,
)
reply_notification = Notification.create(**{ reply_notification = Notification.create(
**{
"user": parent_reaction.createdBy, "user": parent_reaction.createdBy,
"type": NotificationType.NEW_REPLY, "type": NotificationType.NEW_REPLY,
"shout": shout.id, "shout": shout.id,
"reaction": parent_reaction.id, "reaction": parent_reaction.id,
"data": reply_notification_data "data": reply_notification_data,
}) }
)
session.add(reply_notification) session.add(reply_notification)
@ -93,30 +96,39 @@ class NewReactionNotificator:
if reaction.createdBy != shout.createdBy and ( if reaction.createdBy != shout.createdBy and (
parent_reaction is None or parent_reaction.createdBy != shout.createdBy parent_reaction is None or parent_reaction.createdBy != shout.createdBy
): ):
prev_new_comment_notification = session.query(Notification).where( prev_new_comment_notification = (
session.query(Notification)
.where(
and_( and_(
Notification.user == shout.createdBy, Notification.user == shout.createdBy,
Notification.type == NotificationType.NEW_COMMENT, Notification.type == NotificationType.NEW_COMMENT,
Notification.shout == shout.id, Notification.shout == shout.id,
Notification.seen == False Notification.seen == False,
)
)
.first()
) )
).first()
if prev_new_comment_notification: if prev_new_comment_notification:
update_prev_notification(prev_new_comment_notification, user, reaction) update_prev_notification(prev_new_comment_notification, user, reaction)
else: else:
notification_data_string = json.dumps({ notification_data_string = json.dumps(
{
"shout": shout_to_shout_data(shout), "shout": shout_to_shout_data(shout),
"users": [user_to_user_data(user)], "users": [user_to_user_data(user)],
"reactionIds": [reaction.id] "reactionIds": [reaction.id],
}, ensure_ascii=False) },
ensure_ascii=False,
)
author_notification = Notification.create(**{ author_notification = Notification.create(
**{
"user": shout.createdBy, "user": shout.createdBy,
"type": NotificationType.NEW_COMMENT, "type": NotificationType.NEW_COMMENT,
"shout": shout.id, "shout": shout.id,
"data": notification_data_string "data": notification_data_string,
}) }
)
session.add(author_notification) session.add(author_notification)

View File

@ -1,8 +1,8 @@
import asyncio
import json import json
from sse_starlette.sse import EventSourceResponse from sse_starlette.sse import EventSourceResponse
from starlette.requests import Request from starlette.requests import Request
import asyncio
class ConnectionManager: class ConnectionManager:
@ -28,9 +28,7 @@ class ConnectionManager:
return return
for connection in self.connections_by_user_id[user_id]: for connection in self.connections_by_user_id[user_id]:
data = { data = {"type": "newNotifications"}
"type": "newNotifications"
}
data_string = json.dumps(data, ensure_ascii=False) data_string = json.dumps(data, ensure_ascii=False)
await connection.put(data_string) await connection.put(data_string)

View File

@ -1,5 +1,6 @@
import asyncio import asyncio
import json import json
from base.redis import redis from base.redis import redis
from orm.shout import Shout from orm.shout import Shout
from resolvers.zine.load import load_shouts_by from resolvers.zine.load import load_shouts_by
@ -20,12 +21,7 @@ class SearchService:
cached = await redis.execute("GET", text) cached = await redis.execute("GET", text)
if not cached: if not cached:
async with SearchService.lock: async with SearchService.lock:
options = { options = {"title": text, "body": text, "limit": limit, "offset": offset}
"title": text,
"body": text,
"limit": limit,
"offset": offset
}
payload = await load_shouts_by(None, None, options) payload = await load_shouts_by(None, None, options)
await redis.execute("SET", text, json.dumps(payload)) await redis.execute("SET", text, json.dumps(payload))
return payload return payload

View File

@ -1,6 +1,6 @@
import asyncio import asyncio
import time import time
from datetime import timedelta, timezone, datetime from datetime import datetime, timedelta, timezone
from os import environ, path from os import environ, path
from ssl import create_default_context from ssl import create_default_context
@ -9,10 +9,11 @@ from gql.transport.aiohttp import AIOHTTPTransport
from sqlalchemy import func from sqlalchemy import func
from base.orm import local_session from base.orm import local_session
from orm import User, Topic from orm import Topic, User
from orm.shout import ShoutTopic, Shout from orm.shout import Shout, ShoutTopic
load_facts = gql(""" load_facts = gql(
"""
query getDomains { query getDomains {
domains { domains {
id id
@ -25,9 +26,11 @@ query getDomains {
} }
} }
} }
""") """
)
load_pages = gql(""" load_pages = gql(
"""
query getDomains { query getDomains {
domains { domains {
title title
@ -41,7 +44,8 @@ query getDomains {
} }
} }
} }
""") """
)
schema_str = open(path.dirname(__file__) + '/ackee.graphql').read() schema_str = open(path.dirname(__file__) + '/ackee.graphql').read()
token = environ.get("ACKEE_TOKEN", "") token = environ.get("ACKEE_TOKEN", "")
@ -50,10 +54,8 @@ def create_client(headers=None, schema=None):
return Client( return Client(
schema=schema, schema=schema,
transport=AIOHTTPTransport( transport=AIOHTTPTransport(
url="https://ackee.discours.io/api", url="https://ackee.discours.io/api", ssl=create_default_context(), headers=headers
ssl=create_default_context(), ),
headers=headers
)
) )
@ -75,9 +77,9 @@ class ViewedStorage:
self = ViewedStorage self = ViewedStorage
async with self.lock: async with self.lock:
if token: if token:
self.client = create_client({ self.client = create_client(
"Authorization": "Bearer %s" % str(token) {"Authorization": "Bearer %s" % str(token)}, schema=schema_str
}, schema=schema_str) )
print("[stat.viewed] * authorized permanentely by ackee.discours.io: %s" % token) print("[stat.viewed] * authorized permanentely by ackee.discours.io: %s" % token)
else: else:
print("[stat.viewed] * please set ACKEE_TOKEN") print("[stat.viewed] * please set ACKEE_TOKEN")
@ -148,9 +150,13 @@ class ViewedStorage:
def update_topics(session, shout_slug): def update_topics(session, shout_slug):
"""updates topics counters by shout slug""" """updates topics counters by shout slug"""
self = ViewedStorage self = ViewedStorage
for [shout_topic, topic] in session.query(ShoutTopic, Topic).join(Topic).join(Shout).where( for [shout_topic, topic] in (
Shout.slug == shout_slug session.query(ShoutTopic, Topic)
).all(): .join(Topic)
.join(Shout)
.where(Shout.slug == shout_slug)
.all()
):
if not self.by_topics.get(topic.slug): if not self.by_topics.get(topic.slug):
self.by_topics[topic.slug] = {} self.by_topics[topic.slug] = {}
self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug] self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug]
@ -205,9 +211,10 @@ class ViewedStorage:
if failed == 0: if failed == 0:
when = datetime.now(timezone.utc) + timedelta(seconds=self.period) when = datetime.now(timezone.utc) + timedelta(seconds=self.period)
t = format(when.astimezone().isoformat()) t = format(when.astimezone().isoformat())
print("[stat.viewed] ⎩ next update: %s" % ( print(
t.split("T")[0] + " " + t.split("T")[1].split(".")[0] "[stat.viewed] ⎩ next update: %s"
)) % (t.split("T")[0] + " " + t.split("T")[1].split(".")[0])
)
await asyncio.sleep(self.period) await asyncio.sleep(self.period)
else: else:
await asyncio.sleep(10) await asyncio.sleep(10)

View File

@ -3,8 +3,9 @@ from os import environ
PORT = 8080 PORT = 8080
DB_URL = ( DB_URL = (
environ.get("DATABASE_URL") or environ.get("DB_URL") or environ.get("DATABASE_URL")
"postgresql://postgres@localhost:5432/discoursio" or environ.get("DB_URL")
or "postgresql://postgres@localhost:5432/discoursio"
) )
JWT_ALGORITHM = "HS256" JWT_ALGORITHM = "HS256"
JWT_SECRET_KEY = environ.get("JWT_SECRET_KEY") or "8f1bd7696ffb482d8486dfbc6e7d16dd-secret-key" JWT_SECRET_KEY = environ.get("JWT_SECRET_KEY") or "8f1bd7696ffb482d8486dfbc6e7d16dd-secret-key"

View File

@ -1,4 +1,5 @@
from typing import Optional, Text from typing import Optional, Text
from pydantic import BaseModel from pydantic import BaseModel

View File

@ -1,4 +1,5 @@
from typing import Optional, Text, List from typing import List, Optional, Text
from pydantic import BaseModel from pydantic import BaseModel