discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

migration, auth, refactoring, formatting

Browse Source
This commit is contained in:
tonyrewin 2022-09-17 21:12:14 +03:00
parent 6b4c00d9e7
commit 3136eecd7e
68 changed files with 968 additions and 930 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.editorconfig
View File

@ -1,8 +1,7 @@
root = true
[*]
indent_style = tabs
indent_size = 2
indent_size = 4
end_of_line = lf
charset = utf-8
trim_trailing_whitespace=true

2
.flake8
View File

@ -1,5 +1,5 @@
[flake8]
ignore = E203,W504,W191
ignore = E203,W504,W191,W503
exclude = .git,__pycache__,orm/rbac.py
max-complexity = 10
max-line-length = 108

3
auth/__init__.py
View File

@ -1,3 +0,0 @@
from auth.email import load_email_templates
load_email_templates()

86
auth/authenticate.py
View File

@ -1,21 +1,20 @@
from functools import wraps
from typing import Optional, Tuple
from datetime import datetime, timedelta
from graphql import GraphQLResolveInfo
from graphql.type import GraphQLResolveInfo
from jwt import DecodeError, ExpiredSignatureError
from starlette.authentication import AuthenticationBackend
from starlette.requests import HTTPConnection
from auth.credentials import AuthCredentials, AuthUser
from auth.jwtcodec import JWTCodec
from auth.authorize import Authorize, TokenStorage
from auth.tokenstorage import TokenStorage
from base.exceptions import InvalidToken
from orm.user import User
from services.auth.users import UserStorage
from base.orm import local_session
from settings import JWT_AUTH_HEADER, EMAIL_TOKEN_LIFE_SPAN
from settings import SESSION_TOKEN_HEADER
class _Authenticate:
class SessionToken:
@classmethod
async def verify(cls, token: str):
"""
@ -32,33 +31,30 @@ class _Authenticate:
payload = JWTCodec.decode(token)
except ExpiredSignatureError:
payload = JWTCodec.decode(token, verify_exp=False)
if not await cls.exists(payload.user_id, token):
raise InvalidToken("Login expired, please login again")
if payload.device == "mobile": # noqa
"we cat set mobile token to be valid forever"
return payload
if not await cls.get(payload.user_id, token):
raise InvalidToken("Session token has expired, please try again")
except DecodeError as e:
raise InvalidToken("token format error") from e
else:
if not await cls.exists(payload.user_id, token):
raise InvalidToken("Login expired, please login again")
if not await cls.get(payload.user_id, token):
raise InvalidToken("Session token has expired, please login again")
return payload
@classmethod
async def exists(cls, user_id, token):
return await TokenStorage.exist(f"{user_id}-{token}")
async def get(cls, uid, token):
return await TokenStorage.get(f"{uid}-{token}")
class JWTAuthenticate(AuthenticationBackend):
async def authenticate(
self, request: HTTPConnection
) -> Optional[Tuple[AuthCredentials, AuthUser]]:
if JWT_AUTH_HEADER not in request.headers:
if SESSION_TOKEN_HEADER not in request.headers:
return AuthCredentials(scopes=[]), AuthUser(user_id=None)
token = request.headers[JWT_AUTH_HEADER]
token = request.headers[SESSION_TOKEN_HEADER]
try:
payload = await _Authenticate.verify(token)
payload = await SessionToken.verify(token)
except Exception as exc:
return AuthCredentials(scopes=[], error_message=str(exc)), AuthUser(
user_id=None
@ -67,9 +63,6 @@ class JWTAuthenticate(AuthenticationBackend):
if payload is None:
return AuthCredentials(scopes=[]), AuthUser(user_id=None)
if payload.device not in ("pc", "mobile"):
return AuthCredentials(scopes=[]), AuthUser(user_id=None)
user = await UserStorage.get_user(payload.user_id)
if not user:
return AuthCredentials(scopes=[]), AuthUser(user_id=None)
@ -81,55 +74,6 @@ class JWTAuthenticate(AuthenticationBackend):
)
class EmailAuthenticate:
@staticmethod
async def get_email_token(user):
token = await Authorize.authorize(
user, device="email", life_span=EMAIL_TOKEN_LIFE_SPAN
)
return token
@staticmethod
async def authenticate(token):
payload = await _Authenticate.verify(token)
if payload is None:
raise InvalidToken("invalid token")
if payload.device != "email":
raise InvalidToken("invalid token")
with local_session() as session:
user = session.query(User).filter_by(id=payload.user_id).first()
if not user:
raise Exception("user not exist")
if not user.emailConfirmed:
user.emailConfirmed = True
session.commit()
auth_token = await Authorize.authorize(user)
return (auth_token, user)
class ResetPassword:
@staticmethod
async def get_reset_token(user):
exp = datetime.utcnow() + timedelta(seconds=EMAIL_TOKEN_LIFE_SPAN)
token = JWTCodec.encode(user, exp=exp, device="pc")
await TokenStorage.save(f"{user.id}-reset-{token}", EMAIL_TOKEN_LIFE_SPAN, True)
return token
@staticmethod
async def verify(token):
try:
payload = JWTCodec.decode(token)
except ExpiredSignatureError:
raise InvalidToken("Login expired, please login again")
except DecodeError as e:
raise InvalidToken("token format error") from e
else:
if not await TokenStorage.exist(f"{payload.user_id}-reset-{token}"):
raise InvalidToken("Login expired, please login again")
return payload.user_id
def login_required(func):
@wraps(func)
async def wrap(parent, info: GraphQLResolveInfo, *args, **kwargs):

45
auth/authorize.py
View File

@ -1,45 +0,0 @@
from datetime import datetime, timedelta
from auth.jwtcodec import JWTCodec
from base.redis import redis
from settings import JWT_LIFE_SPAN
from auth.validations import User
class TokenStorage:
@staticmethod
async def save(token_key, life_span, auto_delete=True):
await redis.execute("SET", token_key, "True")
if auto_delete:
expire_at = (datetime.now() + timedelta(seconds=life_span)).timestamp()
await redis.execute("EXPIREAT", token_key, int(expire_at))
@staticmethod
async def exist(token_key):
return await redis.execute("GET", token_key)
class Authorize:
@staticmethod
async def authorize(
user: User, device: str = "pc", life_span=JWT_LIFE_SPAN, auto_delete=True
) -> str:
exp = datetime.utcnow() + timedelta(seconds=life_span)
token = JWTCodec.encode(user, exp=exp, device=device)
await TokenStorage.save(f"{user.id}-{token}", life_span, auto_delete)
return token
@staticmethod
async def revoke(token: str) -> bool:
try:
payload = JWTCodec.decode(token)
except: # noqa
pass
else:
await redis.execute("DEL", f"{payload.user_id}-{token}")
return True
@staticmethod
async def revoke_all(user: User):
tokens = await redis.execute("KEYS", f"{user.id}-*")
await redis.execute("DEL", *tokens)

6
auth/credentials.py
View File

@ -1,6 +1,9 @@
from typing import List, Optional, Text
from pydantic import BaseModel
from base.exceptions import OperationNotAllowed
class Permission(BaseModel):
name: Text
@ -17,7 +20,8 @@ class AuthCredentials(BaseModel):
return True
async def permissions(self) -> List[Permission]:
assert self.user_id is not None, "Please login first"
if self.user_id is not None:
raise OperationNotAllowed("Please login first")
return NotImplemented()

112
auth/email.py
View File

@ -1,84 +1,28 @@
import requests
from starlette.responses import RedirectResponse
from auth.authenticate import EmailAuthenticate, ResetPassword
from base.orm import local_session
from settings import (
BACKEND_URL,
MAILGUN_API_KEY,
MAILGUN_DOMAIN,
RESET_PWD_URL,
CONFIRM_EMAIL_URL,
ERROR_URL_ON_FRONTEND,
)
MAILGUN_API_URL = "https://api.mailgun.net/v3/%s/messages" % (MAILGUN_DOMAIN)
MAILGUN_FROM = "discours.io <noreply@%s>" % (MAILGUN_DOMAIN)
AUTH_URL = "%s/email_authorize" % (BACKEND_URL)
email_templates = {"confirm_email": "", "auth_email": "", "reset_password_email": ""}
def load_email_templates():
for name in email_templates:
filename = "auth/templates/%s.tmpl" % name
with open(filename) as f:
email_templates[name] = f.read()
print("[auth.email] templates loaded")
async def send_confirm_email(user):
text = email_templates["confirm_email"]
token = await EmailAuthenticate.get_email_token(user)
await send_email(user, AUTH_URL, text, token)
async def send_auth_email(user):
text = email_templates["auth_email"]
token = await EmailAuthenticate.get_email_token(user)
await send_email(user, AUTH_URL, text, token)
async def send_reset_password_email(user):
text = email_templates["reset_password_email"]
token = await ResetPassword.get_reset_token(user)
await send_email(user, RESET_PWD_URL, text, token)
async def send_email(user, url, text, token):
to = "%s <%s>" % (user.username, user.email)
url_with_token = "%s?token=%s" % (url, token)
text = text % (url_with_token)
response = requests.post(
MAILGUN_API_URL,
auth=("api", MAILGUN_API_KEY),
data={
"from": MAILGUN_FROM,
"to": to,
"subject": "authorize log in",
"html": text,
},
)
response.raise_for_status()
async def email_authorize(request):
token = request.query_params.get("token")
if not token:
url_with_error = "%s?error=%s" % (ERROR_URL_ON_FRONTEND, "INVALID_TOKEN")
return RedirectResponse(url=url_with_error)
try:
auth_token, user = await EmailAuthenticate.authenticate(token)
except:
url_with_error = "%s?error=%s" % (ERROR_URL_ON_FRONTEND, "INVALID_TOKEN")
return RedirectResponse(url=url_with_error)
if not user.emailConfirmed:
with local_session() as session:
user.emailConfirmed = True
session.commit()
response = RedirectResponse(url=CONFIRM_EMAIL_URL)
response.set_cookie("token", auth_token)
return response
import requests
from settings import BACKEND_URL, MAILGUN_API_KEY, MAILGUN_DOMAIN
MAILGUN_API_URL = "https://api.mailgun.net/v3/%s/messages" % MAILGUN_DOMAIN
MAILGUN_FROM = "discours.io <noreply@%s>" % MAILGUN_DOMAIN
async def send_auth_email(user, token):
text = """<html><body>
Follow the <a href='%s'>link</link> to authorize
</body></html>
"""
url = "%s/confirm_email" % BACKEND_URL
to = "%s <%s>" % (user.username, user.email)
url_with_token = "%s?token=%s" % (url, token)
text = text % url_with_token
response = requests.post(
MAILGUN_API_URL,
auth=("api", MAILGUN_API_KEY),
data={
"from": MAILGUN_FROM,
"to": to,
"subject": "Confirm email",
"html": text,
},
)
response.raise_for_status()

63
auth/identity.py
View File

@ -1,16 +1,30 @@
from auth.password import Password
from base.exceptions import InvalidPassword
from orm import User as OrmUser
from base.orm import local_session
from auth.validations import User
from jwt import DecodeError, ExpiredSignatureError
from sqlalchemy import or_
from auth.jwtcodec import JWTCodec
from auth.tokenstorage import TokenStorage
from validations.auth import AuthInput
from base.exceptions import InvalidPassword
from base.exceptions import InvalidToken
from base.orm import local_session
from orm import User
from passlib.hash import bcrypt
class Password:
@staticmethod
def encode(password: str) -> str:
return bcrypt.hash(password)
@staticmethod
def verify(password: str, other: str) -> bool:
return bcrypt.verify(password, other)
class Identity:
@staticmethod
def identity(orm_user: OrmUser, password: str) -> User:
user = User(**orm_user.dict())
def password(orm_user: User, password: str) -> User:
user = AuthInput(**orm_user.dict())
if not user.password:
raise InvalidPassword("User password is empty")
if not Password.verify(password, user.password):
@ -18,22 +32,37 @@ class Identity:
return user
@staticmethod
def identity_oauth(input) -> User:
def oauth(inp: AuthInput) -> User:
with local_session() as session:
user = (
session.query(OrmUser)
.filter(
or_(
OrmUser.oauth == input["oauth"], OrmUser.email == input["email"]
)
)
session.query(User)
.filter(or_(User.oauth == inp["oauth"], User.email == inp["email"]))
.first()
)
if not user:
user = OrmUser.create(**input)
user = User.create(**inp)
if not user.oauth:
user.oauth = input["oauth"]
user.oauth = inp["oauth"]
session.commit()
user = User(**user.dict())
return user
@staticmethod
async def onetime(token: str) -> User:
try:
payload = JWTCodec.decode(token)
if not await TokenStorage.exist(f"{payload.user_id}-{token}"):
raise InvalidToken("Login token has expired, please login again")
except ExpiredSignatureError:
raise InvalidToken("Login token has expired, please try again")
except DecodeError as e:
raise InvalidToken("token format error") from e
with local_session() as session:
user = session.query(User).filter_by(id=payload.user_id).first()
if not user:
raise Exception("user not exist")
if not user.emailConfirmed:
user.emailConfirmed = True
session.commit()
return user

13
auth/jwtcodec.py
View File

@ -1,26 +1,29 @@
from datetime import datetime
import jwt
from validations.auth import TokenPayload, AuthInput
from settings import JWT_ALGORITHM, JWT_SECRET_KEY
from auth.validations import PayLoad, User
class JWTCodec:
@staticmethod
def encode(user: User, exp: datetime, device: str = "pc") -> str:
def encode(user: AuthInput, exp: datetime) -> str:
payload = {
"user_id": user.id,
"device": device,
# "user_email": user.email, # less secure
# "device": device, # no use cases
"exp": exp,
"iat": datetime.utcnow(),
}
return jwt.encode(payload, JWT_SECRET_KEY, JWT_ALGORITHM)
@staticmethod
def decode(token: str, verify_exp: bool = True) -> PayLoad:
def decode(token: str, verify_exp: bool = True) -> TokenPayload:
payload = jwt.decode(
token,
key=JWT_SECRET_KEY,
options={"verify_exp": verify_exp},
algorithms=[JWT_ALGORITHM],
)
return PayLoad(**payload)
return TokenPayload(**payload)

180
auth/oauth.py
View File

@ -1,91 +1,89 @@
from authlib.integrations.starlette_client import OAuth
from starlette.responses import RedirectResponse
from auth.authorize import Authorize
from auth.identity import Identity
from settings import OAUTH_CLIENTS, BACKEND_URL, OAUTH_CALLBACK_URL
oauth = OAuth()
oauth.register(
name="facebook",
client_id=OAUTH_CLIENTS["FACEBOOK"]["id"],
client_secret=OAUTH_CLIENTS["FACEBOOK"]["key"],
access_token_url="https://graph.facebook.com/v11.0/oauth/access_token",
access_token_params=None,
authorize_url="https://www.facebook.com/v11.0/dialog/oauth",
authorize_params=None,
api_base_url="https://graph.facebook.com/",
client_kwargs={"scope": "public_profile email"},
)
oauth.register(
name="github",
client_id=OAUTH_CLIENTS["GITHUB"]["id"],
client_secret=OAUTH_CLIENTS["GITHUB"]["key"],
access_token_url="https://github.com/login/oauth/access_token",
access_token_params=None,
authorize_url="https://github.com/login/oauth/authorize",
authorize_params=None,
api_base_url="https://api.github.com/",
client_kwargs={"scope": "user:email"},
)
oauth.register(
name="google",
client_id=OAUTH_CLIENTS["GOOGLE"]["id"],
client_secret=OAUTH_CLIENTS["GOOGLE"]["key"],
server_metadata_url="https://accounts.google.com/.well-known/openid-configuration",
client_kwargs={"scope": "openid email profile"},
)
async def google_profile(client, request, token):
profile = await client.parse_id_token(request, token)
profile["id"] = profile["sub"]
return profile
async def facebook_profile(client, request, token):
profile = await client.get("me?fields=name,id,email", token=token)
return profile.json()
async def github_profile(client, request, token):
profile = await client.get("user", token=token)
return profile.json()
profile_callbacks = {
"google": google_profile,
"facebook": facebook_profile,
"github": github_profile,
}
async def oauth_login(request):
provider = request.path_params["provider"]
request.session["provider"] = provider
client = oauth.create_client(provider)
redirect_uri = "%s/%s" % (BACKEND_URL, "oauth_authorize")
return await client.authorize_redirect(request, redirect_uri)
async def oauth_authorize(request):
provider = request.session["provider"]
client = oauth.create_client(provider)
token = await client.authorize_access_token(request)
get_profile = profile_callbacks[provider]
profile = await get_profile(client, request, token)
user_oauth_info = "%s:%s" % (provider, profile["id"])
user_input = {
"oauth": user_oauth_info,
"email": profile["email"],
"username": profile["name"],
}
user = Identity.identity_oauth(user_input)
token = await Authorize.authorize(user, device="pc")
response = RedirectResponse(url=OAUTH_CALLBACK_URL)
response.set_cookie("token", token)
return response
from authlib.integrations.starlette_client import OAuth
from starlette.responses import RedirectResponse
from auth.identity import Identity
from auth.tokenstorage import TokenStorage
from settings import OAUTH_CLIENTS, BACKEND_URL, OAUTH_CALLBACK_URL
oauth = OAuth()
oauth.register(
name="facebook",
client_id=OAUTH_CLIENTS["FACEBOOK"]["id"],
client_secret=OAUTH_CLIENTS["FACEBOOK"]["key"],
access_token_url="https://graph.facebook.com/v11.0/oauth/access_token",
access_token_params=None,
authorize_url="https://www.facebook.com/v11.0/dialog/oauth",
authorize_params=None,
api_base_url="https://graph.facebook.com/",
client_kwargs={"scope": "public_profile email"},
)
oauth.register(
name="github",
client_id=OAUTH_CLIENTS["GITHUB"]["id"],
client_secret=OAUTH_CLIENTS["GITHUB"]["key"],
access_token_url="https://github.com/login/oauth/access_token",
access_token_params=None,
authorize_url="https://github.com/login/oauth/authorize",
authorize_params=None,
api_base_url="https://api.github.com/",
client_kwargs={"scope": "user:email"},
)
oauth.register(
name="google",
client_id=OAUTH_CLIENTS["GOOGLE"]["id"],
client_secret=OAUTH_CLIENTS["GOOGLE"]["key"],
server_metadata_url="https://accounts.google.com/.well-known/openid-configuration",
client_kwargs={"scope": "openid email profile"},
)
async def google_profile(client, request, token):
profile = await client.parse_id_token(request, token)
profile["id"] = profile["sub"]
return profile
async def facebook_profile(client, request, token):
profile = await client.get("me?fields=name,id,email", token=token)
return profile.json()
async def github_profile(client, request, token):
profile = await client.get("user", token=token)
return profile.json()
profile_callbacks = {
"google": google_profile,
"facebook": facebook_profile,
"github": github_profile,
}
async def oauth_login(request):
provider = request.path_params["provider"]
request.session["provider"] = provider
client = oauth.create_client(provider)
redirect_uri = "%s/%s" % (BACKEND_URL, "oauth_authorize")
return await client.authorize_redirect(request, redirect_uri)
async def oauth_authorize(request):
provider = request.session["provider"]
client = oauth.create_client(provider)
token = await client.authorize_access_token(request)
get_profile = profile_callbacks[provider]
profile = await get_profile(client, request, token)
user_oauth_info = "%s:%s" % (provider, profile["id"])
user_input = {
"oauth": user_oauth_info,
"email": profile["email"],
"username": profile["name"],
}
user = Identity.oauth(user_input)
session_token = await TokenStorage.create_session(user)
response = RedirectResponse(url=OAUTH_CALLBACK_URL)
response.set_cookie("token", session_token)
return response

11
auth/password.py
View File

@ -1,11 +0,0 @@
from passlib.hash import bcrypt
class Password:
@staticmethod
def encode(password: str) -> str:
return bcrypt.hash(password)
@staticmethod
def verify(password: str, other: str) -> bool:
return bcrypt.verify(password, other)

1
auth/templates/auth_email.tmpl
View File

@ -1 +0,0 @@
<html><body>To enter the site follow the <a href='%s'>link</link></body></html>

1
auth/templates/confirm_email.tmpl
View File

@ -1 +0,0 @@
<html><body>To confirm registration follow the <a href='%s'>link</link></body></html>

1
auth/templates/reset_password_email.tmpl
View File

@ -1 +0,0 @@
<html><body>To reset password follow the <a href='%s'>link</link></body></html>

50
auth/tokenstorage.py Normal file
View File

@ -0,0 +1,50 @@
from datetime import datetime, timedelta
from auth.jwtcodec import JWTCodec
from validations.auth import AuthInput
from base.redis import redis
from settings import SESSION_TOKEN_LIFE_SPAN, ONETIME_TOKEN_LIFE_SPAN
async def save(token_key, life_span, auto_delete=True):
await redis.execute("SET", token_key, "True")
if auto_delete:
expire_at = (datetime.now() + timedelta(seconds=life_span)).timestamp()
await redis.execute("EXPIREAT", token_key, int(expire_at))
class TokenStorage:
@staticmethod
async def get(token_key):
return await redis.execute("GET", token_key)
@staticmethod
async def create_onetime(user: AuthInput) -> str:
life_span = ONETIME_TOKEN_LIFE_SPAN
exp = datetime.utcnow() + timedelta(seconds=life_span)
one_time_token = JWTCodec.encode(user, exp=exp)
await save(f"{user.id}-{one_time_token}", life_span)
return one_time_token
@staticmethod
async def create_session(user: AuthInput) -> str:
life_span = SESSION_TOKEN_LIFE_SPAN
exp = datetime.utcnow() + timedelta(seconds=life_span)
session_token = JWTCodec.encode(user, exp=exp)
await save(f"{user.id}-{session_token}", life_span)
return session_token
@staticmethod
async def revoke(token: str) -> bool:
try:
payload = JWTCodec.decode(token)
except: # noqa
pass
else:
await redis.execute("DEL", f"{payload.user_id}-{token}")
return True
@staticmethod
async def revoke_all(user: AuthInput):
tokens = await redis.execute("KEYS", f"{user.id}-*")
await redis.execute("DEL", *tokens)

27
auth/validations.py
View File

@ -1,27 +0,0 @@
from datetime import datetime
from typing import Optional, Text
from pydantic import BaseModel
class User(BaseModel):
id: Optional[int]
# age: Optional[int]
username: Optional[Text]
# phone: Optional[Text]
password: Optional[Text]
class PayLoad(BaseModel):
user_id: int
device: Text
exp: datetime
iat: datetime
class CreateUser(BaseModel):
email: Text
username: Optional[Text]
# age: Optional[int]
# phone: Optional[Text]
password: Optional[Text]

2
base/exceptions.py
View File

@ -1,4 +1,4 @@
from graphql import GraphQLError
from graphql.error import GraphQLError
class BaseHttpException(GraphQLError):

2
base/orm.py
View File

@ -1,8 +1,10 @@
from typing import TypeVar, Any, Dict, Generic, Callable
from sqlalchemy import create_engine, Column, Integer
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from sqlalchemy.sql.schema import Table
from settings import DB_URL
if DB_URL.startswith("sqlite"):

69
base/redis.py
View File

@ -1,34 +1,35 @@
import aioredis
from settings import REDIS_URL
class Redis:
def __init__(self, uri=REDIS_URL):
self._uri: str = uri
self._instance = None
async def connect(self):
if self._instance is not None:
return
self._instance = aioredis.from_url(self._uri, encoding="utf-8")
async def disconnect(self):
if self._instance is None:
return
self._instance.close()
await self._instance.wait_closed()
self._instance = None
async def execute(self, command, *args, **kwargs):
return await self._instance.execute_command(command, *args, **kwargs)
async def lrange(self, key, start, stop):
return await self._instance.lrange(key, start, stop)
async def mget(self, key, *keys):
return await self._instance.mget(key, *keys)
redis = Redis()
__all__ = ["redis"]
import aioredis
from settings import REDIS_URL
class Redis:
def __init__(self, uri=REDIS_URL):
self._uri: str = uri
self._instance = None
async def connect(self):
if self._instance is not None:
return
self._instance = aioredis.from_url(self._uri, encoding="utf-8")
async def disconnect(self):
if self._instance is None:
return
self._instance.close()
await self._instance.wait_closed()
self._instance = None
async def execute(self, command, *args, **kwargs):
return await self._instance.execute_command(command, *args, **kwargs)
async def lrange(self, key, start, stop):
return await self._instance.lrange(key, start, stop)
async def mget(self, key, *keys):
return await self._instance.mget(key, *keys)
redis = Redis()
__all__ = ["redis"]

13
main.py
View File

@ -1,4 +1,6 @@
import asyncio
from importlib import import_module
from ariadne import load_schema_from_path, make_executable_schema
from ariadne.asgi import GraphQL
from starlette.applications import Starlette
@ -6,19 +8,18 @@ from starlette.middleware import Middleware
from starlette.middleware.authentication import AuthenticationMiddleware
from starlette.middleware.sessions import SessionMiddleware
from starlette.routing import Route
from auth.authenticate import JWTAuthenticate
from auth.oauth import oauth_login, oauth_authorize
from auth.email import email_authorize
from base.redis import redis
from base.resolvers import resolvers
from resolvers.zine import ShoutsCache
from services.main import storages_init
from services.stat.reacted import ReactedStorage
from services.stat.topicstat import TopicStat
from services.stat.viewed import ViewedStorage
from services.zine.gittask import GitTask
from services.stat.topicstat import TopicStat
from services.zine.shoutauthor import ShoutAuthorStorage
import asyncio
import_module("resolvers")
schema = make_executable_schema(load_schema_from_path("schema.graphql"), resolvers) # type: ignore
@ -42,6 +43,8 @@ async def start_up():
print(topic_stat_task)
git_task = asyncio.create_task(GitTask.git_task_worker())
print(git_task)
await storages_init()
print()
async def shutdown():
@ -51,7 +54,7 @@ async def shutdown():
routes = [
Route("/oauth/{provider}", endpoint=oauth_login),
Route("/oauth_authorize", endpoint=oauth_authorize),
Route("/email_authorize", endpoint=email_authorize),
# Route("/confirm_email", endpoint=), # should be called on client
]
app = Starlette(

77
migration/__init__.py
View File

@ -1,33 +1,31 @@
""" cmd managed migration """
import csv
import asyncio
from datetime import datetime
import json
import os
import subprocess
import sys
import os
from datetime import datetime
import bs4
import numpy as np
from migration.tables.comments import migrate as migrateComment
from migration.tables.comments import migrate_2stage as migrateComment_2stage
from migration.tables.content_items import get_shout_slug, migrate as migrateShout
from migration.tables.topics import migrate as migrateTopic
from migration.tables.users import migrate as migrateUser
from migration.tables.users import migrate_2stage as migrateUser_2stage
from orm.reaction import Reaction
from settings import DB_URL
# from export import export_email_subscriptions
from .export import export_mdx, export_slug
from orm.reaction import Reaction
from .tables.users import migrate as migrateUser
from .tables.users import migrate_2stage as migrateUser_2stage
from .tables.content_items import get_shout_slug, migrate as migrateShout
from .tables.topics import migrate as migrateTopic
from .tables.comments import migrate as migrateComment
from .tables.comments import migrate_2stage as migrateComment_2stage
from settings import DB_URL
TODAY = datetime.strftime(datetime.now(), "%Y%m%d")
OLD_DATE = "2016-03-05 22:22:00.350000"
def users_handle(storage):
async def users_handle(storage):
"""migrating users first"""
counter = 0
id_map = {}
@ -47,10 +45,9 @@ def users_handle(storage):
ce = 0
for entry in storage["users"]["data"]:
ce += migrateUser_2stage(entry, id_map)
return storage
def topics_handle(storage):
async def topics_handle(storage):
"""topics from categories and tags"""
counter = 0
for t in storage["topics"]["tags"] + storage["topics"]["cats"]:
@ -78,8 +75,6 @@ def topics_handle(storage):
+ str(len(storage["topics"]["by_slug"].values()))
+ " topics by slug"
)
# raise Exception
return storage
async def shouts_handle(storage, args):
@ -105,9 +100,9 @@ async def shouts_handle(storage, args):
if not shout["topics"]:
print("[migration] no topics!")
# wuth author
author = shout["authors"][0].slug
if author == "discours":
# with author
author: str = shout["authors"][0].dict()
if author["slug"] == "discours":
discours_author += 1
# print('[migration] ' + shout['slug'] + ' with author ' + author)
@ -118,21 +113,21 @@ async def shouts_handle(storage, args):
# print main counter
counter += 1
line = str(counter + 1) + ": " + shout["slug"] + " @" + author
line = str(counter + 1) + ": " + shout["slug"] + " @" + author["slug"]
print(line)
b = bs4.BeautifulSoup(shout["body"], "html.parser")
texts = []
texts.append(shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", ""))
texts = b.findAll(text=True)
texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")]
texts = texts + b.findAll(text=True)
topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts]))
topics_dataset_tlist.append(shout["topics"])
# np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=',', fmt='%s')
# np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=',
# ', fmt='%s')
print("[migration] " + str(counter) + " content items were migrated")
print("[migration] " + str(pub_counter) + " have been published")
print("[migration] " + str(discours_author) + " authored by @discours")
return storage
async def comments_handle(storage):
@ -146,9 +141,9 @@ async def comments_handle(storage):
missed_shouts[reaction] = oldcomment
elif type(reaction) == Reaction:
reaction = reaction.dict()
id = reaction["id"]
rid = reaction["id"]
oid = reaction["oid"]
id_map[oid] = id
id_map[oid] = rid
else:
ignored_counter += 1
@ -161,7 +156,6 @@ async def comments_handle(storage):
for missed in missed_shouts.values():
missed_counter += len(missed)
print("[migration] " + str(missed_counter) + " comments dropped")
return storage
def bson_handle():
@ -180,8 +174,8 @@ def export_one(slug, storage, args=None):
async def all_handle(storage, args):
print("[migration] handle everything")
users_handle(storage)
topics_handle(storage)
await users_handle(storage)
await topics_handle(storage)
await shouts_handle(storage, args)
await comments_handle(storage)
# export_email_subscriptions()
@ -205,11 +199,6 @@ def data_load():
"users": {"by_oid": {}, "by_slug": {}, "data": []},
"replacements": json.loads(open("migration/tables/replacements.json").read()),
}
users_data = []
tags_data = []
cats_data = []
comments_data = []
content_data = []
try:
users_data = json.loads(open("migration/data/users.json").read())
print("[migration.load] " + str(len(users_data)) + " users ")
@ -265,13 +254,13 @@ def data_load():
+ str(len(storage["reactions"]["by_content"].keys()))
+ " with comments"
)
storage["users"]["data"] = users_data
storage["topics"]["tags"] = tags_data
storage["topics"]["cats"] = cats_data
storage["shouts"]["data"] = content_data
storage["reactions"]["data"] = comments_data
except Exception as e:
raise e
storage["users"]["data"] = users_data
storage["topics"]["tags"] = tags_data
storage["topics"]["cats"] = cats_data
storage["shouts"]["data"] = content_data
storage["reactions"]["data"] = comments_data
return storage
@ -301,7 +290,7 @@ def create_pgdump():
async def handle_auto():
print("[migration] no command given, auto mode")
print("[migration] no option given, auto mode")
url = os.getenv("MONGODB_URL")
if url:
mongo_download(url)

5
migration/bson2json.py
View File

@ -1,6 +1,7 @@
import os
import bson
import json
import os
import bson
from .utils import DateTimeEncoder

40
migration/export.py
View File

@ -1,7 +1,9 @@
from datetime import datetime
import json
import os
from datetime import datetime
import frontmatter
from .extract import extract_html, prepare_html_body
from .utils import DateTimeEncoder
@ -67,22 +69,40 @@ def export_slug(slug, storage):
def export_email_subscriptions():
email_subscriptions_data = json.loads(open("migration/data/email_subscriptions.json").read())
email_subscriptions_data = json.loads(
open("migration/data/email_subscriptions.json").read()
)
for data in email_subscriptions_data:
# TODO: migrate to mailgun list manually
# migrate_email_subscription(data)
pass
print("[migration] " + str(len(email_subscriptions_data)) + " email subscriptions exported")
print(
"[migration] "
+ str(len(email_subscriptions_data))
+ " email subscriptions exported"
)
def export_shouts(storage):
# update what was just migrated or load json again
if len(storage["users"]["by_slugs"].keys()) == 0:
storage["users"]["by_slugs"] = json.loads(open(EXPORT_DEST + "authors.json").read())
print("[migration] " + str(len(storage["users"]["by_slugs"].keys())) + " exported authors ")
storage["users"]["by_slugs"] = json.loads(
open(EXPORT_DEST + "authors.json").read()
)
print(
"[migration] "
+ str(len(storage["users"]["by_slugs"].keys()))
+ " exported authors "
)
if len(storage["shouts"]["by_slugs"].keys()) == 0:
storage["shouts"]["by_slugs"] = json.loads(open(EXPORT_DEST + "articles.json").read())
print("[migration] " + str(len(storage["shouts"]["by_slugs"].keys())) + " exported articles ")
storage["shouts"]["by_slugs"] = json.loads(
open(EXPORT_DEST + "articles.json").read()
)
print(
"[migration] "
+ str(len(storage["shouts"]["by_slugs"].keys()))
+ " exported articles "
)
for slug in storage["shouts"]["by_slugs"].keys():
export_slug(slug, storage)
@ -130,4 +150,8 @@ def export_json(
ensure_ascii=False,
)
)
print("[migration] " + str(len(export_comments.items())) + " exported articles with comments")
print(
"[migration] "
+ str(len(export_comments.items()))
+ " exported articles with comments"
)

3
migration/extract.py
View File

@ -1,6 +1,7 @@
import base64
import os
import re
import base64
from .html2text import html2text
TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)"

10
migration/html2text/__init__.py
View File

@ -379,16 +379,16 @@ class HTML2Text(html.parser.HTMLParser):
if start:
if (
self.current_class == "highlight"
and self.inheader == False
and self.span_lead == False
and self.astack == False
and not self.inheader
and not self.span_lead
and not self.astack
):
self.o("`") # NOTE: same as <code>
self.span_highlight = True
elif (
self.current_class == "lead"
and self.inheader == False
and self.span_highlight == False
and not self.inheader
and not self.span_highlight
):
# self.o("==") # NOTE: CriticMarkup {==
self.span_lead = True

1
migration/html2text/cli.py
View File

@ -4,6 +4,7 @@ import sys
from . import HTML2Text, __version__, config
# noinspection DuplicatedCode
def main() -> None:
baseurl = ""

9
migration/html2text/utils.py
View File

@ -68,13 +68,11 @@ def element_style(
:rtype: dict
"""
style = parent_style.copy()
if "class" in attrs:
assert attrs["class"] is not None
if attrs.get("class"):
for css_class in attrs["class"].split():
css_style = style_def.get("." + css_class, {})
style.update(css_style)
if "style" in attrs:
assert attrs["style"] is not None
if attrs.get("style"):
immediate_style = dumb_property_dict(attrs["style"])
style.update(immediate_style)
@ -149,8 +147,7 @@ def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
:rtype: int or None
"""
if "start" in attrs:
assert attrs["start"] is not None
if attrs.get("start"):
try:
return int(attrs["start"]) - 1
except ValueError:

23
migration/tables/comments.py
View File

@ -1,8 +1,10 @@
from datetime import datetime
from dateutil.parser import parse as date_parse
from orm import Reaction, User
from base.orm import local_session
from migration.html2text import html2text
from orm import Reaction, User
from orm.reaction import ReactionKind
from services.stat.reacted import ReactedStorage
@ -46,16 +48,13 @@ async def migrate(entry, storage):
old_thread: String
}
"""
reaction_dict = {}
reaction_dict["createdAt"] = (
ts if not entry.get("createdAt") else date_parse(entry.get("createdAt"))
)
print("[migration] reaction original date %r" % entry.get("createdAt"))
# print('[migration] comment date %r ' % comment_dict['createdAt'])
reaction_dict["body"] = html2text(entry.get("body", ""))
reaction_dict["oid"] = entry["_id"]
if entry.get("createdAt"):
reaction_dict["createdAt"] = date_parse(entry.get("createdAt"))
reaction_dict = {
"createdAt": (
ts if not entry.get("createdAt") else date_parse(entry.get("createdAt"))
),
"body": html2text(entry.get("body", "")),
"oid": entry["_id"],
}
shout_oid = entry.get("contentItem")
if shout_oid not in storage["shouts"]["by_oid"]:
if len(storage["shouts"]["by_oid"]) > 0:
@ -126,7 +125,7 @@ def migrate_2stage(rr, old_new_id):
with local_session() as session:
comment = session.query(Reaction).filter(Reaction.id == new_id).first()
comment.replyTo = old_new_id.get(reply_oid)
comment.save()
session.add(comment)
session.commit()
if not rr["body"]:
raise Exception(rr)

51
migration/tables/content_items.py
View File

@ -1,14 +1,18 @@
from dateutil.parser import parse as date_parse
import sqlalchemy
from orm.shout import Shout, ShoutTopic, User
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedByDay
from transliterate import translit
from datetime import datetime
from dateutil.parser import parse as date_parse
from sqlalchemy.exc import IntegrityError
from transliterate import translit
from base.orm import local_session
from migration.extract import prepare_html_body
from orm.community import Community
from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutTopic, User
from orm.topic import TopicFollower
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedByDay
from services.zine.topics import TopicStorage
OLD_DATE = "2016-03-05 22:22:00.350000"
ts = datetime.now()
@ -72,7 +76,10 @@ async def migrate(entry, storage):
}
else:
userdata = User.default_user.dict()
assert userdata, "no user found for %s from %d" % [oid, len(users_by_oid.keys())]
if not userdata:
raise Exception(
"no user found for %s from %d" % [oid, len(users_by_oid.keys())]
)
r["authors"] = [
userdata,
]
@ -139,32 +146,40 @@ async def migrate(entry, storage):
# del shout_dict['rating'] # NOTE: TypeError: 'rating' is an invalid keyword argument for Shout
# del shout_dict['ratings']
email = userdata.get("email")
slug = userdata.get("slug")
if not slug:
userslug = userdata.get("slug")
if not userslug:
raise Exception
with local_session() as session:
# c = session.query(Community).all().pop()
if email:
user = session.query(User).filter(User.email == email).first()
if not user and slug:
user = session.query(User).filter(User.slug == slug).first()
if not user and userslug:
user = session.query(User).filter(User.slug == userslug).first()
if not user and userdata:
try:
userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-")
user = User.create(**userdata)
except sqlalchemy.exc.IntegrityError:
except IntegrityError:
print("[migration] user error: " + userdata)
userdata["id"] = user.id
userdata["createdAt"] = user.createdAt
storage["users"]["by_slug"][userdata["slug"]] = userdata
storage["users"]["by_oid"][entry["_id"]] = userdata
assert user, "could not get a user"
shout_dict["authors"] = [user, ]
if not user:
raise Exception("could not get a user")
shout_dict["authors"] = [
user,
]
# TODO: subscribe shout user on shout topics
try:
s = Shout.create(**shout_dict)
except sqlalchemy.exc.IntegrityError as e:
with local_session() as session:
topics = session.query(ShoutTopic).where(ShoutTopic.shout == s.slug).all()
for tpc in topics:
TopicFollower.create(topic=tpc.slug, follower=userslug)
await TopicStorage.update_topic(tpc.slug)
except IntegrityError as e:
with local_session() as session:
s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
bump = False
@ -267,9 +282,9 @@ async def migrate(entry, storage):
)
reaction.update(reaction_dict)
else:
reaction_dict["day"] = (
reaction_dict.get("createdAt") or ts
).replace(hour=0, minute=0, second=0, microsecond=0)
# day = (
# reaction_dict.get("createdAt") or ts
# ).replace(hour=0, minute=0, second=0, microsecond=0)
rea = Reaction.create(**reaction_dict)
await ReactedStorage.react(rea)
# shout_dict['ratings'].append(reaction_dict)

36
migration/tables/replacements.json
View File

@ -764,5 +764,37 @@
"blocked-in-russia": "blocked-in-russia",
"kavarga": "kavarga",
"galereya-anna-nova": "gallery-anna-nova",
"derrida": "derrida"
}
"derrida": "derrida",
"dinozavry": "dinosaurs",
"beecake": "beecake",
"literaturnyykaver": "literature-cover",
"dialog": "dialogue",
"dozhd": "rain",
"pomosch": "help",
"igra": "game",
"reportazh-1": "reportage",
"armiya-1": "army",
"ukraina-2": "ukraine",
"nasilie-1": "violence",
"smert-1": "death",
"dnevnik-1": "dairy",
"voyna-na-ukraine": "war-in-ukraine",
"zabota": "care",
"ango": "ango",
"hayku": "haiku",
"utrata": "loss",
"pokoy": "peace",
"kladbische": "cemetery",
"lomonosov": "lomonosov",
"istoriya-nauki": "history-of-sceince",
"sud": "court",
"russkaya-toska": "russian-toska",
"duh": "spirit",
"devyanostye": "90s",
"seksualnoe-nasilie": "sexual-violence",
"gruziya-2": "georgia",
"dokumentalnaya-poeziya": "documentary-poetry",
"kriptovalyuty": "cryptocurrencies",
"magiya": "magic",
"yazychestvo": "paganism"
}

2
migration/tables/topics.py
View File

@ -1,5 +1,5 @@
from migration.extract import extract_md, html2text
from base.orm import local_session
from migration.extract import extract_md, html2text
from orm import Topic, Community

36
migration/tables/users.py
View File

@ -1,8 +1,9 @@
import sqlalchemy
from dateutil.parser import parse
from sqlalchemy.exc import IntegrityError
from base.orm import local_session
from migration.html2text import html2text
from orm import User, UserRating
from dateutil.parser import parse
from base.orm import local_session
def migrate(entry):
@ -21,9 +22,6 @@ def migrate(entry):
"muted": False, # amnesty
"bio": entry["profile"].get("bio", ""),
"notifications": [],
"createdAt": parse(entry["createdAt"]),
"roles": [], # entry['roles'] # roles by community
"ratings": [], # entry['ratings']
"links": [],
"name": "anonymous",
}
@ -86,7 +84,7 @@ def migrate(entry):
user_dict["slug"] = user_dict["slug"].lower().strip().replace(" ", "-")
try:
user = User.create(**user_dict.copy())
except sqlalchemy.exc.IntegrityError:
except IntegrityError:
print("[migration] cannot create user " + user_dict["slug"])
with local_session() as session:
old_user = (
@ -120,28 +118,10 @@ def migrate_2stage(entry, id_map):
with local_session() as session:
try:
user_rating = UserRating.create(**user_rating_dict)
except sqlalchemy.exc.IntegrityError:
old_rating = (
session.query(UserRating)
.filter(UserRating.rater == rater_slug)
.first()
)
print(
"[migration] cannot create "
+ author_slug
+ "`s rate from "
+ rater_slug
)
print(
"[migration] concat rating value %d+%d=%d"
% (
old_rating.value,
rating_entry["value"],
old_rating.value + rating_entry["value"],
)
)
old_rating.update({"value": old_rating.value + rating_entry["value"]})
session.add(user_rating)
session.commit()
except IntegrityError:
print("[migration] cannot rate " + author_slug + "`s by " + rater_slug)
except Exception as e:
print(e)
return ce

70
orm/__init__.py
View File

@ -1,41 +1,29 @@
from orm.rbac import Operation, Resource, Permission, Role
from services.auth.roles import RoleStorage
from orm.community import Community
from orm.user import User, UserRating
from orm.topic import Topic, TopicFollower
from orm.notification import Notification
from orm.shout import Shout
from orm.reaction import Reaction
from services.stat.reacted import ReactedStorage
from services.zine.topics import TopicStorage
from services.auth.users import UserStorage
from services.stat.viewed import ViewedStorage
from base.orm import Base, engine, local_session
__all__ = [
"User",
"Role",
"Operation",
"Permission",
"Community",
"Shout",
"Topic",
"TopicFollower",
"Notification",
"Reaction",
"UserRating",
]
Base.metadata.create_all(engine)
Operation.init_table()
Resource.init_table()
User.init_table()
Community.init_table()
Role.init_table()
with local_session() as session:
ViewedStorage.init(session)
ReactedStorage.init(session)
RoleStorage.init(session)
UserStorage.init(session)
TopicStorage.init(session)
from base.orm import Base, engine
from orm.community import Community
from orm.notification import Notification
from orm.rbac import Operation, Resource, Permission, Role
from orm.reaction import Reaction
from orm.shout import Shout
from orm.topic import Topic, TopicFollower
from orm.user import User, UserRating
__all__ = [
"User",
"Role",
"Operation",
"Permission",
"Community",
"Shout",
"Topic",
"TopicFollower",
"Notification",
"Reaction",
"UserRating",
]
Base.metadata.create_all(engine)
Operation.init_table()
Resource.init_table()
User.init_table()
Community.init_table()
Role.init_table()

2
orm/collab.py
View File

@ -1,5 +1,7 @@
from datetime import datetime
from sqlalchemy import Boolean, Column, String, ForeignKey, DateTime
from base.orm import Base

2
orm/collection.py
View File

@ -1,5 +1,7 @@
from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime
from base.orm import Base

2
orm/community.py
View File

@ -1,5 +1,7 @@
from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime
from base.orm import Base, local_session

1
orm/notification.py
View File

@ -1,4 +1,5 @@
from sqlalchemy import Column, String, JSON as JSONType
from base.orm import Base

2
orm/rbac.py
View File

@ -1,6 +1,8 @@
import warnings
from sqlalchemy import String, Column, ForeignKey, UniqueConstraint, TypeDecorator
from sqlalchemy.orm import relationship
from base.orm import Base, REGISTRY, engine, local_session
from orm.community import Community

4
orm/reaction.py
View File

@ -1,7 +1,9 @@
from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime
from base.orm import Base
from sqlalchemy import Enum
from base.orm import Base
from services.stat.reacted import ReactedStorage, ReactionKind
from services.stat.viewed import ViewedStorage

8
orm/shout.py
View File

@ -1,12 +1,14 @@
from datetime import datetime
from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, Boolean
from sqlalchemy.orm import relationship
from orm.user import User
from orm.topic import Topic, ShoutTopic
from base.orm import Base
from orm.reaction import Reaction
from orm.topic import Topic, ShoutTopic
from orm.user import User
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedStorage
from base.orm import Base
class ShoutReactionsFollower(Base):

2
orm/topic.py
View File

@ -1,5 +1,7 @@
from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime, JSON as JSONType
from base.orm import Base

2
orm/user.py
View File

@ -1,4 +1,5 @@
from datetime import datetime
from sqlalchemy import (
Column,
Integer,
@ -9,6 +10,7 @@ from sqlalchemy import (
JSON as JSONType,
)
from sqlalchemy.orm import relationship
from base.orm import Base, local_session
from orm.rbac import Role
from services.auth.roles import RoleStorage

33
requirements.txt
View File

@ -1,20 +1,25 @@
frontmatter
numpy
aioredis
ariadne
python-frontmatter~=1.0.0
aioredis~=2.0.1
ariadne>=0.16.0
PyYAML>=5.4
pyjwt>=2.0.0
starlette
sqlalchemy
uvicorn
pydantic
passlib
starlette~=0.20.4
sqlalchemy>=1.4.41
graphql-core
uvicorn>=0.18.3
pydantic>=1.10.2
passlib~=1.7.4
itsdangerous
authlib==0.15.5
authlib>=1.1.0
httpx>=0.23.0
psycopg2-binary
transliterate
requests
bcrypt
transliterate~=1.10.2
requests~=2.28.1
bcrypt>=4.0.0
websockets
bson
bson~=0.5.10
flake8
DateTime~=4.7
asyncio~=3.4.3
python-dateutil~=2.8.2
beautifulsoup4~=4.11.1

78
resolvers/__init__.py
View File

@ -3,9 +3,42 @@ from resolvers.auth import (
sign_out,
is_email_used,
register,
confirm,
auth_forget,
auth_reset,
confirm_email,
auth_send_link,
)
from resolvers.collab import remove_author, invite_author
from resolvers.community import (
create_community,
delete_community,
get_community,
get_communities,
)
# from resolvers.collab import invite_author, remove_author
from resolvers.editor import create_shout, delete_shout, update_shout
from resolvers.profile import (
get_users_by_slugs,
get_current_user,
get_user_reacted_shouts,
get_user_roles,
get_top_authors,
)
# from resolvers.feed import shouts_for_feed, my_candidates
from resolvers.reactions import (
create_reaction,
delete_reaction,
update_reaction,
reactions_unfollow,
reactions_follow,
get_shout_reactions,
)
from resolvers.topics import (
topic_follow,
topic_unfollow,
topics_by_author,
topics_by_community,
topics_all,
)
from resolvers.zine import (
get_shout_by_slug,
@ -21,36 +54,6 @@ from resolvers.zine import (
shouts_by_topics,
shouts_by_communities,
)
from resolvers.profile import (
get_users_by_slugs,
get_current_user,
get_user_reacted_shouts,
get_user_roles,
get_top_authors
)
from resolvers.topics import (
topic_follow,
topic_unfollow,
topics_by_author,
topics_by_community,
topics_all,
)
# from resolvers.feed import shouts_for_feed, my_candidates
from resolvers.reactions import (
create_reaction,
delete_reaction,
update_reaction,
)
# from resolvers.collab import invite_author, remove_author
from resolvers.editor import create_shout, delete_shout, update_shout
from resolvers.community import (
create_community,
delete_community,
get_community,
get_communities,
)
__all__ = [
"follow",
@ -59,9 +62,8 @@ __all__ = [
"login",
"register",
"is_email_used",
"confirm",
"auth_forget",
"auth_reset",
"confirm_email",
"auth_send_link",
"sign_out",
# profile
"get_current_user",
@ -69,10 +71,7 @@ __all__ = [
"get_user_roles",
"get_top_authors",
# zine
"shouts_for_feed",
"my_candidates",
"recent_published",
"recent_reacted",
"recent_all",
"shouts_by_topics",
"shouts_by_authors",
@ -82,7 +81,6 @@ __all__ = [
"top_overall",
"top_viewed",
"view_shout",
"view_reaction",
"get_shout_by_slug",
# editor
"create_shout",

148
resolvers/auth.py
View File

@ -1,29 +1,42 @@
from graphql import GraphQLResolveInfo
from transliterate import translit
from urllib.parse import quote_plus
from auth.authenticate import login_required, ResetPassword
from auth.authorize import Authorize
from auth.identity import Identity
from auth.password import Password
from auth.email import send_confirm_email, send_auth_email, send_reset_password_email
from orm import User, Role
from auth.tokenstorage import TokenStorage
from graphql.type import GraphQLResolveInfo
from transliterate import translit
from auth.authenticate import login_required
from auth.email import send_auth_email
from auth.identity import Identity, Password
from base.exceptions import (
InvalidPassword,
InvalidToken,
ObjectNotExist,
OperationNotAllowed,
)
from base.orm import local_session
from base.resolvers import mutation, query
from orm import User, Role
from resolvers.profile import get_user_info
from base.exceptions import InvalidPassword, InvalidToken, ObjectNotExist, OperationNotAllowed
from settings import JWT_AUTH_HEADER
from settings import SESSION_TOKEN_HEADER
@mutation.field("confirmEmail")
async def confirm(*_, confirm_token):
async def confirm_email(*_, confirm_token):
"""confirm owning email address"""
auth_token, user = await Authorize.confirm(confirm_token)
if auth_token:
user.emailConfirmed = True
user.save()
return {"token": auth_token, "user": user}
else:
# not an error, warns user
user_id = None
try:
user_id = await TokenStorage.get(confirm_token)
with local_session() as session:
user = session.query(User).where(User.id == user_id).first()
session_token = TokenStorage.create_session(user)
user.emailConfirmed = True
session.add(user)
session.commit()
return {"token": session_token, "user": user}
except InvalidToken as e:
raise InvalidToken(e.message)
except Exception as e:
print(e) # FIXME: debug only
return {"error": "email not confirmed"}
@ -50,40 +63,21 @@ async def register(*_, email: str, password: str = ""):
session.add(user)
session.commit()
await send_confirm_email(user)
token = await TokenStorage.create_onetime(user)
await send_auth_email(user, token)
return {"user": user}
@mutation.field("requestPasswordUpdate")
async def auth_forget(_, info, email):
"""send email to recover account"""
@mutation.field("sendLink")
async def auth_send_link(_, info, email):
"""send link with confirm code to email"""
with local_session() as session:
user = session.query(User).filter(User.email == email).first()
if not user:
raise ObjectNotExist("User not found")
await send_reset_password_email(user)
return {}
@mutation.field("updatePassword")
async def auth_reset(_, info, password, resetToken):
"""set the new password"""
try:
user_id = await ResetPassword.verify(resetToken)
except InvalidToken as e:
raise InvalidToken(e.message)
# return {"error": e.message}
with local_session() as session:
user = session.query(User).filter_by(id=user_id).first()
if not user:
raise ObjectNotExist("User not found")
user.password = Password.encode(password)
session.commit()
token = await TokenStorage.create_onetime(user)
await send_auth_email(user, token)
return {}
@ -92,48 +86,44 @@ async def login(_, info: GraphQLResolveInfo, email: str, password: str = ""):
with local_session() as session:
orm_user = session.query(User).filter(User.email == email).first()
if orm_user is None:
print(f"signIn {email}: email not found")
# return {"error": "email not found"}
raise ObjectNotExist("User not found")
if orm_user is None:
print(f"[auth] {email}: email not found")
# return {"error": "email not found"}
raise ObjectNotExist("User not found") # contains webserver status
if not password:
print(f"signIn {email}: send auth email")
await send_auth_email(orm_user)
return {""}
if not password:
print(f"[auth] send confirm link to {email}")
token = await TokenStorage.create_onetime(orm_user)
await send_auth_email(orm_user, token)
# FIXME: not an error, warning
return {"error": "no password, email link was sent"}
if not orm_user.emailConfirmed:
# not an error, warns users
return {"error": "email not confirmed"}
try:
device = info.context["request"].headers["device"]
except KeyError:
device = "pc"
auto_delete = False if device == "mobile" else True # why autodelete with mobile?
try:
user = Identity.identity(orm_user, password)
except InvalidPassword:
print(f"signIn {email}: invalid password")
raise InvalidPassword("invalid passoword")
# return {"error": "invalid password"}
token = await Authorize.authorize(user, device=device, auto_delete=auto_delete)
print(f"signIn {email}: OK")
return {
"token": token,
"user": orm_user,
"info": await get_user_info(orm_user.slug),
}
else:
# sign in using password
if not orm_user.emailConfirmed:
# not an error, warns users
return {"error": "please, confirm email"}
else:
try:
user = Identity.password(orm_user, password)
session_token = await TokenStorage.create_session(user)
print(f"[auth] user {email} authorized")
return {
"token": session_token,
"user": user,
"info": await get_user_info(user.slug),
}
except InvalidPassword:
print(f"[auth] {email}: invalid password")
raise InvalidPassword("invalid passoword") # contains webserver status
# return {"error": "invalid password"}
@query.field("signOut")
@login_required
async def sign_out(_, info: GraphQLResolveInfo):
token = info.context["request"].headers[JWT_AUTH_HEADER]
status = await Authorize.revoke(token)
token = info.context["request"].headers[SESSION_TOKEN_HEADER]
status = await TokenStorage.revoke(token)
return status

12
resolvers/collab.py
View File

@ -1,10 +1,11 @@
from datetime import datetime
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import query, mutation
from orm.collab import Collab
from orm.shout import Shout
from orm.user import User
from base.resolvers import query, mutation
from auth.authenticate import login_required
@query.field("getCollabs")
@ -12,11 +13,10 @@ from auth.authenticate import login_required
async def get_collabs(_, info):
auth = info.context["request"].auth
user_id = auth.user_id
collabs = []
with local_session() as session:
user = session.query(User).where(User.id == user_id).first()
collabs = session.query(Collab).filter(user.slug in Collab.authors)
return collabs
return collabs
@mutation.field("inviteAuthor")
@ -37,7 +37,7 @@ async def invite_author(_, info, author, shout):
return {"error": "already added"}
shout.authors.append(author)
shout.updated_at = datetime.now()
shout.save()
session.add(shout)
session.commit()
# TODO: email notify
@ -63,7 +63,7 @@ async def remove_author(_, info, author, shout):
return {"error": "not in authors"}
shout.authors.remove(author)
shout.updated_at = datetime.now()
shout.save()
session.add(shout)
session.commit()
# result = Result("INVITED")

15
resolvers/collection.py
View File

@ -1,11 +1,13 @@
from orm.collection import Collection, ShoutCollection
from base.orm import local_session
from orm.user import User
from base.resolvers import mutation, query
from auth.authenticate import login_required
from datetime import datetime
from sqlalchemy import and_
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.collection import Collection, ShoutCollection
from orm.user import User
@mutation.field("createCollection")
@login_required
@ -27,7 +29,7 @@ async def create_collection(_, _info, inp):
async def update_collection(_, info, inp):
auth = info.context["request"].auth
user_id = auth.user_id
collection_slug = input.get("slug", "")
collection_slug = inp.get("slug", "")
with local_session() as session:
owner = session.query(User).filter(User.id == user_id) # note list here
collection = (
@ -57,6 +59,7 @@ async def delete_collection(_, info, slug):
if collection.owner != user_id:
return {"error": "access denied"}
collection.deletedAt = datetime.now()
session.add(collection)
session.commit()
return {}

16
resolvers/community.py
View File

@ -1,12 +1,14 @@
from orm.community import Community, CommunityFollower
from base.orm import local_session
from orm.user import User
from base.resolvers import mutation, query
from auth.authenticate import login_required
from datetime import datetime
from typing import List
from sqlalchemy import and_
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.community import Community, CommunityFollower
from orm.user import User
@mutation.field("createCommunity")
@login_required
@ -23,6 +25,8 @@ async def create_community(_, info, input):
createdBy=user.slug,
createdAt=datetime.now(),
)
session.add(community)
session.commit()
return {"community": community}
@ -48,6 +52,7 @@ async def update_community(_, info, input):
community.desc = input.get("desc", "")
community.pic = input.get("pic", "")
community.updatedAt = datetime.now()
session.add(community)
session.commit()
@ -64,6 +69,7 @@ async def delete_community(_, info, slug):
if community.owner != user_id:
return {"error": "access denied"}
community.deletedAt = datetime.now()
session.add(community)
session.commit()
return {}

35
resolvers/editor.py
View File

@ -1,37 +1,38 @@
from orm import Shout
from datetime import datetime
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation
from orm import Shout
from orm.rbac import Resource
from orm.shout import ShoutAuthor, ShoutTopic
from orm.user import User
from base.resolvers import mutation
from resolvers.reactions import reactions_follow, reactions_unfollow
from auth.authenticate import login_required
from datetime import datetime
from services.zine.gittask import GitTask
@mutation.field("createShout")
@login_required
async def create_shout(_, info, input):
async def create_shout(_, info, inp):
user = info.context["request"].user
topic_slugs = input.get("topic_slugs", [])
topic_slugs = inp.get("topic_slugs", [])
if topic_slugs:
del input["topic_slugs"]
del inp["topic_slugs"]
new_shout = Shout.create(**input)
new_shout = Shout.create(**inp)
ShoutAuthor.create(shout=new_shout.slug, user=user.slug)
reactions_follow(user, new_shout.slug, True)
if "mainTopic" in input:
topic_slugs.append(input["mainTopic"])
if "mainTopic" in inp:
topic_slugs.append(inp["mainTopic"])
for slug in topic_slugs:
ShoutTopic.create(shout=new_shout.slug, topic=slug)
new_shout.topic_slugs = topic_slugs
GitTask(input, user.username, user.email, "new shout %s" % (new_shout.slug))
GitTask(inp, user.username, user.email, "new shout %s" % (new_shout.slug))
# await ShoutCommentsStorage.send_shout(new_shout)
@ -40,11 +41,11 @@ async def create_shout(_, info, input):
@mutation.field("updateShout")
@login_required
async def update_shout(_, info, input):
async def update_shout(_, info, inp):
auth = info.context["request"].auth
user_id = auth.user_id
slug = input["slug"]
slug = inp["slug"]
session = local_session()
user = session.query(User).filter(User.id == user_id).first()
@ -60,15 +61,16 @@ async def update_shout(_, info, input):
if Resource.shout_id not in scopes:
return {"error": "access denied"}
shout.update(input)
shout.update(inp)
shout.updatedAt = datetime.now()
session.add(shout)
session.commit()
session.close()
for topic in input.get("topic_slugs", []):
for topic in inp.get("topic_slugs", []):
ShoutTopic.create(shout=slug, topic=topic)
GitTask(input, user.username, user.email, "update shout %s" % (slug))
GitTask(inp, user.username, user.email, "update shout %s" % (slug))
return {"shout": shout}
@ -89,6 +91,7 @@ async def delete_shout(_, info, slug):
for a in authors:
reactions_unfollow(a.slug, slug, True)
shout.deletedAt = datetime.now()
session.add(shout)
session.commit()
return {}

13
resolvers/feed.py
View File

@ -1,11 +1,13 @@
from typing import List
from sqlalchemy import and_, desc
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import query
from sqlalchemy import and_, desc
from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.topic import TopicFollower
from orm.user import AuthorFollower
from typing import List
from services.zine.shoutscache import prepare_shouts
@ -22,14 +24,14 @@ def get_user_feed(_, info, offset, limit) -> List[Shout]:
.where(AuthorFollower.follower == user.slug)
.order_by(desc(Shout.createdAt))
)
topicrows = (
topic_rows = (
session.query(Shout)
.join(ShoutTopic)
.join(TopicFollower)
.where(TopicFollower.follower == user.slug)
.order_by(desc(Shout.createdAt))
)
shouts = shouts.union(topicrows).limit(limit).offset(offset).all()
shouts = shouts.union(topic_rows).limit(limit).offset(offset).all()
return shouts
@ -37,7 +39,6 @@ def get_user_feed(_, info, offset, limit) -> List[Shout]:
@login_required
async def user_unpublished_shouts(_, info, offset, limit) -> List[Shout]:
user = info.context["request"].user
shouts = []
with local_session() as session:
shouts = prepare_shouts(
session.query(Shout)
@ -48,4 +49,4 @@ async def user_unpublished_shouts(_, info, offset, limit) -> List[Shout]:
.offset(offset)
.all()
)
return shouts
return shouts

7
resolvers/inbox.py
View File

@ -1,10 +1,11 @@
from base.resolvers import mutation, query, subscription
from auth.authenticate import login_required
import asyncio
import uuid
import json
import uuid
from datetime import datetime
from auth.authenticate import login_required
from base.redis import redis
from base.resolvers import mutation, query, subscription
class ChatFollowing:

37
resolvers/profile.py
View File

@ -1,18 +1,21 @@
from datetime import datetime
from orm.user import User, UserRole, Role, UserRating, AuthorFollower
from services.auth.users import UserStorage
from orm.shout import Shout
from orm.reaction import Reaction
from base.orm import local_session
from orm.topic import Topic, TopicFollower
from base.resolvers import mutation, query
from resolvers.community import get_followed_communities
from resolvers.reactions import get_shout_reactions
from auth.authenticate import login_required
from resolvers.inbox import get_unread_counter
from typing import List
from sqlalchemy import and_, desc
from sqlalchemy.orm import selectinload
from typing import List
from auth.authenticate import login_required
from auth.tokenstorage import TokenStorage
from base.orm import local_session
from base.resolvers import mutation, query
from orm.reaction import Reaction
from orm.shout import Shout
from orm.topic import Topic, TopicFollower
from orm.user import User, UserRole, Role, UserRating, AuthorFollower
from resolvers.community import get_followed_communities
from resolvers.inbox import get_unread_counter
from resolvers.reactions import get_shout_reactions
from services.auth.users import UserStorage
@query.field("userReactedShouts")
@ -87,12 +90,13 @@ async def get_user_info(slug):
@login_required
async def get_current_user(_, info):
user = info.context["request"].user
user.lastSeen = datetime.now()
with local_session() as session:
user.lastSeen = datetime.now()
user.save()
session.add(user)
session.commit()
token = await TokenStorage.create_session(user)
return {
"token": "", # same token?
"token": token,
"user": user,
"info": await get_user_info(user.slug),
}
@ -133,7 +137,8 @@ async def update_profile(_, info, profile):
user = session.query(User).filter(User.id == user_id).first()
if user:
User.update(user, **profile)
session.commit()
session.add(user)
session.commit()
return {}

10
resolvers/reactions.py
View File

@ -1,11 +1,13 @@
from datetime import datetime
from sqlalchemy import desc
from orm.reaction import Reaction
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.reaction import Reaction
from orm.shout import ShoutReactionsFollower
from orm.user import User
from base.resolvers import mutation, query
from auth.authenticate import login_required
from datetime import datetime
from services.auth.users import UserStorage
from services.stat.reacted import ReactedStorage

16
resolvers/topics.py
View File

@ -1,12 +1,14 @@
from orm.topic import Topic, TopicFollower
from services.zine.topics import TopicStorage
from services.stat.topicstat import TopicStat
import random
from sqlalchemy import and_
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from auth.authenticate import login_required
from sqlalchemy import and_
import random
from orm.topic import Topic, TopicFollower
from services.stat.topicstat import TopicStat
from services.zine.shoutscache import ShoutsCache
from services.zine.topics import TopicStorage
@query.field("topicsAll")
@ -60,7 +62,7 @@ async def update_topic(_, _info, inp):
async def topic_follow(user, slug):
TopicFollower.create(follower=user.slug, topic=slug)
TopicFollower.create(topic=slug, follower=user.slug)
await TopicStorage.update_topic(slug)

37
resolvers/zine.py
View File

@ -1,18 +1,19 @@
from sqlalchemy.orm import selectinload
from sqlalchemy.sql.expression import and_, select, desc
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.collection import ShoutCollection
from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.topic import Topic
from base.orm import local_session
from base.resolvers import mutation, query
from resolvers.community import community_follow, community_unfollow
from resolvers.profile import author_follow, author_unfollow
from resolvers.reactions import reactions_follow, reactions_unfollow
from resolvers.topics import topic_follow, topic_unfollow
from services.stat.viewed import ViewedStorage
from services.zine.shoutauthor import ShoutAuthorStorage
from services.zine.shoutscache import ShoutsCache
from services.stat.viewed import ViewedStorage
from resolvers.profile import author_follow, author_unfollow
from resolvers.topics import topic_follow, topic_unfollow
from resolvers.community import community_follow, community_unfollow
from resolvers.reactions import reactions_follow, reactions_unfollow
from auth.authenticate import login_required
from sqlalchemy import select, desc, asc, and_
from sqlalchemy.orm import selectinload
@mutation.field("incrementView")
@ -33,6 +34,12 @@ async def top_month(_, _info, offset, limit):
return ShoutsCache.top_month[offset : offset + limit]
@query.field("topCommented")
async def top_commented(_, _info, offset, limit):
async with ShoutsCache.lock:
return ShoutsCache.top_commented[offset : offset + limit]
@query.field("topOverall")
async def top_overall(_, _info, offset, limit):
async with ShoutsCache.lock:
@ -105,7 +112,7 @@ async def get_search_results(_, _info, query, offset, limit):
for s in shouts:
for a in s.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
s.stat.search = 1 # FIXME
s.stat.relevance = 1 # FIXME
return shouts
@ -116,7 +123,7 @@ async def shouts_by_topics(_, _info, slugs, offset, limit):
session.query(Shout)
.join(ShoutTopic)
.where(and_(ShoutTopic.topic.in_(slugs), bool(Shout.publishedAt)))
.order_by(asc(Shout.publishedAt))
.order_by(desc(Shout.publishedAt))
.limit(limit)
.offset(offset)
)
@ -134,7 +141,7 @@ async def shouts_by_collection(_, _info, collection, offset, limit):
session.query(Shout)
.join(ShoutCollection, ShoutCollection.collection == collection)
.where(and_(ShoutCollection.shout == Shout.slug, bool(Shout.publishedAt)))
.order_by(asc(Shout.publishedAt))
.order_by(desc(Shout.publishedAt))
.limit(limit)
.offset(offset)
)
@ -151,7 +158,7 @@ async def shouts_by_authors(_, _info, slugs, offset, limit):
session.query(Shout)
.join(ShoutAuthor)
.where(and_(ShoutAuthor.user.in_(slugs), bool(Shout.publishedAt)))
.order_by(asc(Shout.publishedAt))
.order_by(desc(Shout.publishedAt))
.limit(limit)
.offset(offset)
)
@ -184,7 +191,7 @@ async def shouts_by_communities(_, info, slugs, offset, limit):
),
)
)
.order_by(desc(Shout.publishedAt))
.order_by(desc("publishedAt"))
.limit(limit)
.offset(offset)
)

6
schema.graphql
View File

@ -148,11 +148,10 @@ type Mutation {
markAsRead(chatId: String!, ids: [Int]!): Result!
# auth
confirmEmail(token: String!): AuthResult!
refreshSession: AuthResult!
registerUser(email: String!, password: String): AuthResult!
requestPasswordUpdate(email: String!): Result!
updatePassword(password: String!, token: String!): Result!
sendLink(email: String!): Result!
confirmEmail(code: String!): AuthResult!
# shout
createShout(input: ShoutInput!): Result!
@ -237,6 +236,7 @@ type Query {
topAuthors(offset: Int!, limit: Int!): [Author]!
topMonth(offset: Int!, limit: Int!): [Shout]!
topOverall(offset: Int!, limit: Int!): [Shout]!
topCommented(offset: Int!, limit: Int!): [Shout]!
recentPublished(offset: Int!, limit: Int!): [Shout]! # homepage
recentReacted(offset: Int!, limit: Int!): [Shout]! # test
recentAll(offset: Int!, limit: Int!): [Shout]!

59
server.py
View File

@ -1,30 +1,29 @@
import uvicorn
from settings import PORT
import sys
if __name__ == "__main__":
x = ""
if len(sys.argv) > 1:
x = sys.argv[1]
if x == "dev":
print("DEV MODE")
headers = [
("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD"),
("Access-Control-Allow-Origin", "http://localhost:3000"),
(
"Access-Control-Allow-Headers",
"DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range",
),
("Access-Control-Expose-Headers", "Content-Length,Content-Range"),
("Access-Control-Allow-Credentials", "true"),
]
uvicorn.run(
"main:app", host="localhost", port=8080, headers=headers
) # , ssl_keyfile="discours.key", ssl_certfile="discours.crt", reload=True)
elif x == "migrate":
from migration import migrate
migrate()
else:
uvicorn.run("main:app", host="0.0.0.0", port=PORT)
import sys
import uvicorn
from settings import PORT
if __name__ == "__main__":
x = ""
if len(sys.argv) > 1:
x = sys.argv[1]
if x == "dev":
print("DEV MODE")
headers = [
("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD"),
("Access-Control-Allow-Origin", "http://localhost:3000"),
(
"Access-Control-Allow-Headers",
"DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range",
),
("Access-Control-Expose-Headers", "Content-Length,Content-Range"),
("Access-Control-Allow-Credentials", "true"),
]
uvicorn.run(
"main:app", host="localhost", port=8080, headers=headers
) # , ssl_keyfile="discours.key", ssl_certfile="discours.crt", reload=True)
elif x == "migrate":
from migration import migrate
migrate()
else:
uvicorn.run("main:app", host="0.0.0.0", port=PORT)

2
services/auth/roles.py
View File

@ -1,5 +1,7 @@
import asyncio
from sqlalchemy.orm import selectinload
from orm.rbac import Role

2
services/auth/users.py
View File

@ -1,5 +1,7 @@
import asyncio
from sqlalchemy.orm import selectinload
from orm.user import User

17
services/main.py Normal file
View File

@ -0,0 +1,17 @@
from services.stat.viewed import ViewedStorage
from services.stat.reacted import ReactedStorage
from services.auth.roles import RoleStorage
from services.auth.users import UserStorage
from services.zine.topics import TopicStorage
from base.orm import local_session
async def storages_init():
with local_session() as session:
print('[main] initialize storages')
ViewedStorage.init(session)
ReactedStorage.init(session)
RoleStorage.init(session)
UserStorage.init(session)
TopicStorage.init(session)
session.commit()

58
services/stat/reacted.py
View File

@ -1,11 +1,13 @@
import asyncio
from datetime import datetime
from enum import Enum as Enumeration
from sqlalchemy import Column, DateTime, ForeignKey, Boolean
from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy.types import Enum as ColumnEnum
from base.orm import Base, local_session
from orm.topic import ShoutTopic
from enum import Enum as Enumeration
from sqlalchemy.types import Enum as ColumnEnum
class ReactionKind(Enumeration):
@ -139,26 +141,23 @@ class ReactedStorage:
self = ReactedStorage
async with self.lock:
reactions = self.reacted["shouts"].get(reaction.shout)
if reaction.replyTo:
reactions = self.reacted["reactions"].get(reaction.id)
for r in reactions.values():
r = {
"day": datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0
),
"reaction": reaction.id,
"kind": reaction.kind,
"shout": reaction.shout,
}
if reaction.replyTo:
r["replyTo"] = reaction.replyTo
if reaction.body:
r["comment"] = True
reaction: ReactedByDay = ReactedByDay.create(**r) # type: ignore
self.reacted["shouts"][reaction.shout] = self.reacted["shouts"].get(
reaction.shout, []
)
reactions = {}
# iterate sibling reactions
reactions = self.reacted["shouts"].get(reaction.shout, {})
for r in reactions.values():
reaction = ReactedByDay.create({
"day": datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0
),
"reaction": r.id,
"kind": r.kind,
"shout": r.shout,
"comment": bool(r.body),
"replyTo": r.replyTo
})
# renew sorted by shouts store
self.reacted["shouts"][reaction.shout] = self.reacted["shouts"].get(reaction.shout, [])
self.reacted["shouts"][reaction.shout].append(reaction)
if reaction.replyTo:
self.reacted["reaction"][reaction.replyTo] = self.reacted[
@ -169,11 +168,12 @@ class ReactedStorage:
"reactions"
].get(reaction.replyTo, 0) + kind_to_rate(reaction.kind)
else:
# rate only by root reactions on shout
self.rating["shouts"][reaction.replyTo] = self.rating["shouts"].get(
reaction.shout, 0
) + kind_to_rate(reaction.kind)
flag_modified(r, "value")
flag_modified(reaction, "value")
@staticmethod
def init(session):
@ -218,16 +218,20 @@ class ReactedStorage:
async def flush_changes(session):
self = ReactedStorage
async with self.lock:
for slug in dict(self.reacted['shouts']).keys():
topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == slug).all()
reactions = self.reacted['shouts'].get(slug, [])
for slug in dict(self.reacted["shouts"]).keys():
topics = (
session.query(ShoutTopic.topic)
.where(ShoutTopic.shout == slug)
.all()
)
reactions = self.reacted["shouts"].get(slug, [])
# print('[stat.reacted] shout {' + str(slug) + "}: " + str(len(reactions)))
for ts in list(topics):
tslug = ts[0]
topic_reactions = self.reacted["topics"].get(tslug, [])
topic_reactions += reactions
# print('[stat.reacted] topic {' + str(tslug) + "}: " + str(len(topic_reactions)))
reactions += list(self.reacted['reactions'].values())
reactions += list(self.reacted["reactions"].values())
for reaction in reactions:
if getattr(reaction, "modified", False):
session.add(reaction)

20
services/stat/topicstat.py
View File

@ -1,19 +1,11 @@
import asyncio
from base.orm import local_session
from orm.shout import Shout
from orm.topic import ShoutTopic, TopicFollower
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedStorage
from services.zine.shoutauthor import ShoutAuthorStorage
from orm.topic import ShoutTopic, TopicFollower
def unique(list1):
# insert the list to the set
list_set = set(list1)
# convert the set to the list
unique_list = (list(list_set))
return unique_list
class TopicStat:
@ -27,7 +19,7 @@ class TopicStat:
async def load_stat(session):
self = TopicStat
shout_topics = session.query(ShoutTopic).all()
print('[stat.topics] shout topics amount', len(shout_topics))
print("[stat.topics] shout topics amount", len(shout_topics))
for shout_topic in shout_topics:
# shouts by topics
@ -35,7 +27,11 @@ class TopicStat:
shout = shout_topic.shout
sss = set(self.shouts_by_topic.get(topic, []))
shout = session.query(Shout).where(Shout.slug == shout).first()
sss.union([shout, ])
sss.union(
[
shout,
]
)
self.shouts_by_topic[topic] = list(sss)
# authors by topics

2
services/stat/viewed.py
View File

@ -1,7 +1,9 @@
import asyncio
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer
from sqlalchemy.orm.attributes import flag_modified
from base.orm import Base, local_session
from orm.topic import ShoutTopic

3
services/zine/gittask.py
View File

@ -1,6 +1,7 @@
import asyncio
import subprocess
from pathlib import Path
import asyncio
from settings import SHOUTS_REPO

1
services/zine/shoutauthor.py
View File

@ -1,4 +1,5 @@
import asyncio
from base.orm import local_session
from orm.shout import ShoutAuthor

168
services/zine/shoutscache.py
View File

@ -1,7 +1,9 @@
import asyncio
from datetime import datetime, timedelta
from sqlalchemy import and_, desc, func, select
from sqlalchemy.orm import selectinload
from base.orm import local_session
from orm.reaction import Reaction
from orm.shout import Shout, ShoutAuthor, ShoutTopic
@ -27,6 +29,7 @@ class ShoutsCache:
top_month = []
top_overall = []
top_viewed = []
top_commented = []
by_author = {}
by_topic = {}
@ -34,14 +37,17 @@ class ShoutsCache:
@staticmethod
async def prepare_recent_published():
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.where(bool(Shout.publishedAt))
.group_by(Shout.slug)
.order_by(desc("publishedAt"))
.limit(ShoutsCache.limit)
))
shouts = await prepare_shouts(
session,
(
select(Shout)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.where(bool(Shout.publishedAt))
.group_by(Shout.slug)
.order_by(desc("publishedAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock:
ShoutsCache.recent_published = shouts
print("[zine.cache] %d recently published shouts " % len(shouts))
@ -49,14 +55,17 @@ class ShoutsCache:
@staticmethod
async def prepare_recent_all():
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("createdAt"))
.limit(ShoutsCache.limit)
))
shouts = await prepare_shouts(
session,
(
select(Shout)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("createdAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock:
ShoutsCache.recent_all = shouts
print("[zine.cache] %d recently created shouts " % len(shouts))
@ -64,18 +73,23 @@ class ShoutsCache:
@staticmethod
async def prepare_recent_reacted():
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout, func.max(Reaction.createdAt).label("reactionCreatedAt"))
.options(
selectinload(Shout.authors),
selectinload(Shout.topics),
)
.join(Reaction, Reaction.shout == Shout.slug)
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reactionCreatedAt"))
.limit(ShoutsCache.limit)
))
shouts = await prepare_shouts(
session,
(
select(
Shout, func.max(Reaction.createdAt).label("reactionCreatedAt")
)
.options(
selectinload(Shout.authors),
selectinload(Shout.topics),
)
.join(Reaction, Reaction.shout == Shout.slug)
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reactionCreatedAt"))
.limit(ShoutsCache.limit)
),
)
async with ShoutsCache.lock:
ShoutsCache.recent_reacted = shouts
print("[zine.cache] %d recently reacted shouts " % len(shouts))
@ -84,20 +98,23 @@ class ShoutsCache:
async def prepare_top_overall():
with local_session() as session:
# with reacted times counter
shouts = await prepare_shouts(session, (
select(Shout, func.count(Reaction.id).label("reacted"))
.options(
selectinload(Shout.authors),
selectinload(Shout.topics),
selectinload(Shout.reactions),
)
.join(Reaction)
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reacted"))
.limit(ShoutsCache.limit)
))
shouts.sort(key=lambda s: s.stats['rating'], reverse=True)
shouts = await prepare_shouts(
session,
(
select(Shout, func.count(Reaction.id).label("reacted"))
.options(
selectinload(Shout.authors),
selectinload(Shout.topics),
selectinload(Shout.reactions),
)
.join(Reaction)
.where(and_(bool(Shout.publishedAt), bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reacted"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["rating"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top shouts " % len(shouts))
ShoutsCache.top_overall = shouts
@ -106,34 +123,61 @@ class ShoutsCache:
async def prepare_top_month():
month_ago = datetime.now() - timedelta(days=30)
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout, func.count(Reaction.id).label("reacted"))
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(Reaction)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reacted"))
.limit(ShoutsCache.limit)
))
shouts.sort(key=lambda s: s.stats['rating'], reverse=True)
shouts = await prepare_shouts(
session,
(
select(Shout, func.count(Reaction.id).label("reacted"))
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(Reaction)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("reacted"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["rating"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top month shouts " % len(shouts))
ShoutsCache.top_month = shouts
@staticmethod
async def prepare_top_commented():
month_ago = datetime.now() - timedelta(days=30)
with local_session() as session:
shouts = await prepare_shouts(
session,
(
select(Shout, Reaction)
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(Reaction)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("commented"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["commented"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top commented shouts " % len(shouts))
ShoutsCache.top_viewed = shouts
@staticmethod
async def prepare_top_viewed():
month_ago = datetime.now() - timedelta(days=30)
with local_session() as session:
shouts = await prepare_shouts(session, (
select(Shout, func.sum(ViewedByDay.value).label("viewed"))
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(ViewedByDay)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("viewed"))
.limit(ShoutsCache.limit)
))
shouts.sort(key=lambda s: s.stats['viewed'], reverse=True)
shouts = await prepare_shouts(
session,
(
select(Shout, func.sum(ViewedByDay.value).label("viewed"))
.options(selectinload(Shout.authors), selectinload(Shout.topics))
.join(ViewedByDay)
.where(and_(Shout.createdAt > month_ago, bool(Reaction.deletedAt)))
.group_by(Shout.slug)
.order_by(desc("viewed"))
.limit(ShoutsCache.limit)
),
)
shouts.sort(key=lambda s: s.stats["viewed"], reverse=True)
async with ShoutsCache.lock:
print("[zine.cache] %d top viewed shouts " % len(shouts))
ShoutsCache.top_viewed = shouts

9
settings.py
View File

@ -5,8 +5,7 @@ INBOX_SERVICE_PORT = 8081
BACKEND_URL = environ.get("BACKEND_URL") or "https://localhost:8080"
OAUTH_CALLBACK_URL = environ.get("OAUTH_CALLBACK_URL") or "https://localhost:8080"
RESET_PWD_URL = environ.get("RESET_PWD_URL") or "https://localhost:8080/reset_pwd"
CONFIRM_EMAIL_URL = environ.get("CONFIRM_EMAIL_URL") or "https://new.discours.io"
CONFIRM_EMAIL_URL = environ.get("AUTH_CONFIRM_URL") or BACKEND_URL + "/confirm"
ERROR_URL_ON_FRONTEND = (
environ.get("ERROR_URL_ON_FRONTEND") or "https://new.discours.io"
)
@ -17,9 +16,9 @@ DB_URL = (
)
JWT_ALGORITHM = "HS256"
JWT_SECRET_KEY = "8f1bd7696ffb482d8486dfbc6e7d16dd-secret-key"
JWT_LIFE_SPAN = 24 * 60 * 60 # seconds
JWT_AUTH_HEADER = "Auth"
EMAIL_TOKEN_LIFE_SPAN = 1 * 60 * 60 # seconds
SESSION_TOKEN_HEADER = "Auth"
SESSION_TOKEN_LIFE_SPAN = 24 * 60 * 60 # seconds
ONETIME_TOKEN_LIFE_SPAN = 1 * 60 * 60 # seconds
REDIS_URL = environ.get("REDIS_URL") or "redis://127.0.0.1"
MAILGUN_API_KEY = environ.get("MAILGUN_API_KEY")

16
validations/auth.py Normal file
View File

@ -0,0 +1,16 @@
from datetime import datetime
from typing import Optional, Text
from pydantic import BaseModel
class AuthInput(BaseModel):
id: Optional[int]
username: Optional[Text]
password: Optional[Text]
class TokenPayload(BaseModel):
user_id: int
exp: datetime
iat: datetime