discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

schemaup

Browse Source
This commit is contained in:
tonyrewin 2022-11-30 23:20:08 +03:00
commit c86549507e
35 changed files with 617 additions and 782 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -148,3 +148,4 @@ dump
*dump.sql *dump.sql
*.csv *.csv
dev-server-status.txt dev-server-status.txt
/resetdb.sh

4
CHECKS
View File

@ -1,5 +1,5 @@
WAIT=30 WAIT=10
TIMEOUT=10 TIMEOUT=10
ATTEMPTS=60 # 60 * 30 = 30 min ATTEMPTS=30 # 10 * 30 = 5 min
/ Playground / Playground

31
auth/authenticate.py
View File

@ -2,11 +2,14 @@ from functools import wraps
from typing import Optional, Tuple from typing import Optional, Tuple
from graphql.type import GraphQLResolveInfo from graphql.type import GraphQLResolveInfo
from sqlalchemy.orm import joinedload, exc
from starlette.authentication import AuthenticationBackend from starlette.authentication import AuthenticationBackend
from starlette.requests import HTTPConnection from starlette.requests import HTTPConnection
from auth.credentials import AuthCredentials, AuthUser from auth.credentials import AuthCredentials, AuthUser
from services.auth.users import UserStorage from base.orm import local_session
from orm import User, Role
from settings import SESSION_TOKEN_HEADER from settings import SESSION_TOKEN_HEADER
from auth.tokenstorage import SessionToken from auth.tokenstorage import SessionToken
from base.exceptions import InvalidToken, OperationNotAllowed, Unauthorized from base.exceptions import InvalidToken, OperationNotAllowed, Unauthorized
@ -32,10 +35,26 @@ class JWTAuthenticate(AuthenticationBackend):
payload = await SessionToken.verify(token) payload = await SessionToken.verify(token)
if payload is None: if payload is None:
return AuthCredentials(scopes=[]), AuthUser(user_id=None) return AuthCredentials(scopes=[]), AuthUser(user_id=None)
user = await UserStorage.get_user(payload.user_id)
with local_session() as session:
try:
user = (
session.query(User).options(
joinedload(User.roles),
joinedload(Role.permissions),
joinedload(User.ratings)
).filter(
User.id == id
).one()
)
except exc.NoResultFound:
user = None
if not user: if not user:
return AuthCredentials(scopes=[]), AuthUser(user_id=None) return AuthCredentials(scopes=[]), AuthUser(user_id=None)
scopes = await user.get_permission()
scopes = user.get_permission()
return ( return (
AuthCredentials( AuthCredentials(
user_id=payload.user_id, user_id=payload.user_id,
@ -46,10 +65,10 @@ class JWTAuthenticate(AuthenticationBackend):
) )
else: else:
InvalidToken("please try again") InvalidToken("please try again")
except Exception as exc: except Exception as e:
print("[auth.authenticate] session token verify error") print("[auth.authenticate] session token verify error")
print(exc) print(e)
return AuthCredentials(scopes=[], error_message=str(exc)), AuthUser(user_id=None) return AuthCredentials(scopes=[], error_message=str(e)), AuthUser(user_id=None)
def login_required(func): def login_required(func):

12
main.py
View File

@ -16,12 +16,8 @@ from base.redis import redis
from base.resolvers import resolvers from base.resolvers import resolvers
from resolvers.auth import confirm_email_handler from resolvers.auth import confirm_email_handler
from services.main import storages_init from services.main import storages_init
# from services.stat.reacted import ReactedStorage
from services.stat.topicstat import TopicStat
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
from services.zine.topics import TopicStorage
from services.zine.gittask import GitTask from services.zine.gittask import GitTask
from services.zine.shoutauthor import ShoutAuthorStorage
from settings import DEV_SERVER_STATUS_FILE_NAME from settings import DEV_SERVER_STATUS_FILE_NAME
import_module("resolvers") import_module("resolvers")
@ -37,16 +33,8 @@ async def start_up():
init_tables() init_tables()
await redis.connect() await redis.connect()
await storages_init() await storages_init()
topics_random_work = asyncio.create_task(TopicStorage().worker())
print(topics_random_work)
views_stat_task = asyncio.create_task(ViewedStorage().worker()) views_stat_task = asyncio.create_task(ViewedStorage().worker())
print(views_stat_task) print(views_stat_task)
# reacted_storage_task = asyncio.create_task(ReactedStorage.worker())
# print(reacted_storage_task)
shout_author_task = asyncio.create_task(ShoutAuthorStorage.worker())
print(shout_author_task)
topic_stat_task = asyncio.create_task(TopicStat.worker())
print(topic_stat_task)
git_task = asyncio.create_task(GitTask.git_task_worker()) git_task = asyncio.create_task(GitTask.git_task_worker())
print(git_task) print(git_task)

27
migration/__init__.py
View File

@ -96,16 +96,16 @@ async def shouts_handle(storage, args):
continue continue
# migrate # migrate
shout = await migrateShout(entry, storage) shout_dict = await migrateShout(entry, storage)
if shout: if shout_dict:
storage["shouts"]["by_oid"][entry["_id"]] = shout storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
storage["shouts"]["by_slug"][shout["slug"]] = shout storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict
# shouts.topics # shouts.topics
if not shout["topics"]: if not shout_dict["topics"]:
print("[migration] no topics!") print("[migration] no topics!")
# with author # with author
author: str = shout["authors"][0].dict() author = shout_dict["authors"][0]
if author["slug"] == "discours": if author["slug"] == "discours":
discours_author += 1 discours_author += 1
if author["slug"] == "anonymous": if author["slug"] == "anonymous":
@ -114,19 +114,20 @@ async def shouts_handle(storage, args):
if entry.get("published"): if entry.get("published"):
if "mdx" in args: if "mdx" in args:
export_mdx(shout) export_mdx(shout_dict)
pub_counter += 1 pub_counter += 1
# print main counter # print main counter
counter += 1 counter += 1
line = str(counter + 1) + ": " + shout["slug"] + " @" + author["slug"] print('[migration] shouts_handle %d: %s @%s' % (
print(line) (counter + 1), shout_dict["slug"], author["slug"]
))
b = bs4.BeautifulSoup(shout["body"], "html.parser") b = bs4.BeautifulSoup(shout_dict["body"], "html.parser")
texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")]
texts = texts + b.findAll(text=True) texts = texts + b.findAll(text=True)
topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts]))
topics_dataset_tlist.append(shout["topics"]) topics_dataset_tlist.append(shout_dict["topics"])
else: else:
ignored += 1 ignored += 1
@ -134,9 +135,7 @@ async def shouts_handle(storage, args):
# ', fmt='%s') # ', fmt='%s')
print("[migration] " + str(counter) + " content items were migrated") print("[migration] " + str(counter) + " content items were migrated")
print("[migration] " + str(ignored) + " content items were ignored")
print("[migration] " + str(pub_counter) + " have been published") print("[migration] " + str(pub_counter) + " have been published")
print("[migration] " + str(discours_author) + " authored by @discours")
print("[migration] " + str(anonymous_author) + " authored by @anonymous") print("[migration] " + str(anonymous_author) + " authored by @anonymous")

50
migration/tables/comments.py
View File

@ -8,12 +8,13 @@ from orm.reaction import Reaction, ReactionKind
from orm.shout import ShoutReactionsFollower from orm.shout import ShoutReactionsFollower
from orm.topic import TopicFollower from orm.topic import TopicFollower
from orm.user import User from orm.user import User
from orm.shout import Shout
# from services.stat.reacted import ReactedStorage # from services.stat.reacted import ReactedStorage
ts = datetime.now(tz=timezone.utc) ts = datetime.now(tz=timezone.utc)
def auto_followers(session, shout_dict, reaction_dict): def auto_followers(session, topics, reaction_dict):
# creating shout's reactions following for reaction author # creating shout's reactions following for reaction author
following1 = session.query( following1 = session.query(
ShoutReactionsFollower ShoutReactionsFollower
@ -30,18 +31,18 @@ def auto_followers(session, shout_dict, reaction_dict):
) )
session.add(following1) session.add(following1)
# creating topics followings for reaction author # creating topics followings for reaction author
for t in shout_dict["topics"]: for t in topics:
tf = session.query( tf = session.query(
TopicFollower TopicFollower
).where( ).where(
TopicFollower.follower == reaction_dict["createdBy"] TopicFollower.follower == reaction_dict["createdBy"]
).filter( ).filter(
TopicFollower.topic == t TopicFollower.topic == t['id']
).first() ).first()
if not tf: if not tf:
topic_following = TopicFollower.create( topic_following = TopicFollower.create(
follower=reaction_dict["createdBy"], follower=reaction_dict["createdBy"],
topic=t, topic=t['id'],
auto=True auto=True
) )
session.add(topic_following) session.add(topic_following)
@ -60,7 +61,7 @@ def migrate_ratings(session, entry, reaction_dict):
"kind": ReactionKind.LIKE "kind": ReactionKind.LIKE
if comment_rating_old["value"] > 0 if comment_rating_old["value"] > 0
else ReactionKind.DISLIKE, else ReactionKind.DISLIKE,
"createdBy": rater.slug if rater else "anonymous", "createdBy": rater.id if rater else 1,
} }
cts = comment_rating_old.get("createdAt") cts = comment_rating_old.get("createdAt")
if cts: if cts:
@ -108,9 +109,7 @@ async def migrate(entry, storage):
"updatedAt": "2020-05-27 19:22:57.091000+00:00", "updatedAt": "2020-05-27 19:22:57.091000+00:00",
"updatedBy": "0" "updatedBy": "0"
} }
-> ->
type Reaction { type Reaction {
id: Int! id: Int!
shout: Shout! shout: Shout!
@ -143,30 +142,41 @@ async def migrate(entry, storage):
raise Exception raise Exception
return return
else: else:
stage = "started"
reaction = None
with local_session() as session: with local_session() as session:
author = session.query(User).filter(User.oid == entry["createdBy"]).first() author = session.query(User).filter(User.oid == entry["createdBy"]).first()
shout_dict = storage["shouts"]["by_oid"][shout_oid] old_shout = storage["shouts"]["by_oid"].get(shout_oid)
if shout_dict: if not old_shout:
reaction_dict["shout"] = shout_dict["slug"] raise Exception("no old shout in storage")
reaction_dict["createdBy"] = author.slug if author else "discours" else:
stage = "author and old id found"
try:
shout = session.query(
Shout
).where(Shout.slug == old_shout["slug"]).one()
if shout:
reaction_dict["shout"] = shout.id
reaction_dict["createdBy"] = author.id if author else 1
reaction_dict["kind"] = ReactionKind.COMMENT reaction_dict["kind"] = ReactionKind.COMMENT
# creating reaction from old comment # creating reaction from old comment
reaction = Reaction.create(**reaction_dict) reaction = Reaction.create(**reaction_dict)
session.add(reaction) session.add(reaction)
# await ReactedStorage.react(reaction) # session.commit()
stage = "new reaction commited"
reaction_dict = reaction.dict() reaction_dict = reaction.dict()
topics = [t.dict() for t in shout.topics]
auto_followers(session, topics, reaction_dict)
auto_followers(session, shout_dict, reaction_dict) migrate_ratings(session, entry, reaction_dict)
migrate_ratings(session, shout_dict, reaction_dict)
else:
print(
"[migration] error: cannot find shout for comment %r"
% reaction_dict
)
return reaction return reaction
except Exception as e:
print(e)
print(reaction)
raise Exception(stage)
return
def migrate_2stage(old_comment, idmap): def migrate_2stage(old_comment, idmap):

118
migration/tables/content_items.py
View File

@ -8,9 +8,10 @@ from migration.extract import extract_html, extract_media
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
from orm.user import User from orm.user import User
from orm.topic import TopicFollower from orm.topic import TopicFollower, Topic
# from services.stat.reacted import ReactedStorage # from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
import re
OLD_DATE = "2016-03-05 22:22:00.350000" OLD_DATE = "2016-03-05 22:22:00.350000"
ts = datetime.now(tz=timezone.utc) ts = datetime.now(tz=timezone.utc)
@ -22,6 +23,8 @@ type2layout = {
"Image": "image", "Image": "image",
} }
anondict = {"slug": "anonymous", "id": 1, "name": "Аноним"}
def get_shout_slug(entry): def get_shout_slug(entry):
slug = entry.get("slug", "") slug = entry.get("slug", "")
@ -30,6 +33,7 @@ def get_shout_slug(entry):
slug = friend.get("slug", "") slug = friend.get("slug", "")
if slug: if slug:
break break
slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
return slug return slug
@ -40,13 +44,8 @@ def create_author_from_app(app):
user = session.query(User).where(User.email == app['email']).first() user = session.query(User).where(User.email == app['email']).first()
if not user: if not user:
name = app.get('name') name = app.get('name')
slug = ( slug = translit(name, "ru", reversed=True).lower()
translit(name, "ru", reversed=True) slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
.replace(" ", "-")
.replace("'", "")
.replace(".", "-")
.lower()
)
# check if nameslug is used # check if nameslug is used
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
# get slug from email # get slug from email
@ -82,18 +81,33 @@ def create_author_from_app(app):
return userdata return userdata
async def create_shout(shout_dict, userslug): async def create_shout(shout_dict, user):
s = Shout.create(**shout_dict) s = Shout.create(**shout_dict)
with local_session() as session: with local_session() as session:
srf = session.query(ShoutReactionsFollower).where( srf = session.query(ShoutReactionsFollower).where(
ShoutReactionsFollower.shout == s.slug ShoutReactionsFollower.shout == s.id
).filter( ).filter(
ShoutReactionsFollower.follower == userslug ShoutReactionsFollower.follower == user.id
).first() ).first()
if not srf: if not srf:
srf = ShoutReactionsFollower.create(shout=s.slug, follower=userslug, auto=True) srf = ShoutReactionsFollower.create(shout=s.id, follower=user.id, auto=True)
session.add(srf) session.add(srf)
session.commit() session.commit()
return s
def get_userdata(entry, storage):
user_oid = entry.get("createdBy", "")
userdata = None
app = entry.get("application")
if app:
userdata = create_author_from_app(app) or anondict
else:
userdata = storage["users"]["by_oid"].get(user_oid) or anondict
slug = userdata.get("slug")
slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
userdata["slug"] = slug
return userdata, user_oid
def get_userdata(entry, storage): def get_userdata(entry, storage):
@ -109,12 +123,12 @@ def get_userdata(entry, storage):
async def migrate(entry, storage): async def migrate(entry, storage):
userslug, userdata, user_oid = get_userdata(entry, storage) userdata, user_oid = get_userdata(entry, storage)
user = await get_user(userslug, userdata, storage, user_oid) user = await get_user(userdata, storage, user_oid)
r = { r = {
"layout": type2layout[entry["type"]], "layout": type2layout[entry["type"]],
"title": entry["title"], "title": entry["title"],
"authors": [userslug, ], "authors": [userdata["slug"], ],
"slug": get_shout_slug(entry), "slug": get_shout_slug(entry),
"cover": ( "cover": (
"https://assets.discours.io/unsafe/1600x/" + "https://assets.discours.io/unsafe/1600x/" +
@ -125,7 +139,7 @@ async def migrate(entry, storage):
"deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None, "deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None,
"createdAt": date_parse(entry.get("createdAt", OLD_DATE)), "createdAt": date_parse(entry.get("createdAt", OLD_DATE)),
"updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts, "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts,
"topics": await add_topics_follower(entry, storage, userslug), "topics": await add_topics_follower(entry, storage, user),
"body": extract_html(entry) "body": extract_html(entry)
} }
@ -136,7 +150,7 @@ async def migrate(entry, storage):
if entry.get("published"): if entry.get("published"):
with local_session() as session: with local_session() as session:
# update user.emailConfirmed if published # update user.emailConfirmed if published
author = session.query(User).where(User.slug == userslug).first() author = session.query(User).where(User.slug == userdata["slug"]).first()
author.emailConfirmed = True author.emailConfirmed = True
session.add(author) session.add(author)
session.commit() session.commit()
@ -153,13 +167,18 @@ async def migrate(entry, storage):
del shout_dict["topics"] del shout_dict["topics"]
try: try:
# save shout to db # save shout to db
await create_shout(shout_dict, userslug) shout_dict["oid"] = entry.get("_id", "")
shout = await create_shout(shout_dict, user)
except IntegrityError as e: except IntegrityError as e:
print(e) print('[migration] create_shout integrity error', e)
await resolve_create_shout(shout_dict, userslug) shout = await resolve_create_shout(shout_dict, userdata["slug"])
except Exception as e: except Exception as e:
raise Exception(e) raise Exception(e)
# udpate data
shout_dict = shout.dict()
shout_dict["authors"] = [user.dict(), ]
# shout topics aftermath # shout topics aftermath
shout_dict["topics"] = await topics_aftermath(r, storage) shout_dict["topics"] = await topics_aftermath(r, storage)
@ -170,13 +189,12 @@ async def migrate(entry, storage):
await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1)) await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1))
# del shout_dict['ratings'] # del shout_dict['ratings']
shout_dict["oid"] = entry.get("_id", "")
storage["shouts"]["by_oid"][entry["_id"]] = shout_dict storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict
return shout_dict return shout_dict
async def add_topics_follower(entry, storage, userslug): async def add_topics_follower(entry, storage, user):
topics = set([]) topics = set([])
category = entry.get("category") category = entry.get("category")
topics_by_oid = storage["topics"]["by_oid"] topics_by_oid = storage["topics"]["by_oid"]
@ -188,25 +206,26 @@ async def add_topics_follower(entry, storage, userslug):
ttt = list(topics) ttt = list(topics)
# add author as TopicFollower # add author as TopicFollower
with local_session() as session: with local_session() as session:
for tpc in topics: for tpcslug in topics:
try: try:
tpc = session.query(Topic).where(Topic.slug == tpcslug).first()
tf = session.query( tf = session.query(
TopicFollower TopicFollower
).where( ).where(
TopicFollower.follower == userslug TopicFollower.follower == user.id
).filter( ).filter(
TopicFollower.topic == tpc TopicFollower.topic == tpc.id
).first() ).first()
if not tf: if not tf:
tf = TopicFollower.create( tf = TopicFollower.create(
topic=tpc, topic=tpc.id,
follower=userslug, follower=user.id,
auto=True auto=True
) )
session.add(tf) session.add(tf)
session.commit() session.commit()
except IntegrityError: except IntegrityError:
print('[migration.shout] hidden by topic ' + tpc) print('[migration.shout] hidden by topic ' + tpc.slug)
# main topic # main topic
maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug")) maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug"))
if maintopic in ttt: if maintopic in ttt:
@ -215,19 +234,24 @@ async def add_topics_follower(entry, storage, userslug):
return ttt return ttt
async def get_user(userslug, userdata, storage, oid): async def get_user(userdata, storage, oid):
user = None user = None
with local_session() as session: with local_session() as session:
if not user and userslug: uid = userdata.get("id")
user = session.query(User).filter(User.slug == userslug).first() if uid:
if not user and userdata: user = session.query(User).filter(User.id == uid).first()
elif userdata:
try: try:
userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-") slug = userdata["slug"].lower().strip()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
userdata["slug"] = slug
user = User.create(**userdata) user = User.create(**userdata)
session.add(user) session.add(user)
session.commit() session.commit()
except IntegrityError: except IntegrityError:
print("[migration] user error: " + userdata) print("[migration] user creating with slug %s" % userdata["slug"])
print("[migration] from userdata: %r" % userdata)
raise Exception("[migration] cannot create user in content_items.get_user()")
userdata["id"] = user.id userdata["id"] = user.id
userdata["createdAt"] = user.createdAt userdata["createdAt"] = user.createdAt
storage["users"]["by_slug"][userdata["slug"]] = userdata storage["users"]["by_slug"][userdata["slug"]] = userdata
@ -269,6 +293,7 @@ async def resolve_create_shout(shout_dict, userslug):
print("[migration] something went wrong with shout: \n%r" % shout_dict) print("[migration] something went wrong with shout: \n%r" % shout_dict)
raise Exception("") raise Exception("")
session.commit() session.commit()
return s
async def topics_aftermath(entry, storage): async def topics_aftermath(entry, storage):
@ -276,27 +301,35 @@ async def topics_aftermath(entry, storage):
for tpc in filter(lambda x: bool(x), entry["topics"]): for tpc in filter(lambda x: bool(x), entry["topics"]):
oldslug = tpc oldslug = tpc
newslug = storage["replacements"].get(oldslug, oldslug) newslug = storage["replacements"].get(oldslug, oldslug)
if newslug: if newslug:
with local_session() as session: with local_session() as session:
shout = session.query(Shout).where(Shout.slug == entry["slug"]).one()
new_topic = session.query(Topic).where(Topic.slug == newslug).one()
shout_topic_old = ( shout_topic_old = (
session.query(ShoutTopic) session.query(ShoutTopic)
.filter(ShoutTopic.shout == entry["slug"]) .join(Shout)
.filter(ShoutTopic.topic == oldslug) .join(Topic)
.filter(Shout.slug == entry["slug"])
.filter(Topic.slug == oldslug)
.first() .first()
) )
if shout_topic_old: if shout_topic_old:
shout_topic_old.update({"slug": newslug}) shout_topic_old.update({"topic": new_topic.id})
else: else:
shout_topic_new = ( shout_topic_new = (
session.query(ShoutTopic) session.query(ShoutTopic)
.filter(ShoutTopic.shout == entry["slug"]) .join(Shout)
.filter(ShoutTopic.topic == newslug) .join(Topic)
.filter(Shout.slug == entry["slug"])
.filter(Topic.slug == newslug)
.first() .first()
) )
if not shout_topic_new: if not shout_topic_new:
try: try:
ShoutTopic.create( ShoutTopic.create(
**{"shout": entry["slug"], "topic": newslug} **{"shout": shout.id, "topic": new_topic.id}
) )
except Exception: except Exception:
print("[migration] shout topic error: " + newslug) print("[migration] shout topic error: " + newslug)
@ -318,14 +351,15 @@ async def content_ratings_to_reactions(entry, slug):
.filter(User.oid == content_rating["createdBy"]) .filter(User.oid == content_rating["createdBy"])
.first() .first()
) or User.default_user ) or User.default_user
shout = session.query(Shout).where(Shout.slug == slug).first()
cts = content_rating.get("createdAt") cts = content_rating.get("createdAt")
reaction_dict = { reaction_dict = {
"createdAt": date_parse(cts) if cts else None, "createdAt": date_parse(cts) if cts else None,
"kind": ReactionKind.LIKE "kind": ReactionKind.LIKE
if content_rating["value"] > 0 if content_rating["value"] > 0
else ReactionKind.DISLIKE, else ReactionKind.DISLIKE,
"createdBy": rater.slug, "createdBy": rater.id,
"shout": slug "shout": shout.id
} }
reaction = ( reaction = (
session.query(Reaction) session.query(Reaction)

5
migration/tables/topics.py
View File

@ -9,9 +9,10 @@ def migrate(entry):
topic_dict = { topic_dict = {
"slug": entry["slug"], "slug": entry["slug"],
"oid": entry["_id"], "oid": entry["_id"],
"title": entry["title"].replace(" ", " ") "title": entry["title"].replace(" ", " "),
"body": extract_md(html2text(body_orig), entry["_id"])
} }
topic_dict["body"] = extract_md(html2text(body_orig), entry["_id"])
with local_session() as session: with local_session() as session:
slug = topic_dict["slug"] slug = topic_dict["slug"]
topic = session.query(Topic).filter(Topic.slug == slug).first() or Topic.create( topic = session.query(Topic).filter(Topic.slug == slug).first() or Topic.create(

30
migration/tables/users.py
View File

@ -1,7 +1,7 @@
from dateutil.parser import parse from dateutil.parser import parse
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import re
from base.orm import local_session from base.orm import local_session
from orm.user import AuthorFollower, User, UserRating from orm.user import AuthorFollower, User, UserRating
@ -23,17 +23,18 @@ def migrate(entry):
"notifications": [], "notifications": [],
"links": [], "links": [],
"name": "anonymous", "name": "anonymous",
"password": entry["services"]["password"].get("bcrypt")
} }
user_dict["password"] = entry["services"]["password"].get("bcrypt")
if "updatedAt" in entry: if "updatedAt" in entry:
user_dict["updatedAt"] = parse(entry["updatedAt"]) user_dict["updatedAt"] = parse(entry["updatedAt"])
if "wasOnineAt" in entry: if "wasOnineAt" in entry:
user_dict["lastSeen"] = parse(entry["wasOnlineAt"]) user_dict["lastSeen"] = parse(entry["wasOnlineAt"])
if entry.get("profile"): if entry.get("profile"):
# slug # slug
user_dict["slug"] = ( slug = entry["profile"].get("path").lower()
entry["profile"].get("path").lower().replace(" ", "-").strip() slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip()
) user_dict["slug"] = slug
bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text
if bio.startswith('<'): if bio.startswith('<'):
print('[migration] bio! ' + bio) print('[migration] bio! ' + bio)
@ -114,18 +115,23 @@ def migrate_2stage(entry, id_map):
continue continue
oid = entry["_id"] oid = entry["_id"]
author_slug = id_map.get(oid) author_slug = id_map.get(oid)
user_rating_dict = {
"value": rating_entry["value"],
"rater": rater_slug,
"user": author_slug,
}
with local_session() as session: with local_session() as session:
try: try:
rater = session.query(User).where(User.slug == rater_slug).one()
user = session.query(User).where(User.slug == author_slug).one()
user_rating_dict = {
"value": rating_entry["value"],
"raterId": rater.id,
"user": user.id,
}
user_rating = UserRating.create(**user_rating_dict) user_rating = UserRating.create(**user_rating_dict)
if user_rating_dict['value'] > 0: if user_rating_dict['value'] > 0:
af = AuthorFollower.create( af = AuthorFollower.create(
author=user_rating_dict['user'], author=user.id,
follower=user_rating_dict['rater'], follower=rater.id,
auto=True auto=True
) )
session.add(af) session.add(af)

2
orm/__init__.py
View File

@ -21,7 +21,7 @@ __all__ = [
"TopicFollower", "TopicFollower",
"Notification", "Notification",
"Reaction", "Reaction",
"UserRating" "UserRating",
"ViewedEntry" "ViewedEntry"
] ]

17
orm/collab.py
View File

@ -11,23 +11,16 @@ class CollabAuthor(Base):
id = None # type: ignore id = None # type: ignore
collab = Column(ForeignKey("collab.id"), primary_key=True) collab = Column(ForeignKey("collab.id"), primary_key=True)
author = Column(ForeignKey("user.slug"), primary_key=True) author = Column(ForeignKey("user.id"), primary_key=True)
invitedBy = Column(ForeignKey("user.slug")) accepted = Column(Boolean, default=False)
class CollabInvited(Base):
__tablename__ = "collab_invited"
id = None # type: ignore
collab = Column(ForeignKey("collab.id"), primary_key=True)
author = Column(ForeignKey("user.slug"), primary_key=True)
invitedBy = Column(ForeignKey("user.slug"))
class Collab(Base): class Collab(Base):
__tablename__ = "collab" __tablename__ = "collab"
shout = Column(ForeignKey("shout.id"), primary_key=True) title = Column(String, nullable=True, comment="Title")
body = Column(String, nullable=True, comment="Body")
pic = Column(String, nullable=True, comment="Picture")
authors = relationship(lambda: User, secondary=CollabAuthor.__tablename__) authors = relationship(lambda: User, secondary=CollabAuthor.__tablename__)
invites = relationship(lambda: User, secondary=CollabInvited.__tablename__) invites = relationship(lambda: User, secondary=CollabInvited.__tablename__)
createdAt = Column(DateTime, default=datetime.now, comment="Created At") createdAt = Column(DateTime, default=datetime.now, comment="Created At")

4
orm/collection.py
View File

@ -9,8 +9,8 @@ class ShoutCollection(Base):
__tablename__ = "shout_collection" __tablename__ = "shout_collection"
id = None # type: ignore id = None # type: ignore
shout = Column(ForeignKey("shout.slug"), primary_key=True) shout = Column(ForeignKey("shout.id"), primary_key=True)
collection = Column(ForeignKey("collection.slug"), primary_key=True) collectionId = Column(ForeignKey("collection.id"), primary_key=True)
class Collection(Base): class Collection(Base):

4
orm/community.py
View File

@ -8,8 +8,8 @@ class CommunityFollower(Base):
__tablename__ = "community_followers" __tablename__ = "community_followers"
id = None # type: ignore id = None # type: ignore
follower = Column(ForeignKey("user.slug"), primary_key=True) follower = Column(ForeignKey("user.id"), primary_key=True)
community = Column(ForeignKey("community.slug"), primary_key=True) communityId = Column(ForeignKey("community.id"), primary_key=True)
joinedAt = Column( joinedAt = Column(
DateTime, nullable=False, default=datetime.now, comment="Created at" DateTime, nullable=False, default=datetime.now, comment="Created at"
) )

22
orm/rbac.py
View File

@ -122,7 +122,7 @@ class Operation(Base):
class Resource(Base): class Resource(Base):
__tablename__ = "resource" __tablename__ = "resource"
resource_class = Column( resourceClass = Column(
String, nullable=False, unique=True, comment="Resource class" String, nullable=False, unique=True, comment="Resource class"
) )
name = Column(String, nullable=False, unique=True, comment="Resource name") name = Column(String, nullable=False, unique=True, comment="Resource name")
@ -134,7 +134,7 @@ class Resource(Base):
for res in ["shout", "topic", "reaction", "chat", "message", "invite", "community", "user"]: for res in ["shout", "topic", "reaction", "chat", "message", "invite", "community", "user"]:
r = session.query(Resource).filter(Resource.name == res).first() r = session.query(Resource).filter(Resource.name == res).first()
if not r: if not r:
r = Resource.create(name=res, resource_class=res) r = Resource.create(name=res, resourceClass=res)
session.add(r) session.add(r)
session.commit() session.commit()
@ -142,19 +142,19 @@ class Resource(Base):
class Permission(Base): class Permission(Base):
__tablename__ = "permission" __tablename__ = "permission"
__table_args__ = ( __table_args__ = (
UniqueConstraint("role_id", "operation_id", "resource_id"), UniqueConstraint("roleId", "operationId", "resourceId"),
{"extend_existing": True}, {"extend_existing": True},
) )
role_id = Column( roleId = Column(
ForeignKey("role.id", ondelete="CASCADE"), nullable=False, comment="Role" ForeignKey("role.id", ondelete="CASCADE"), nullable=False, comment="Role"
) )
operation_id = Column( operationId = Column(
ForeignKey("operation.id", ondelete="CASCADE"), ForeignKey("operation.id", ondelete="CASCADE"),
nullable=False, nullable=False,
comment="Operation", comment="Operation",
) )
resource_id = Column( resourceId = Column(
ForeignKey("resource.id", ondelete="CASCADE"), ForeignKey("resource.id", ondelete="CASCADE"),
nullable=False, nullable=False,
comment="Resource", comment="Resource",
@ -164,11 +164,11 @@ class Permission(Base):
if __name__ == "__main__": if __name__ == "__main__":
Base.metadata.create_all(engine) Base.metadata.create_all(engine)
ops = [ ops = [
Permission(role_id=1, operation_id=1, resource_id=1), Permission(roleId=1, operationId=1, resourceId=1),
Permission(role_id=1, operation_id=2, resource_id=1), Permission(roleId=1, operationId=2, resourceId=1),
Permission(role_id=1, operation_id=3, resource_id=1), Permission(roleId=1, operationId=3, resourceId=1),
Permission(role_id=1, operation_id=4, resource_id=1), Permission(roleId=1, operationId=4, resourceId=1),
Permission(role_id=2, operation_id=4, resource_id=1), Permission(roleId=2, operationId=4, resourceId=1),
] ]
global_session.add_all(ops) global_session.add_all(ops)
global_session.commit() global_session.commit()

8
orm/reaction.py
View File

@ -28,12 +28,12 @@ class Reaction(Base):
createdAt = Column( createdAt = Column(
DateTime, nullable=False, default=datetime.now, comment="Created at" DateTime, nullable=False, default=datetime.now, comment="Created at"
) )
createdBy = Column(ForeignKey("user.slug"), nullable=False, comment="Sender") createdBy = Column(ForeignKey("user.id"), nullable=False, index=True, comment="Sender")
updatedAt = Column(DateTime, nullable=True, comment="Updated at") updatedAt = Column(DateTime, nullable=True, comment="Updated at")
updatedBy = Column(ForeignKey("user.slug"), nullable=True, comment="Last Editor") updatedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Last Editor")
deletedAt = Column(DateTime, nullable=True, comment="Deleted at") deletedAt = Column(DateTime, nullable=True, comment="Deleted at")
deletedBy = Column(ForeignKey("user.slug"), nullable=True, comment="Deleted by") deletedBy = Column(ForeignKey("user.id"), nullable=True, index=True, comment="Deleted by")
shout = Column(ForeignKey("shout.slug"), nullable=False) shout = Column(ForeignKey("shout.id"), nullable=False, index=True)
replyTo = Column( replyTo = Column(
ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID" ForeignKey("reaction.id"), nullable=True, comment="Reply to reaction ID"
) )

14
orm/shout.py
View File

@ -13,16 +13,16 @@ class ShoutTopic(Base):
__tablename__ = "shout_topic" __tablename__ = "shout_topic"
id = None # type: ignore id = None # type: ignore
shout = Column(ForeignKey("shout.slug"), primary_key=True) shout = Column(ForeignKey("shout.id"), primary_key=True, index=True)
topic = Column(ForeignKey("topic.slug"), primary_key=True) topic = Column(ForeignKey("topic.id"), primary_key=True, index=True)
class ShoutReactionsFollower(Base): class ShoutReactionsFollower(Base):
__tablename__ = "shout_reactions_followers" __tablename__ = "shout_reactions_followers"
id = None # type: ignore id = None # type: ignore
follower = Column(ForeignKey("user.slug"), primary_key=True) follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
shout = Column(ForeignKey("shout.slug"), primary_key=True) shout = Column(ForeignKey("shout.id"), primary_key=True, index=True)
auto = Column(Boolean, nullable=False, default=False) auto = Column(Boolean, nullable=False, default=False)
createdAt = Column( createdAt = Column(
DateTime, nullable=False, default=datetime.now, comment="Created at" DateTime, nullable=False, default=datetime.now, comment="Created at"
@ -34,8 +34,8 @@ class ShoutAuthor(Base):
__tablename__ = "shout_author" __tablename__ = "shout_author"
id = None # type: ignore id = None # type: ignore
shout = Column(ForeignKey("shout.slug"), primary_key=True) shout = Column(ForeignKey("shout.id"), primary_key=True, index=True)
user = Column(ForeignKey("user.slug"), primary_key=True) user = Column(ForeignKey("user.id"), primary_key=True, index=True)
caption = Column(String, nullable=True, default="") caption = Column(String, nullable=True, default="")
@ -55,7 +55,7 @@ class Shout(Base):
topics = relationship(lambda: Topic, secondary=ShoutTopic.__tablename__) topics = relationship(lambda: Topic, secondary=ShoutTopic.__tablename__)
reactions = relationship(lambda: Reaction) reactions = relationship(lambda: Reaction)
visibility = Column(String, nullable=True) # owner authors community public visibility = Column(String, nullable=True) # owner authors community public
versionOf = Column(ForeignKey("shout.slug"), nullable=True) versionOf = Column(ForeignKey("shout.id"), nullable=True)
oid = Column(String, nullable=True) oid = Column(String, nullable=True)
media = Column(JSON, nullable=True) media = Column(JSON, nullable=True)

4
orm/topic.py
View File

@ -9,8 +9,8 @@ class TopicFollower(Base):
__tablename__ = "topic_followers" __tablename__ = "topic_followers"
id = None # type: ignore id = None # type: ignore
follower = Column(ForeignKey("user.slug"), primary_key=True) follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
topic = Column(ForeignKey("topic.slug"), primary_key=True) topic = Column(ForeignKey("topic.id"), primary_key=True, index=True)
createdAt = Column( createdAt = Column(
DateTime, nullable=False, default=datetime.now, comment="Created at" DateTime, nullable=False, default=datetime.now, comment="Created at"
) )

25
orm/user.py
View File

@ -6,13 +6,12 @@ from sqlalchemy.orm import relationship
from base.orm import Base, local_session from base.orm import Base, local_session
from orm.rbac import Role from orm.rbac import Role
from services.auth.roles import RoleStorage
class UserNotifications(Base): class UserNotifications(Base):
__tablename__ = "user_notifications" __tablename__ = "user_notifications"
# id auto # id auto
user_id = Column(Integer, ForeignKey("user.id")) user = Column(Integer, ForeignKey("user.id"))
kind = Column(String, ForeignKey("notification.kind")) kind = Column(String, ForeignKey("notification.kind"))
values = Column(JSONType, nullable=True) # [ <var1>, .. ] values = Column(JSONType, nullable=True) # [ <var1>, .. ]
@ -21,8 +20,8 @@ class UserRating(Base):
__tablename__ = "user_rating" __tablename__ = "user_rating"
id = None # type: ignore id = None # type: ignore
rater = Column(ForeignKey("user.slug"), primary_key=True) raterId = Column(ForeignKey("user.id"), primary_key=True, index=True)
user = Column(ForeignKey("user.slug"), primary_key=True) user = Column(ForeignKey("user.id"), primary_key=True, index=True)
value = Column(Integer) value = Column(Integer)
@staticmethod @staticmethod
@ -34,16 +33,16 @@ class UserRole(Base):
__tablename__ = "user_role" __tablename__ = "user_role"
id = None # type: ignore id = None # type: ignore
user_id = Column(ForeignKey("user.id"), primary_key=True) user = Column(ForeignKey("user.id"), primary_key=True, index=True)
role_id = Column(ForeignKey("role.id"), primary_key=True) roleId = Column(ForeignKey("role.id"), primary_key=True, index=True)
class AuthorFollower(Base): class AuthorFollower(Base):
__tablename__ = "author_follower" __tablename__ = "author_follower"
id = None # type: ignore id = None # type: ignore
follower = Column(ForeignKey("user.slug"), primary_key=True) follower = Column(ForeignKey("user.id"), primary_key=True, index=True)
author = Column(ForeignKey("user.slug"), primary_key=True) author = Column(ForeignKey("user.id"), primary_key=True, index=True)
createdAt = Column( createdAt = Column(
DateTime, nullable=False, default=datetime.now, comment="Created at" DateTime, nullable=False, default=datetime.now, comment="Created at"
) )
@ -103,12 +102,12 @@ class User(Base):
async def get_permission(self): async def get_permission(self):
scope = {} scope = {}
for user_role in self.roles: for role in self.roles:
role: Role = await RoleStorage.get_role(user_role.id) # type: ignore
for p in role.permissions: for p in role.permissions:
if p.resource_id not in scope: if p.resourceId not in scope:
scope[p.resource_id] = set() scope[p.resourceId] = set()
scope[p.resource_id].add(p.operation_id) scope[p.resourceId].add(p.operationId)
return scope return scope

4
orm/viewed.py
View File

@ -6,8 +6,8 @@ from base.orm import Base, local_session
class ViewedEntry(Base): class ViewedEntry(Base):
__tablename__ = "viewed" __tablename__ = "viewed"
viewer = Column(ForeignKey("user.slug"), default='anonymous') viewerId = Column(ForeignKey("user.id"), index=True, default=1)
shout = Column(ForeignKey("shout.slug"), default="genesis-block") shout = Column(ForeignKey("shout.id"), index=True, default=1)
amount = Column(Integer, default=1) amount = Column(Integer, default=1)
createdAt = Column( createdAt = Column(
DateTime, nullable=False, default=datetime.now, comment="Created at" DateTime, nullable=False, default=datetime.now, comment="Created at"

3
resolvers/__init__.py
View File

@ -15,7 +15,8 @@ from resolvers.create.editor import create_shout, delete_shout, update_shout
from resolvers.zine.profile import ( from resolvers.zine.profile import (
load_authors_by, load_authors_by,
rate_user, rate_user,
update_profile update_profile,
get_authors_all
) )
from resolvers.zine.reactions import ( from resolvers.zine.reactions import (

25
resolvers/create/editor.py
View File

@ -1,13 +1,13 @@
from datetime import datetime, timezone from datetime import datetime, timezone
from sqlalchemy import and_
from auth.authenticate import login_required from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation from base.resolvers import mutation
from orm.rbac import Resource from orm.rbac import Resource
from orm.shout import Shout, ShoutAuthor, ShoutTopic from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.collab import Collab from orm.topic import TopicFollower, Topic
from services.inbox import MessagesStorage
from orm.topic import TopicFollower
from orm.user import User from orm.user import User
from resolvers.zine.reactions import reactions_follow, reactions_unfollow from resolvers.zine.reactions import reactions_follow, reactions_unfollow
from services.zine.gittask import GitTask from services.zine.gittask import GitTask
@ -49,7 +49,7 @@ async def create_shout(_, info, inp):
session.add(new_collab) session.add(new_collab)
# NOTE: shout made by one first author # NOTE: shout made by one first author
sa = ShoutAuthor.create(shout=new_shout.slug, user=user.slug) sa = ShoutAuthor.create(shout=new_shout.id, user=user.id)
session.add(sa) session.add(sa)
reactions_follow(user, new_shout.slug, True) reactions_follow(user, new_shout.slug, True)
@ -58,11 +58,16 @@ async def create_shout(_, info, inp):
topic_slugs.append(inp["mainTopic"]) topic_slugs.append(inp["mainTopic"])
for slug in topic_slugs: for slug in topic_slugs:
st = ShoutTopic.create(shout=new_shout.slug, topic=slug) topic = session.query(Topic).where(Topic.slug == slug).one()
st = ShoutTopic.create(shout=new_shout.id, topic=topic.id)
session.add(st) session.add(st)
tf = session.query(TopicFollower).where(follower=user.slug, topic=slug) tf = session.query(TopicFollower).where(
and_(TopicFollower.follower == user.id, TopicFollower.topic == topic.id)
)
if not tf: if not tf:
tf = TopicFollower.create(follower=user.slug, topic=slug, auto=True) tf = TopicFollower.create(follower=user.id, topic=topic.id, auto=True)
session.add(tf) session.add(tf)
new_shout.topic_slugs = topic_slugs new_shout.topic_slugs = topic_slugs
@ -70,7 +75,7 @@ async def create_shout(_, info, inp):
session.commit() session.commit()
GitTask(inp, user.username, user.email, "new shout %s" % (new_shout.slug)) GitTask(inp, user.username, user.email, "new shout %s" % new_shout.slug)
return {"shout": new_shout} return {"shout": new_shout}
@ -92,7 +97,7 @@ async def update_shout(_, info, inp):
if user_id not in authors: if user_id not in authors:
scopes = auth.scopes scopes = auth.scopes
print(scopes) print(scopes)
if Resource.shout_id not in scopes: if Resource.shout not in scopes:
return {"error": "access denied"} return {"error": "access denied"}
else: else:
shout.update(inp) shout.update(inp)
@ -100,7 +105,7 @@ async def update_shout(_, info, inp):
session.add(shout) session.add(shout)
if inp.get("topics"): if inp.get("topics"):
# remove old links # remove old links
links = session.query(ShoutTopic).where(ShoutTopic.shout == slug).all() links = session.query(ShoutTopic).where(ShoutTopic.shout == shout.id).all()
for topiclink in links: for topiclink in links:
session.delete(topiclink) session.delete(topiclink)
# add new topic links # add new topic links

16
resolvers/inbox/search.py
View File

@ -4,7 +4,7 @@ from auth.authenticate import login_required
from base.redis import redis from base.redis import redis
from base.resolvers import query from base.resolvers import query
from base.orm import local_session from base.orm import local_session
from orm.user import AuthorFollower from orm.user import AuthorFollower, User
@query.field("searchRecipients") @query.field("searchRecipients")
@ -30,13 +30,19 @@ async def search_recipients(_, info, query: str, limit: int = 50, offset: int =
with local_session() as session: with local_session() as session:
# followings # followings
result += session.query(AuthorFollower.author).where(AuthorFollower.follower.startswith(query))\ result += session.query(AuthorFollower.author).join(
.offset(offset + len(result)).limit(more_amount) User, User.id == AuthorFollower.follower
).where(
User.slug.startswith(query)
).offset(offset + len(result)).limit(more_amount)
more_amount = limit more_amount = limit
# followers # followers
result += session.query(AuthorFollower.follower).where(AuthorFollower.author.startswith(query))\ result += session.query(AuthorFollower.follower).join(
.offset(offset + len(result)).limit(offset + len(result) + limit) User, User.id == AuthorFollower.author
).where(
User.slug.startswith(query)
).offset(offset + len(result)).limit(offset + len(result) + limit)
return { return {
"members": list(result), "members": list(result),
"error": None "error": None

31
resolvers/zine/_common.py Normal file
View File

@ -0,0 +1,31 @@
from sqlalchemy import func, case
from sqlalchemy.orm import aliased
from orm.reaction import Reaction, ReactionKind
def add_common_stat_columns(q):
aliased_reaction = aliased(Reaction)
q = q.outerjoin(aliased_reaction).add_columns(
func.sum(
aliased_reaction.id
).label('reacted_stat'),
func.sum(
case(
(aliased_reaction.body.is_not(None), 1),
else_=0
)
).label('commented_stat'),
func.sum(case(
(aliased_reaction.kind == ReactionKind.AGREE, 1),
(aliased_reaction.kind == ReactionKind.DISAGREE, -1),
(aliased_reaction.kind == ReactionKind.PROOF, 1),
(aliased_reaction.kind == ReactionKind.DISPROOF, -1),
(aliased_reaction.kind == ReactionKind.ACCEPT, 1),
(aliased_reaction.kind == ReactionKind.REJECT, -1),
(aliased_reaction.kind == ReactionKind.LIKE, 1),
(aliased_reaction.kind == ReactionKind.DISLIKE, -1),
else_=0)
).label('rating_stat'))
return q

121
resolvers/zine/load.py
View File

@ -1,51 +1,31 @@
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
import sqlalchemy as sa
from sqlalchemy.orm import joinedload, aliased from sqlalchemy.orm import joinedload, aliased
from sqlalchemy.sql.expression import desc, asc, select, case from sqlalchemy.sql.expression import desc, asc, select, func
from base.orm import local_session from base.orm import local_session
from base.resolvers import query from base.resolvers import query
from orm import ViewedEntry from orm import ViewedEntry
from orm.shout import Shout from orm.shout import Shout, ShoutAuthor
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction
from services.zine.shoutauthor import ShoutAuthorStorage from resolvers.zine._common import add_common_stat_columns
from services.stat.viewed import ViewedStorage
def calc_reactions(q): def add_stat_columns(q):
aliased_reaction = aliased(Reaction) q = q.outerjoin(ViewedEntry).add_columns(func.sum(ViewedEntry.amount).label('viewed_stat'))
return q.join(aliased_reaction).add_columns(
sa.func.sum(case( return add_common_stat_columns(q)
(aliased_reaction.kind == ReactionKind.AGREE, 1),
(aliased_reaction.kind == ReactionKind.DISAGREE, -1),
(aliased_reaction.kind == ReactionKind.PROOF, 1),
(aliased_reaction.kind == ReactionKind.DISPROOF, -1),
(aliased_reaction.kind == ReactionKind.ACCEPT, 1),
(aliased_reaction.kind == ReactionKind.REJECT, -1),
(aliased_reaction.kind == ReactionKind.LIKE, 1),
(aliased_reaction.kind == ReactionKind.DISLIKE, -1),
else_=0)
).label('rating'),
sa.func.sum(
case(
(aliased_reaction.body.is_not(None), 1),
else_=0
)
).label('commented'),
sa.func.sum(
aliased_reaction.id
).label('reacted')
)
def apply_filters(q, filters, user=None): def apply_filters(q, filters, user=None):
filters = {} if filters is None else filters
if filters.get("reacted") and user: if filters.get("reacted") and user:
q.join(Reaction, Reaction.createdBy == user.slug) q.join(Reaction, Reaction.createdBy == user.id)
v = filters.get("visibility") v = filters.get("visibility")
if v == "public": if v == "public":
q = q.filter(Shout.visibility == filters.get("visibility")) q = q.filter(Shout.visibility == filters.get("visibility"))
if v == "community": if v == "community":
q = q.filter(Shout.visibility.in_(["public", "community"])) q = q.filter(Shout.visibility.in_(["public", "community"]))
if filters.get("layout"): if filters.get("layout"):
q = q.filter(Shout.layout == filters.get("layout")) q = q.filter(Shout.layout == filters.get("layout"))
if filters.get("author"): if filters.get("author"):
@ -59,6 +39,7 @@ def apply_filters(q, filters, user=None):
if filters.get("days"): if filters.get("days"):
before = datetime.now(tz=timezone.utc) - timedelta(days=int(filters.get("days")) or 30) before = datetime.now(tz=timezone.utc) - timedelta(days=int(filters.get("days")) or 30)
q = q.filter(Shout.createdAt > before) q = q.filter(Shout.createdAt > before)
return q return q
@ -69,24 +50,27 @@ async def load_shout(_, info, slug):
joinedload(Shout.authors), joinedload(Shout.authors),
joinedload(Shout.topics), joinedload(Shout.topics),
) )
q = calc_reactions(q) q = add_stat_columns(q)
q = q.filter( q = q.filter(
Shout.slug == slug Shout.slug == slug
).filter( ).filter(
Shout.deletedAt.is_(None) Shout.deletedAt.is_(None)
).group_by(Shout.id) ).group_by(Shout.id)
[shout, rating, commented, reacted] = session.execute(q).unique().one() [shout, viewed_stat, reacted_stat, commented_stat, rating_stat] = session.execute(q).unique().one()
for a in shout.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(shout.slug, a.slug)
viewed = await ViewedStorage.get_shout(shout.slug)
shout.stat = { shout.stat = {
"rating": rating, "viewed": viewed_stat,
"viewed": viewed, "reacted": reacted_stat,
"commented": commented, "commented": commented_stat,
"reacted": reacted "rating": rating_stat
} }
for author_caption in session.query(ShoutAuthor).join(Shout).where(Shout.slug == slug):
for author in shout.authors:
if author.id == author_caption.user:
author.caption = author_caption.caption
return shout return shout
@ -105,7 +89,7 @@ async def load_shouts_by(_, info, options):
} }
offset: 0 offset: 0
limit: 50 limit: 50
order_by: 'createdAt' order_by: 'createdAt' | 'commented' | 'reacted' | 'rating'
order_by_desc: true order_by_desc: true
} }
@ -118,47 +102,36 @@ async def load_shouts_by(_, info, options):
).where( ).where(
Shout.deletedAt.is_(None) Shout.deletedAt.is_(None)
) )
q = add_stat_columns(q)
user = info.context["request"].user user = info.context["request"].user
q = apply_filters(q, options.get("filters"), user) q = apply_filters(q, options.get("filters", {}), user)
q = calc_reactions(q)
o = options.get("order_by") order_by = options.get("order_by", Shout.createdAt)
if o: if order_by == 'reacted':
if o == 'comments': aliased_reaction = aliased(Reaction)
q = q.add_columns(sa.func.count(Reaction.id).label(o)) q.outerjoin(aliased_reaction).add_columns(func.max(aliased_reaction.createdAt).label('reacted'))
q = q.join(Reaction, Shout.slug == Reaction.shout)
q = q.filter(Reaction.body.is_not(None))
elif o == 'reacted':
q = q.join(
Reaction
).add_columns(
sa.func.max(Reaction.createdAt).label(o)
)
elif o == 'views':
q = q.join(ViewedEntry)
q = q.add_columns(sa.func.sum(ViewedEntry.amount).label(o))
order_by = o
else:
order_by = Shout.createdAt
order_by_desc = True if options.get('order_by_desc') is None else options.get('order_by_desc') order_by_desc = options.get('order_by_desc', True)
query_order_by = desc(order_by) if order_by_desc else asc(order_by) query_order_by = desc(order_by) if order_by_desc else asc(order_by)
offset = options.get("offset", 0) offset = options.get("offset", 0)
limit = options.get("limit", 10) limit = options.get("limit", 10)
q = q.group_by(Shout.id).order_by(query_order_by).limit(limit).offset(offset) q = q.group_by(Shout.id).order_by(query_order_by).limit(limit).offset(offset)
shouts = []
with local_session() as session: with local_session() as session:
for [shout, rating, commented, reacted] in session.execute(q).unique(): shouts = []
shout.stat = {
"rating": rating, for [shout, viewed_stat, reacted_stat, commented_stat, rating_stat] in session.execute(q).unique():
"viewed": await ViewedStorage.get_shout(shout.slug),
"commented": commented,
"reacted": reacted
}
# NOTE: no need authors captions in arrays
# for author in shout.authors:
# author.caption = await ShoutAuthorStorage.get_author_caption(shout.slug, author.slug)
shouts.append(shout) shouts.append(shout)
shout.stat = {
"viewed": viewed_stat,
"reacted": reacted_stat,
"commented": commented_stat,
"rating": rating_stat
}
return shouts return shouts

203
resolvers/zine/profile.py
View File

@ -1,20 +1,77 @@
from typing import List from typing import List
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, func from sqlalchemy import and_, func, distinct, select, literal
from sqlalchemy.orm import selectinload from sqlalchemy.orm import aliased, joinedload
from auth.authenticate import login_required from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from orm.reaction import Reaction from orm.reaction import Reaction
from orm.shout import ShoutAuthor from orm.shout import ShoutAuthor, ShoutTopic
from orm.topic import Topic, TopicFollower from orm.topic import Topic
from orm.user import AuthorFollower, Role, User, UserRating, UserRole from orm.user import AuthorFollower, Role, User, UserRating, UserRole
from services.stat.topicstat import TopicStat
# from .community import followed_communities # from .community import followed_communities
from resolvers.inbox.unread import get_total_unread_counter from resolvers.inbox.unread import get_total_unread_counter
from .topics import get_topic_stat from resolvers.zine.topics import followed_by_user
def add_author_stat_columns(q):
author_followers = aliased(AuthorFollower)
author_following = aliased(AuthorFollower)
shout_author_aliased = aliased(ShoutAuthor)
user_rating_aliased = aliased(UserRating)
q = q.outerjoin(shout_author_aliased).add_columns(
func.count(distinct(shout_author_aliased.shout)).label('shouts_stat')
)
q = q.outerjoin(author_followers, author_followers.author == User.id).add_columns(
func.count(distinct(author_followers.follower)).label('followers_stat')
)
q = q.outerjoin(author_following, author_following.follower == User.id).add_columns(
func.count(distinct(author_following.author)).label('followings_stat')
)
q = q.add_columns(literal(0).label('rating_stat'))
# FIXME
# q = q.outerjoin(user_rating_aliased, user_rating_aliased.user == User.id).add_columns(
# # TODO: check
# func.sum(user_rating_aliased.value).label('rating_stat')
# )
q = q.add_columns(literal(0).label('commented_stat'))
# FIXME
# q = q.outerjoin(Reaction, and_(Reaction.createdBy == User.id, Reaction.body.is_not(None))).add_columns(
# func.count(distinct(Reaction.id)).label('commented_stat')
# )
q = q.group_by(User.id)
return q
def add_stat(author, stat_columns):
[shouts_stat, followers_stat, followings_stat, rating_stat, commented_stat] = stat_columns
author.stat = {
"shouts": shouts_stat,
"followers": followers_stat,
"followings": followings_stat,
"rating": rating_stat,
"commented": commented_stat
}
return author
def get_authors_from_query(q):
authors = []
with local_session() as session:
for [author, *stat_columns] in session.execute(q):
author = add_stat(author, stat_columns)
authors.append(author)
return authors
async def user_subscriptions(slug: str): async def user_subscriptions(slug: str):
@ -27,23 +84,6 @@ async def user_subscriptions(slug: str):
} }
async def get_author_stat(slug):
with local_session() as session:
return {
"shouts": session.query(ShoutAuthor).where(ShoutAuthor.user == slug).count(),
"followers": session.query(AuthorFollower).where(AuthorFollower.author == slug).count(),
"followings": session.query(AuthorFollower).where(AuthorFollower.follower == slug).count(),
"rating": session.query(func.sum(UserRating.value)).where(UserRating.user == slug).first(),
"commented": session.query(
Reaction.id
).where(
Reaction.createdBy == slug
).filter(
Reaction.body.is_not(None)
).count()
}
# @query.field("userFollowedDiscussions") # @query.field("userFollowedDiscussions")
@login_required @login_required
async def followed_discussions(_, info, slug) -> List[Topic]: async def followed_discussions(_, info, slug) -> List[Topic]:
@ -56,7 +96,7 @@ async def followed_reactions(slug):
return session.query( return session.query(
Reaction.shout Reaction.shout
).where( ).where(
Reaction.createdBy == slug Reaction.createdBy == user.id
).filter( ).filter(
Reaction.createdAt > user.lastSeen Reaction.createdAt > user.lastSeen
).all() ).all()
@ -69,17 +109,7 @@ async def get_followed_topics(_, info, slug) -> List[Topic]:
async def followed_topics(slug): async def followed_topics(slug):
topics = [] return followed_by_user(slug)
with local_session() as session:
topics = (
session.query(Topic)
.join(TopicFollower)
.where(TopicFollower.follower == slug)
.all()
)
for topic in topics:
topic.stat = await get_topic_stat(topic.slug)
return topics
@query.field("userFollowedAuthors") @query.field("userFollowedAuthors")
@ -88,29 +118,26 @@ async def get_followed_authors(_, _info, slug) -> List[User]:
async def followed_authors(slug) -> List[User]: async def followed_authors(slug) -> List[User]:
authors = [] q = select(User)
with local_session() as session: q = add_author_stat_columns(q)
authors = ( q = q.join(AuthorFollower).join(User, User.id == AuthorFollower.follower).where(User.slug == slug)
session.query(User)
.join(AuthorFollower, User.slug == AuthorFollower.author) return get_authors_from_query(q)
.where(AuthorFollower.follower == slug)
.all()
)
for author in authors:
author.stat = await get_author_stat(author.slug)
return authors
@query.field("userFollowers") @query.field("userFollowers")
async def user_followers(_, _info, slug) -> List[User]: async def user_followers(_, _info, slug) -> List[User]:
with local_session() as session: q = select(User)
users = ( q = add_author_stat_columns(q)
session.query(User)
.join(AuthorFollower, User.slug == AuthorFollower.follower) aliased_user = aliased(User)
.where(AuthorFollower.author == slug) q = q.join(AuthorFollower).join(
.all() aliased_user, aliased_user.id == AuthorFollower.author
).where(
aliased_user.slug == slug
) )
return users
return get_authors_from_query(q)
async def get_user_roles(slug): async def get_user_roles(slug):
@ -118,11 +145,12 @@ async def get_user_roles(slug):
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
roles = ( roles = (
session.query(Role) session.query(Role)
.options(selectinload(Role.permissions)) .options(joinedload(Role.permissions))
.join(UserRole) .join(UserRole)
.where(UserRole.user_id == user.id) .where(UserRole.user == user.id)
.all() .all()
) )
return roles return roles
@ -164,7 +192,8 @@ async def rate_user(_, info, rated_userslug, value):
# for mutation.field("follow") # for mutation.field("follow")
def author_follow(user, slug): def author_follow(user, slug):
with local_session() as session: with local_session() as session:
af = AuthorFollower.create(follower=user.slug, author=slug) author = session.query(User).where(User.slug == slug).one()
af = AuthorFollower.create(follower=user.id, author=author.id)
session.add(af) session.add(af)
session.commit() session.commit()
@ -173,13 +202,13 @@ def author_follow(user, slug):
def author_unfollow(user, slug): def author_unfollow(user, slug):
with local_session() as session: with local_session() as session:
flw = ( flw = (
session.query(AuthorFollower) session.query(
.filter( AuthorFollower
).join(User, User.id == AuthorFollower.author).filter(
and_( and_(
AuthorFollower.follower == user.slug, AuthorFollower.author == slug AuthorFollower.follower == user.id, User.slug == slug
) )
) ).first()
.first()
) )
if not flw: if not flw:
raise Exception("[resolvers.profile] follower not exist, cant unfollow") raise Exception("[resolvers.profile] follower not exist, cant unfollow")
@ -190,49 +219,41 @@ def author_unfollow(user, slug):
@query.field("authorsAll") @query.field("authorsAll")
async def get_authors_all(_, _info): async def get_authors_all(_, _info):
with local_session() as session: q = select(User)
authors = session.query(User).join(ShoutAuthor).all() q = add_author_stat_columns(q)
for author in authors: q = q.join(ShoutAuthor, User.id == ShoutAuthor.user)
author.stat = await get_author_stat(author.slug)
return authors return get_authors_from_query(q)
@query.field("getAuthor") @query.field("getAuthor")
async def get_author(_, _info, slug): async def get_author(_, _info, slug):
with local_session() as session: q = select(User).where(User.slug == slug)
author = session.query(User).where(User.slug == slug).first() q = add_author_stat_columns(q)
author.stat = await get_author_stat(author.slug)
return author authors = get_authors_from_query(q)
return authors[0]
@query.field("loadAuthorsBy") @query.field("loadAuthorsBy")
async def load_authors_by(_, info, by, limit, offset): async def load_authors_by(_, info, by, limit, offset):
authors = [] q = select(User)
with local_session() as session: q = add_author_stat_columns(q)
aq = session.query(User)
if by.get("slug"): if by.get("slug"):
aq = aq.filter(User.slug.ilike(f"%{by['slug']}%")) q = q.filter(User.slug.ilike(f"%{by['slug']}%"))
elif by.get("name"): elif by.get("name"):
aq = aq.filter(User.name.ilike(f"%{by['name']}%")) q = q.filter(User.name.ilike(f"%{by['name']}%"))
elif by.get("topic"): elif by.get("topic"):
aaa = list(map(lambda a: a.slug, TopicStat.authors_by_topic.get(by["topic"]))) q = q.join(ShoutAuthor).join(ShoutTopic).join(Topic).where(Topic.slug == by["topic"])
aq = aq.filter(User.name._in(aaa))
if by.get("lastSeen"): # in days if by.get("lastSeen"): # in days
days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"]) days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"])
aq = aq.filter(User.lastSeen > days_before) q = q.filter(User.lastSeen > days_before)
elif by.get("createdAt"): # in days elif by.get("createdAt"): # in days
days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"]) days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"])
aq = aq.filter(User.createdAt > days_before) q = q.filter(User.createdAt > days_before)
aq = aq.group_by(
User.id q = q.order_by(
).order_by( by.get("order", User.createdAt)
by.get("order") or "createdAt"
).limit(limit).offset(offset) ).limit(limit).offset(offset)
print(aq)
authors = list(map(lambda r: r.User, session.execute(aq))) return get_authors_from_query(q)
if by.get("stat"):
for a in authors:
a.stat = await get_author_stat(a.slug)
authors = list(set(authors))
# authors = sorted(authors, key=lambda a: a["stat"].get(by.get("stat")))
return authors

66
resolvers/zine/reactions.py
View File

@ -1,29 +1,34 @@
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, asc, desc, select, text, func from sqlalchemy import and_, asc, desc, select, text, func
from sqlalchemy.orm import aliased from sqlalchemy.orm import aliased
from auth.authenticate import login_required from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutReactionsFollower from orm.shout import Shout, ShoutReactionsFollower
from orm.user import User from orm.user import User
# from services.stat.reacted import ReactedStorage from resolvers.zine._common import add_common_stat_columns
from resolvers.zine.load import calc_reactions
def add_reaction_stat_columns(q):
return add_common_stat_columns(q)
def reactions_follow(user: User, slug: str, auto=False): def reactions_follow(user: User, slug: str, auto=False):
with local_session() as session: with local_session() as session:
shout = session.query(Shout).where(Shout.slug == slug).one()
following = ( following = (
session.query(ShoutReactionsFollower).where(and_( session.query(ShoutReactionsFollower).where(and_(
ShoutReactionsFollower.follower == user.slug, ShoutReactionsFollower.follower == user.id,
ShoutReactionsFollower.shout == slug ShoutReactionsFollower.shout == shout.id,
)).first() )).first()
) )
if not following: if not following:
following = ShoutReactionsFollower.create( following = ShoutReactionsFollower.create(
follower=user.slug, follower=user.id,
shout=slug, shout=shout.id,
auto=auto auto=auto
) )
session.add(following) session.add(following)
@ -32,12 +37,15 @@ def reactions_follow(user: User, slug: str, auto=False):
def reactions_unfollow(user, slug): def reactions_unfollow(user, slug):
with local_session() as session: with local_session() as session:
shout = session.query(Shout).where(Shout.slug == slug).one()
following = ( following = (
session.query(ShoutReactionsFollower).where(and_( session.query(ShoutReactionsFollower).where(and_(
ShoutReactionsFollower.follower == user.slug, ShoutReactionsFollower.follower == user.id,
ShoutReactionsFollower.shout == slug ShoutReactionsFollower.shout == shout.id
)).first() )).first()
) )
if following: if following:
session.delete(following) session.delete(following)
session.commit() session.commit()
@ -134,7 +142,6 @@ async def create_reaction(_, info, inp):
elif check_to_publish(session, user, reaction): elif check_to_publish(session, user, reaction):
set_published(session, reaction.shout, reaction.createdBy) set_published(session, reaction.shout, reaction.createdBy)
# ReactedStorage.react(reaction)
try: try:
reactions_follow(user, inp["shout"], True) reactions_follow(user, inp["shout"], True)
except Exception as e: except Exception as e:
@ -157,9 +164,9 @@ async def update_reaction(_, info, inp):
with local_session() as session: with local_session() as session:
user = session.query(User).where(User.id == user_id).first() user = session.query(User).where(User.id == user_id).first()
q = select(Reaction).filter(Reaction.id == inp.id) q = select(Reaction).filter(Reaction.id == inp.id)
q = calc_reactions(q) q = add_reaction_stat_columns(q)
[reaction, rating, commented, reacted] = session.execute(q).unique().one() [reaction, reacted_stat, commented_stat, rating_stat] = session.execute(q).unique().one()
if not reaction: if not reaction:
return {"error": "invalid reaction id"} return {"error": "invalid reaction id"}
@ -175,9 +182,9 @@ async def update_reaction(_, info, inp):
reaction.range = inp.get("range") reaction.range = inp.get("range")
session.commit() session.commit()
reaction.stat = { reaction.stat = {
"commented": commented, "commented": commented_stat,
"reacted": reacted, "reacted": reacted_stat,
"rating": rating "rating": rating_stat
} }
return {"reaction": reaction} return {"reaction": reaction}
@ -199,6 +206,7 @@ async def delete_reaction(_, info, rid):
session.commit() session.commit()
return {} return {}
@query.field("loadReactionsBy") @query.field("loadReactionsBy")
async def load_reactions_by(_, _info, by, limit=50, offset=0): async def load_reactions_by(_, _info, by, limit=50, offset=0):
""" """
@ -222,28 +230,33 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
q = select( q = select(
Reaction, CreatedByUser, ReactedShout Reaction, CreatedByUser, ReactedShout
).join( ).join(
CreatedByUser, Reaction.createdBy == CreatedByUser.slug CreatedByUser, Reaction.createdBy == CreatedByUser.id
).join( ).join(
ReactedShout, Reaction.shout == ReactedShout.slug ReactedShout, Reaction.shout == ReactedShout.id
) )
if by.get("shout"): if by.get("shout"):
q = q.filter(Reaction.shout == by["shout"]) aliased_shout = aliased(Shout)
q = q.join(aliased_shout).filter(aliased_shout.slug == by["shout"])
elif by.get("shouts"): elif by.get("shouts"):
q = q.filter(Reaction.shout.in_(by["shouts"])) aliased_shout = aliased(Shout)
q = q.join(aliased_shout).filter(aliased_shout.shout.in_(by["shouts"]))
if by.get("createdBy"): if by.get("createdBy"):
q = q.filter(Reaction.createdBy == by.get("createdBy")) aliased_user = aliased(User)
q = q.join(aliased_user).filter(aliased_user.slug == by.get("createdBy"))
if by.get("topic"): if by.get("topic"):
# TODO: check
q = q.filter(Shout.topics.contains(by["topic"])) q = q.filter(Shout.topics.contains(by["topic"]))
if by.get("comment"): if by.get("comment"):
q = q.filter(func.length(Reaction.body) > 0) q = q.filter(func.length(Reaction.body) > 0)
if by.get('search', 0) > 2: if len(by.get('search', '')) > 2:
q = q.filter(Reaction.body.ilike(f'%{by["body"]}%')) q = q.filter(Reaction.body.ilike(f'%{by["body"]}%'))
if by.get("days"): if by.get("days"):
after = datetime.now(tz=timezone.utc) - timedelta(days=int(by["days"]) or 30) after = datetime.now(tz=timezone.utc) - timedelta(days=int(by["days"]) or 30)
q = q.filter(Reaction.createdAt > after) q = q.filter(Reaction.createdAt > after)
order_way = asc if by.get("sort", "").startswith("-") else desc order_way = asc if by.get("sort", "").startswith("-") else desc
# replace "-" -> "" ?
order_field = by.get("sort") or Reaction.createdAt order_field = by.get("sort") or Reaction.createdAt
q = q.group_by( q = q.group_by(
@ -252,23 +265,24 @@ async def load_reactions_by(_, _info, by, limit=50, offset=0):
order_way(order_field) order_way(order_field)
) )
q = calc_reactions(q) q = add_reaction_stat_columns(q)
q = q.where(Reaction.deletedAt.is_(None)) q = q.where(Reaction.deletedAt.is_(None))
q = q.limit(limit).offset(offset) q = q.limit(limit).offset(offset)
reactions = [] reactions = []
with local_session() as session: with local_session() as session:
for [reaction, user, shout, rating, commented, reacted] in session.execute(q): for [reaction, user, shout, reacted_stat, commented_stat, rating_stat] in session.execute(q):
reaction.createdBy = user reaction.createdBy = user
reaction.shout = shout reaction.shout = shout
reaction.stat = { reaction.stat = {
"rating": rating, "rating": rating_stat,
"commented": commented, "commented": commented_stat,
"reacted": reacted "reacted": reacted_stat
} }
reactions.append(reaction) reactions.append(reaction)
# ?
if by.get("stat"): if by.get("stat"):
reactions.sort(lambda r: r.stat.get(by["stat"]) or r.createdAt) reactions.sort(lambda r: r.stat.get(by["stat"]) or r.createdAt)

114
resolvers/zine/topics.py
View File

@ -1,56 +1,91 @@
from sqlalchemy import and_ from sqlalchemy import and_, select, distinct, func
from auth.authenticate import login_required from auth.authenticate import login_required
from base.orm import local_session from base.orm import local_session
from base.resolvers import mutation, query from base.resolvers import mutation, query
from orm.shout import ShoutTopic, ShoutAuthor
from orm.topic import Topic, TopicFollower from orm.topic import Topic, TopicFollower
from services.zine.topics import TopicStorage from orm import Shout, User
from services.stat.topicstat import TopicStat
# from services.stat.viewed import ViewedStorage def add_topic_stat_columns(q):
q = q.outerjoin(ShoutTopic, Topic.id == ShoutTopic.topic).add_columns(
func.count(distinct(ShoutTopic.shout)).label('shouts_stat')
).outerjoin(ShoutAuthor, ShoutTopic.shout == ShoutAuthor.shout).add_columns(
func.count(distinct(ShoutAuthor.user)).label('authors_stat')
).outerjoin(TopicFollower,
and_(
TopicFollower.topic == Topic.id,
TopicFollower.follower == ShoutAuthor.id
)).add_columns(
func.count(distinct(TopicFollower.follower)).label('followers_stat')
)
q = q.group_by(Topic.id)
return q
async def get_topic_stat(slug): def add_stat(topic, stat_columns):
return { [shouts_stat, authors_stat, followers_stat] = stat_columns
"shouts": len(TopicStat.shouts_by_topic.get(slug, {}).keys()), topic.stat = {
"authors": len(TopicStat.authors_by_topic.get(slug, {}).keys()), "shouts": shouts_stat,
"followers": len(TopicStat.followers_by_topic.get(slug, {}).keys()) "authors": authors_stat,
"followers": followers_stat
} }
return topic
def get_topics_from_query(q):
topics = []
with local_session() as session:
for [topic, *stat_columns] in session.execute(q):
topic = add_stat(topic, stat_columns)
topics.append(topic)
return topics
def followed_by_user(user_slug):
q = select(Topic)
q = add_topic_stat_columns(q)
q = q.join(User).where(User.slug == user_slug)
return get_topics_from_query(q)
@query.field("topicsAll") @query.field("topicsAll")
async def topics_all(_, _info): async def topics_all(_, _info):
topics = await TopicStorage.get_topics_all() q = select(Topic)
for topic in topics: q = add_topic_stat_columns(q)
topic.stat = await get_topic_stat(topic.slug)
return topics return get_topics_from_query(q)
@query.field("topicsByCommunity") @query.field("topicsByCommunity")
async def topics_by_community(_, info, community): async def topics_by_community(_, info, community):
topics = await TopicStorage.get_topics_by_community(community) q = select(Topic).where(Topic.community == community)
for topic in topics: q = add_topic_stat_columns(q)
topic.stat = await get_topic_stat(topic.slug)
return topics return get_topics_from_query(q)
@query.field("topicsByAuthor") @query.field("topicsByAuthor")
async def topics_by_author(_, _info, author): async def topics_by_author(_, _info, author):
shouts = TopicStorage.get_topics_by_author(author) q = select(Topic)
author_topics = set() q = add_topic_stat_columns(q)
for s in shouts: q = q.join(User).where(User.slug == author)
for tpc in s.topics:
tpc = await TopicStorage.topics[tpc.slug] return get_topics_from_query(q)
tpc.stat = await get_topic_stat(tpc.slug)
author_topics.add(tpc)
return list(author_topics)
@query.field("getTopic") @query.field("getTopic")
async def get_topic(_, _info, slug): async def get_topic(_, _info, slug):
t = TopicStorage.topics[slug] q = select(Topic).where(Topic.slug == slug)
t.stat = await get_topic_stat(slug) q = add_topic_stat_columns(q)
return t
topics = get_topics_from_query(q)
return topics[0]
@mutation.field("createTopic") @mutation.field("createTopic")
@ -61,7 +96,7 @@ async def create_topic(_, _info, inp):
new_topic = Topic.create(**inp) new_topic = Topic.create(**inp)
session.add(new_topic) session.add(new_topic)
session.commit() session.commit()
await TopicStorage.update_topic(new_topic)
return {"topic": new_topic} return {"topic": new_topic}
@ -76,25 +111,26 @@ async def update_topic(_, _info, inp):
else: else:
topic.update(**inp) topic.update(**inp)
session.commit() session.commit()
await TopicStorage.update_topic(topic.slug)
return {"topic": topic} return {"topic": topic}
async def topic_follow(user, slug): async def topic_follow(user, slug):
with local_session() as session: with local_session() as session:
following = TopicFollower.create(topic=slug, follower=user.slug) topic = session.query(Topic).where(Topic.slug == slug).one()
following = TopicFollower.create(topic=topic.id, follower=user.id)
session.add(following) session.add(following)
session.commit() session.commit()
await TopicStorage.update_topic(slug)
async def topic_unfollow(user, slug): async def topic_unfollow(user, slug):
with local_session() as session: with local_session() as session:
sub = ( sub = (
session.query(TopicFollower).filter( session.query(TopicFollower).join(Topic).filter(
and_( and_(
TopicFollower.follower == user.slug, TopicFollower.follower == user.id,
TopicFollower.topic == slug Topic.slug == slug
) )
).first() ).first()
) )
@ -103,9 +139,13 @@ async def topic_unfollow(user, slug):
else: else:
session.delete(sub) session.delete(sub)
session.commit() session.commit()
await TopicStorage.update_topic(slug)
@query.field("topicsRandom") @query.field("topicsRandom")
async def topics_random(_, info, amount=12): async def topics_random(_, info, amount=12):
return TopicStorage.get_random_topics(amount) q = select(Topic)
q = add_topic_stat_columns(q)
q = q.join(Shout, ShoutTopic.shout == Shout.id).group_by(Topic.id).having(func.count(Shout.id) > 2)
q = q.order_by(func.random()).limit(amount)
return get_topics_from_query(q)

21
schema.graphql
View File

@ -321,8 +321,8 @@ type Operation {
} }
type Permission { type Permission {
operation_id: Int! operationId: Int!
resource_id: Int! resourceId: Int!
} }
type Role { type Role {
@ -373,6 +373,23 @@ type User {
oid: String oid: String
} }
<<<<<<< HEAD
=======
type Draft {
title: String
body: String
createdBy: Int
}
type Collab {
authors: [String]!
invites: [String]
createdAt: DateTime!
title: String
body: String
}
>>>>>>> migation-fix2
enum ReactionKind { enum ReactionKind {
LIKE LIKE
DISLIKE DISLIKE

35
services/auth/roles.py
View File

@ -1,35 +0,0 @@
import asyncio
from sqlalchemy.orm import selectinload
from orm.rbac import Role
class RoleStorage:
roles = {}
lock = asyncio.Lock()
@staticmethod
def init(session):
self = RoleStorage
roles = session.query(Role).options(selectinload(Role.permissions)).all()
self.roles = dict([(role.id, role) for role in roles])
print("[auth.roles] %d precached" % len(roles))
@staticmethod
async def get_role(id):
self = RoleStorage
async with self.lock:
return self.roles.get(id)
@staticmethod
async def add_role(role):
self = RoleStorage
async with self.lock:
self.roles[id] = role
@staticmethod
async def del_role(id):
self = RoleStorage
async with self.lock:
del self.roles[id]

72
services/auth/users.py
View File

@ -1,72 +0,0 @@
import asyncio
from sqlalchemy.orm import selectinload, exc
from orm.user import User
from base.orm import local_session
class UserStorage:
users = {}
lock = asyncio.Lock()
@staticmethod
def init(session):
self = UserStorage
users = (
session.query(User)
.options(selectinload(User.roles), selectinload(User.ratings))
.all()
)
self.users = dict([(user.id, user) for user in users])
print("[auth.users] %d precached" % len(self.users))
@staticmethod
async def get_user(id):
with local_session() as session:
try:
user = (
session.query(User).options(
selectinload(User.roles),
selectinload(User.ratings)
).filter(
User.id == id
).one()
)
return user
except exc.NoResultFound:
return None
@staticmethod
async def get_all_users():
self = UserStorage
async with self.lock:
aaa = list(self.users.values())
aaa.sort(key=lambda user: user.createdAt)
return aaa
@staticmethod
async def get_top_users():
self = UserStorage
async with self.lock:
aaa = list(self.users.values())
aaa.sort(key=lambda user: user.rating)
return aaa
@staticmethod
async def get_user_by_slug(slug):
self = UserStorage
async with self.lock:
for user in self.users.values():
if user.slug == slug:
return user
@staticmethod
async def add_user(user):
self = UserStorage
async with self.lock:
self.users[user.id] = user
@staticmethod
async def del_user(id):
self = UserStorage
async with self.lock:
del self.users[id]

14
services/main.py
View File

@ -1,7 +1,3 @@
# from services.stat.reacted import ReactedStorage
from services.auth.roles import RoleStorage
from services.auth.users import UserStorage
from services.zine.topics import TopicStorage
from services.search import SearchService from services.search import SearchService
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
from base.orm import local_session from base.orm import local_session
@ -9,11 +5,9 @@ from base.orm import local_session
async def storages_init(): async def storages_init():
with local_session() as session: with local_session() as session:
print('[main] initialize storages') print('[main] initialize SearchService')
# ReactedStorage.init(session)
RoleStorage.init(session)
UserStorage.init(session)
TopicStorage.init(session)
await SearchService.init(session) await SearchService.init(session)
session.commit() print('[main] SearchService initialized')
print('[main] initialize storages')
await ViewedStorage.init() await ViewedStorage.init()
print('[main] storages initialized')

72
services/stat/topicstat.py
View File

@ -1,72 +0,0 @@
import asyncio
import time
from base.orm import local_session
from orm.shout import Shout, ShoutTopic, ShoutAuthor
from orm.topic import TopicFollower
# from sqlalchemy.sql.expression import select
class TopicStat:
# by slugs
shouts_by_topic = {} # Shout object stored
authors_by_topic = {} # User
followers_by_topic = {} # User
#
lock = asyncio.Lock()
period = 30 * 60 # sec
@staticmethod
async def load_stat(session):
print("[stat.topics] ⎧ loading stat -------")
ts = time.time()
self = TopicStat
shout_topics = session.query(ShoutTopic, Shout).join(Shout).all() # ~ 10 secs
print("[stat.topics] ⎪ shout topics joined query took %fs " % (time.time() - ts))
print("[stat.topics] ⎪ indexing %d links..." % len(shout_topics))
for [shout_topic, shout] in shout_topics:
tpc = shout_topic.topic
self.shouts_by_topic[tpc] = self.shouts_by_topic.get(tpc, dict())
self.shouts_by_topic[tpc][shout.slug] = shout
self.authors_by_topic[tpc] = self.authors_by_topic.get(tpc, dict())
authors = session.query(
ShoutAuthor.user, ShoutAuthor.caption
).filter(
ShoutAuthor.shout == shout.slug
).all()
for a in authors:
self.authors_by_topic[tpc][a[0]] = a[1]
self.followers_by_topic = {}
followings = session.query(TopicFollower).all()
print("[stat.topics] ⎪ indexing %d followings..." % len(followings))
for flw in followings:
topic = flw.topic
userslug = flw.follower
self.followers_by_topic[topic] = self.followers_by_topic.get(topic, dict())
self.followers_by_topic[topic][userslug] = userslug
@staticmethod
async def get_shouts(topic):
self = TopicStat
async with self.lock:
return self.shouts_by_topic.get(topic, dict())
@staticmethod
async def worker():
self = TopicStat
first_run = True
while True:
try:
with local_session() as session:
ts = time.time()
async with self.lock:
await self.load_stat(session)
print("[stat.topics] ⎩ load_stat took %fs " % (time.time() - ts))
except Exception as err:
raise Exception(err)
if first_run:
# sleep for period + 1 min after first run
# to distribute load on server by workers with the same period
await asyncio.sleep(60)
first_run = False
await asyncio.sleep(self.period)

26
services/stat/viewed.py
View File

@ -5,7 +5,9 @@ from gql import Client, gql
from gql.transport.aiohttp import AIOHTTPTransport from gql.transport.aiohttp import AIOHTTPTransport
from base.orm import local_session from base.orm import local_session
from sqlalchemy import func from sqlalchemy import func
from orm.shout import ShoutTopic
from orm import User, Topic
from orm.shout import ShoutTopic, Shout
from orm.viewed import ViewedEntry from orm.viewed import ViewedEntry
from ssl import create_default_context from ssl import create_default_context
from os import environ, path from os import environ, path
@ -113,6 +115,7 @@ class ViewedStorage:
async with self.lock: async with self.lock:
return self.client.execute_async(load_facts) return self.client.execute_async(load_facts)
# unused yet
@staticmethod @staticmethod
async def get_shout(shout_slug): async def get_shout(shout_slug):
""" getting shout views metric by slug """ """ getting shout views metric by slug """
@ -123,8 +126,9 @@ class ViewedStorage:
shout_views = 0 shout_views = 0
with local_session() as session: with local_session() as session:
try: try:
shout = session.query(Shout).where(Shout.slug == shout_slug).one()
shout_views = session.query(func.sum(ViewedEntry.amount)).where( shout_views = session.query(func.sum(ViewedEntry.amount)).where(
ViewedEntry.shout == shout_slug ViewedEntry.shout == shout.id
).all()[0][0] ).all()[0][0]
self.by_shouts[shout_slug] = shout_views self.by_shouts[shout_slug] = shout_views
self.update_topics(session, shout_slug) self.update_topics(session, shout_slug)
@ -147,11 +151,12 @@ class ViewedStorage:
def update_topics(session, shout_slug): def update_topics(session, shout_slug):
""" updates topics counters by shout slug """ """ updates topics counters by shout slug """
self = ViewedStorage self = ViewedStorage
for t in session.query(ShoutTopic).where(ShoutTopic.shout == shout_slug).all(): for [shout_topic, topic] in session.query(ShoutTopic, Topic).join(Topic).join(Shout).where(
tpc = t.topic Shout.slug == shout_slug
if not self.by_topics.get(tpc): ).all():
self.by_topics[tpc] = {} if not self.by_topics.get(topic.slug):
self.by_topics[tpc][shout_slug] = self.by_shouts[shout_slug] self.by_topics[topic.slug] = {}
self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug]
@staticmethod @staticmethod
async def increment(shout_slug, amount=1, viewer='anonymous'): async def increment(shout_slug, amount=1, viewer='anonymous'):
@ -159,9 +164,12 @@ class ViewedStorage:
self = ViewedStorage self = ViewedStorage
async with self.lock: async with self.lock:
with local_session() as session: with local_session() as session:
shout = session.query(Shout).where(Shout.slug == shout_slug).one()
viewer = session.query(User).where(User.slug == viewer).one()
viewed = ViewedEntry.create(**{ viewed = ViewedEntry.create(**{
"viewer": viewer, "viewerId": viewer.id,
"shout": shout_slug, "shout": shout.id,
"amount": amount "amount": amount
}) })
session.add(viewed) session.add(viewed)

49
services/zine/shoutauthor.py
View File

@ -1,49 +0,0 @@
import asyncio
import time
from base.orm import local_session
from orm.shout import ShoutAuthor
class ShoutAuthorStorage:
authors_by_shout = {}
lock = asyncio.Lock()
# period = 30 * 60 # sec
@staticmethod
async def load_captions(session):
self = ShoutAuthorStorage
sas = session.query(ShoutAuthor).all()
for sa in sas:
self.authors_by_shout[sa.shout] = self.authors_by_shout.get(sa.shout, {})
self.authors_by_shout[sa.shout][sa.user] = sa.caption
print("[zine.authors] ⎧ %d shouts indexed by authors" % len(self.authors_by_shout))
@staticmethod
async def get_author_caption(shout, author):
self = ShoutAuthorStorage
async with self.lock:
return self.authors_by_shout.get(shout, {}).get(author)
@staticmethod
async def set_author_caption(shout, author, caption):
self = ShoutAuthorStorage
async with self.lock:
self.authors_by_shout[shout] = self.authors_by_shout.get(shout, {})
self.authors_by_shout[shout][author] = caption
return {
"error": None,
}
@staticmethod
async def worker():
self = ShoutAuthorStorage
async with self.lock:
# while True:
try:
with local_session() as session:
ts = time.time()
await self.load_captions(session)
print("[zine.authors] ⎩ load_captions took %fs " % (time.time() - ts))
except Exception as err:
print("[zine.authors] ⎩ error indexing by author: %s" % (err))
# await asyncio.sleep(self.period)

97
services/zine/topics.py
View File

@ -1,97 +0,0 @@
import asyncio
from base.orm import local_session
from orm.topic import Topic
from orm.shout import Shout
import sqlalchemy as sa
from sqlalchemy import select
class TopicStorage:
topics = {}
lock = asyncio.Lock()
random_topics = []
@staticmethod
def init(session):
self = TopicStorage
topics = session.query(Topic)
self.topics = dict([(topic.slug, topic) for topic in topics])
for tpc in self.topics.values():
# self.load_parents(tpc)
pass
print("[zine.topics] %d precached" % len(self.topics.keys()))
# @staticmethod
# def load_parents(topic):
# self = TopicStorage
# parents = []
# for parent in self.topics.values():
# if topic.slug in parent.children:
# parents.append(parent.slug)
# topic.parents = parents
# return topic
@staticmethod
def get_random_topics(amount):
return TopicStorage.random_topics[0:amount]
@staticmethod
def renew_topics_random():
with local_session() as session:
q = select(Topic).join(Shout).group_by(Topic.id).having(sa.func.count(Shout.id) > 2).order_by(
sa.func.random()).limit(50)
TopicStorage.random_topics = list(map(
lambda result_item: result_item.Topic, session.execute(q)
))
@staticmethod
async def worker():
self = TopicStorage
async with self.lock:
while True:
try:
self.renew_topics_random()
except Exception as err:
print("[zine.topics] error %s" % (err))
await asyncio.sleep(300) # 5 mins
@staticmethod
async def get_topics_all():
self = TopicStorage
async with self.lock:
return list(self.topics.values())
@staticmethod
async def get_topics_by_slugs(slugs):
self = TopicStorage
async with self.lock:
if not slugs:
return self.topics.values()
topics = filter(lambda topic: topic.slug in slugs, self.topics.values())
return list(topics)
@staticmethod
async def get_topics_by_community(community):
self = TopicStorage
async with self.lock:
topics = filter(
lambda topic: topic.community == community, self.topics.values()
)
return list(topics)
@staticmethod
async def get_topics_by_author(author):
self = TopicStorage
async with self.lock:
topics = filter(
lambda topic: topic.community == author, self.topics.values()
)
return list(topics)
@staticmethod
async def update_topic(topic):
self = TopicStorage
async with self.lock:
self.topics[topic.slug] = topic
# self.load_parents(topic)