discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

topic stat via queries

Browse Source
This commit is contained in:
Igor Lobanov 2022-11-28 21:29:02 +01:00
parent 32d668b53c
commit 49ac3e97e5
7 changed files with 139 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
migration/tables/topics.py
View File

@ -9,9 +9,10 @@ def migrate(entry):
topic_dict = {
"slug": entry["slug"],
"oid": entry["_id"],
"title": entry["title"].replace(" ", " ")
"title": entry["title"].replace(" ", " "),
"body": extract_md(html2text(body_orig), entry["_id"])
}
topic_dict["body"] = extract_md(html2text(body_orig), entry["_id"])
with local_session() as session:
slug = topic_dict["slug"]
topic = session.query(Topic).filter(Topic.slug == slug).first() or Topic.create(

3
migration/tables/users.py
View File

@ -23,8 +23,9 @@ def migrate(entry):
"notifications": [],
"links": [],
"name": "anonymous",
"password": entry["services"]["password"].get("bcrypt")
}
user_dict["password"] = entry["services"]["password"].get("bcrypt")
if "updatedAt" in entry:
user_dict["updatedAt"] = parse(entry["updatedAt"])
if "wasOnineAt" in entry:

86
resolvers/zine/load.py
View File

@ -9,72 +9,47 @@ from orm.shout import Shout, ShoutAuthor
from orm.reaction import Reaction, ReactionKind
def add_viewed_stat_column(q):
return q.outerjoin(ViewedEntry).add_columns(sa.func.sum(ViewedEntry.amount).label('viewed_stat'))
def add_stat_columns(q):
q = q.outerjoin(ViewedEntry).add_columns(sa.func.sum(ViewedEntry.amount).label('viewed_stat'))
def add_reacted_stat_column(q):
aliased_reaction = aliased(Reaction)
return q.outerjoin(aliased_reaction).add_columns(sa.func.count(aliased_reaction.id).label('reacted_stat'))
def add_commented_stat_column(q):
aliased_reaction = aliased(Reaction)
return q.outerjoin(
aliased_reaction,
aliased_reaction.shout == Shout.slug and aliased_reaction.body.is_not(None)
).add_columns(sa.func.count(aliased_reaction.id).label('commented_stat'))
def add_rating_stat_column(q):
return q.outerjoin(Reaction).add_columns(sa.func.sum(case(
(Reaction.kind == ReactionKind.AGREE, 1),
(Reaction.kind == ReactionKind.DISAGREE, -1),
(Reaction.kind == ReactionKind.PROOF, 1),
(Reaction.kind == ReactionKind.DISPROOF, -1),
(Reaction.kind == ReactionKind.ACCEPT, 1),
(Reaction.kind == ReactionKind.REJECT, -1),
(Reaction.kind == ReactionKind.LIKE, 1),
(Reaction.kind == ReactionKind.DISLIKE, -1),
q = q.outerjoin(aliased_reaction).add_columns(
sa.func.sum(
aliased_reaction.id
).label('reacted_stat'),
sa.func.sum(
case(
(aliased_reaction.body.is_not(None), 1),
else_=0
)).label('rating_stat'))
)
).label('commented_stat'),
sa.func.sum(case(
(aliased_reaction.kind == ReactionKind.AGREE, 1),
(aliased_reaction.kind == ReactionKind.DISAGREE, -1),
(aliased_reaction.kind == ReactionKind.PROOF, 1),
(aliased_reaction.kind == ReactionKind.DISPROOF, -1),
(aliased_reaction.kind == ReactionKind.ACCEPT, 1),
(aliased_reaction.kind == ReactionKind.REJECT, -1),
(aliased_reaction.kind == ReactionKind.LIKE, 1),
(aliased_reaction.kind == ReactionKind.DISLIKE, -1),
else_=0)
).label('rating_stat'))
# def calc_reactions(q):
# aliased_reaction = aliased(Reaction)
# return q.join(aliased_reaction).add_columns(
# sa.func.sum(case(
# (aliased_reaction.kind == ReactionKind.AGREE, 1),
# (aliased_reaction.kind == ReactionKind.DISAGREE, -1),
# (aliased_reaction.kind == ReactionKind.PROOF, 1),
# (aliased_reaction.kind == ReactionKind.DISPROOF, -1),
# (aliased_reaction.kind == ReactionKind.ACCEPT, 1),
# (aliased_reaction.kind == ReactionKind.REJECT, -1),
# (aliased_reaction.kind == ReactionKind.LIKE, 1),
# (aliased_reaction.kind == ReactionKind.DISLIKE, -1),
# else_=0)
# ).label('rating'),
# sa.func.sum(
# case(
# (aliased_reaction.body.is_not(None), 1),
# else_=0
# )
# ).label('commented'),
# sa.func.sum(
# aliased_reaction.id
# ).label('reacted')
# )
return q
def apply_filters(q, filters, user=None):
filters = {} if filters is None else filters
if filters.get("reacted") and user:
q.join(Reaction, Reaction.createdBy == user.slug)
v = filters.get("visibility")
if v == "public":
q = q.filter(Shout.visibility == filters.get("visibility"))
if v == "community":
q = q.filter(Shout.visibility.in_(["public", "community"]))
if filters.get("layout"):
q = q.filter(Shout.layout == filters.get("layout"))
if filters.get("author"):
@ -88,14 +63,7 @@ def apply_filters(q, filters, user=None):
if filters.get("days"):
before = datetime.now(tz=timezone.utc) - timedelta(days=int(filters.get("days")) or 30)
q = q.filter(Shout.createdAt > before)
return q
def add_stat_columns(q):
q = add_viewed_stat_column(q)
q = add_reacted_stat_column(q)
q = add_commented_stat_column(q)
q = add_rating_stat_column(q)
return q
@ -162,7 +130,7 @@ async def load_shouts_by(_, info, options):
q = add_stat_columns(q)
user = info.context["request"].user
q = apply_filters(q, options.get("filters"), user)
q = apply_filters(q, options.get("filters", {}), user)
order_by = options.get("order_by", Shout.createdAt)
if order_by == 'reacted':

35
resolvers/zine/profile.py
View File

@ -1,18 +1,20 @@
from typing import List
from datetime import datetime, timedelta, timezone
from sqlalchemy import and_, func
from sqlalchemy import and_, func, select
from sqlalchemy.orm import selectinload
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.reaction import Reaction
from orm.shout import ShoutAuthor
from orm.shout import ShoutAuthor, ShoutTopic
from orm.topic import Topic, TopicFollower
from orm.user import AuthorFollower, Role, User, UserRating, UserRole
# from .community import followed_communities
from resolvers.inbox.unread import get_total_unread_counter
from resolvers.zine.topics import followed_by_user
async def user_subscriptions(slug: str):
return {
@ -66,17 +68,7 @@ async def get_followed_topics(_, info, slug) -> List[Topic]:
async def followed_topics(slug):
topics = []
with local_session() as session:
topics = (
session.query(Topic)
.join(TopicFollower)
.where(TopicFollower.follower == slug)
.all()
)
for topic in topics:
topic.stat = await get_topic_stat(topic.slug)
return topics
return followed_by_user(slug)
@query.field("userFollowedAuthors")
@ -204,7 +196,6 @@ async def get_author(_, _info, slug):
@query.field("loadAuthorsBy")
async def load_authors_by(_, info, by, limit, offset):
authors = []
with local_session() as session:
aq = session.query(User)
if by.get("slug"):
@ -212,24 +203,26 @@ async def load_authors_by(_, info, by, limit, offset):
elif by.get("name"):
aq = aq.filter(User.name.ilike(f"%{by['name']}%"))
elif by.get("topic"):
aaa = list(map(lambda a: a.slug, TopicStat.authors_by_topic.get(by["topic"])))
aq = aq.filter(User.name._in(aaa))
aq = aq.join(ShoutAuthor).join(ShoutTopic).where(ShoutTopic.topic == by["topic"])
if by.get("lastSeen"): # in days
days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["lastSeen"])
aq = aq.filter(User.lastSeen > days_before)
elif by.get("createdAt"): # in days
days_before = datetime.now(tz=timezone.utc) - timedelta(days=by["createdAt"])
aq = aq.filter(User.createdAt > days_before)
aq = aq.group_by(
User.id
).order_by(
by.get("order") or "createdAt"
).limit(limit).offset(offset)
print(aq)
authors = list(map(lambda r: r.User, session.execute(aq)))
authors = []
for [author] in session.execute(aq):
if by.get("stat"):
for a in authors:
a.stat = await get_author_stat(a.slug)
authors = list(set(authors))
# authors = sorted(authors, key=lambda a: a["stat"].get(by.get("stat")))
author.stat = await get_author_stat(author.slug)
authors.append(author)
return authors

10
resolvers/zine/reactions.py
View File

@ -10,15 +10,6 @@ from orm.shout import Shout, ShoutReactionsFollower
from orm.user import User
async def get_reaction_stat(reaction_id):
return {
# "viewed": await ViewedStorage.get_reaction(reaction_id),
"reacted": len(await ReactedStorage.get_reaction(reaction_id)),
"rating": await ReactedStorage.get_reaction_rating(reaction_id),
"commented": len(await ReactedStorage.get_reaction_comments(reaction_id)),
}
def reactions_follow(user: User, slug: str, auto=False):
with local_session() as session:
following = (
@ -205,6 +196,7 @@ async def delete_reaction(_, info, rid):
session.commit()
return {}
@query.field("loadReactionsBy")
async def load_reactions_by(_, _info, by, limit=50, offset=0):
"""

109
resolvers/zine/topics.py
View File

@ -1,52 +1,94 @@
import sqlalchemy as sa
from sqlalchemy import and_, select
from sqlalchemy import and_, select, distinct
from auth.authenticate import login_required
from base.orm import local_session
from base.resolvers import mutation, query
from orm.shout import ShoutTopic, ShoutAuthor
from orm.topic import Topic, TopicFollower
from orm import Shout
async def get_topic_stat(slug):
return {
"shouts": len(TopicStat.shouts_by_topic.get(slug, {}).keys()),
"authors": len(TopicStat.authors_by_topic.get(slug, {}).keys()),
"followers": len(TopicStat.followers_by_topic.get(slug, {}).keys())
def add_topic_stat_columns(q):
q = q.outerjoin(ShoutTopic, Topic.slug == ShoutTopic.topic).add_columns(
sa.func.count(distinct(ShoutTopic.shout)).label('shouts_stat')
).outerjoin(ShoutAuthor, ShoutTopic.shout == ShoutAuthor.shout).add_columns(
sa.func.count(distinct(ShoutAuthor.user)).label('authors_stat')
).outerjoin(TopicFollower,
and_(
TopicFollower.topic == Topic.slug,
TopicFollower.follower == ShoutAuthor.user
)).add_columns(
sa.func.count(distinct(TopicFollower.follower)).label('followers_stat')
)
q = q.group_by(Topic.id)
return q
def add_stat(topic, stat_columns):
[shouts_stat, authors_stat, followers_stat] = stat_columns
topic.stat = {
"shouts": shouts_stat,
"authors": authors_stat,
"followers": followers_stat
}
return topic
def get_topics_from_query(q):
topics = []
with local_session() as session:
for [topic, *stat_columns] in session.execute(q):
topic = add_stat(topic, stat_columns)
topics.append(topic)
return topics
def followed_by_user(user_slug):
q = select(Topic)
q = add_topic_stat_columns(q)
q = q.where(TopicFollower.follower == user_slug)
return get_topics_from_query(q)
@query.field("topicsAll")
async def topics_all(_, _info):
topics = await TopicStorage.get_topics_all()
for topic in topics:
topic.stat = await get_topic_stat(topic.slug)
return topics
q = select(Topic)
q = add_topic_stat_columns(q)
return get_topics_from_query(q)
@query.field("topicsByCommunity")
async def topics_by_community(_, info, community):
topics = await TopicStorage.get_topics_by_community(community)
for topic in topics:
topic.stat = await get_topic_stat(topic.slug)
return topics
q = select(Topic).where(Topic.community == community)
q = add_topic_stat_columns(q)
return get_topics_from_query(q)
@query.field("topicsByAuthor")
async def topics_by_author(_, _info, author):
shouts = TopicStorage.get_topics_by_author(author)
author_topics = set()
for s in shouts:
for tpc in s.topics:
tpc = await TopicStorage.topics[tpc.slug]
tpc.stat = await get_topic_stat(tpc.slug)
author_topics.add(tpc)
return list(author_topics)
q = select(Topic)
q = add_topic_stat_columns(q)
q = q.where(ShoutAuthor.user == author)
return get_topics_from_query(q)
@query.field("getTopic")
async def get_topic(_, _info, slug):
t = TopicStorage.topics[slug]
t.stat = await get_topic_stat(slug)
return t
q = select(Topic).where(Topic.slug == slug)
q = add_topic_stat_columns(q)
topics = get_topics_from_query(q)
return topics[0]
@mutation.field("createTopic")
@ -57,7 +99,7 @@ async def create_topic(_, _info, inp):
new_topic = Topic.create(**inp)
session.add(new_topic)
session.commit()
await TopicStorage.update_topic(new_topic)
return {"topic": new_topic}
@ -72,7 +114,7 @@ async def update_topic(_, _info, inp):
else:
topic.update(**inp)
session.commit()
await TopicStorage.update_topic(topic.slug)
return {"topic": topic}
@ -81,7 +123,6 @@ async def topic_follow(user, slug):
following = TopicFollower.create(topic=slug, follower=user.slug)
session.add(following)
session.commit()
await TopicStorage.update_topic(slug)
async def topic_unfollow(user, slug):
@ -99,13 +140,13 @@ async def topic_unfollow(user, slug):
else:
session.delete(sub)
session.commit()
await TopicStorage.update_topic(slug)
@query.field("topicsRandom")
async def topics_random(_, info, amount=12):
with local_session() as session:
q = select(Topic).join(Shout).group_by(Topic.id).having(sa.func.count(Shout.id) > 2).order_by(
sa.func.random()).limit(amount)
random_topics = list(map(lambda result_item: result_item.Topic, session.execute(q)))
return random_topics
q = select(Topic)
q = add_topic_stat_columns(q)
q = q.join(Shout, ShoutTopic.shout == Shout.slug).group_by(Topic.id).having(sa.func.count(Shout.id) > 2)
q = q.order_by(sa.func.random()).limit(amount)
return get_topics_from_query(q)

97
services/zine/topics.py
View File

@ -1,97 +0,0 @@
import asyncio
from base.orm import local_session
from orm.topic import Topic
from orm.shout import Shout
import sqlalchemy as sa
from sqlalchemy import select
class TopicStorage:
topics = {}
lock = asyncio.Lock()
random_topics = []
@staticmethod
def init(session):
self = TopicStorage
topics = session.query(Topic)
self.topics = dict([(topic.slug, topic) for topic in topics])
for tpc in self.topics.values():
# self.load_parents(tpc)
pass
print("[zine.topics] %d precached" % len(self.topics.keys()))
# @staticmethod
# def load_parents(topic):
# self = TopicStorage
# parents = []
# for parent in self.topics.values():
# if topic.slug in parent.children:
# parents.append(parent.slug)
# topic.parents = parents
# return topic
@staticmethod
def get_random_topics(amount):
return TopicStorage.random_topics[0:amount]
@staticmethod
def renew_topics_random():
with local_session() as session:
q = select(Topic).join(Shout).group_by(Topic.id).having(sa.func.count(Shout.id) > 2).order_by(
sa.func.random()).limit(50)
TopicStorage.random_topics = list(map(
lambda result_item: result_item.Topic, session.execute(q)
))
@staticmethod
async def worker():
self = TopicStorage
async with self.lock:
while True:
try:
self.renew_topics_random()
except Exception as err:
print("[zine.topics] error %s" % (err))
await asyncio.sleep(300) # 5 mins
@staticmethod
async def get_topics_all():
self = TopicStorage
async with self.lock:
return list(self.topics.values())
@staticmethod
async def get_topics_by_slugs(slugs):
self = TopicStorage
async with self.lock:
if not slugs:
return self.topics.values()
topics = filter(lambda topic: topic.slug in slugs, self.topics.values())
return list(topics)
@staticmethod
async def get_topics_by_community(community):
self = TopicStorage
async with self.lock:
topics = filter(
lambda topic: topic.community == community, self.topics.values()
)
return list(topics)
@staticmethod
async def get_topics_by_author(author):
self = TopicStorage
async with self.lock:
topics = filter(
lambda topic: topic.community == author, self.topics.values()
)
return list(topics)
@staticmethod
async def update_topic(topic):
self = TopicStorage
async with self.lock:
self.topics[topic.slug] = topic
# self.load_parents(topic)