discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

resolvers, orm, migration, schema fixes

Browse Source
This commit is contained in:
tonyrewin 2022-08-14 15:48:35 +03:00
parent 3bda452455
commit ab9d03aac6
13 changed files with 211 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -145,5 +145,5 @@ migration/content/**/*.md
.DS_Store
dump
.vscode
*.sql
*dump.sql
*.csv

8
base/resolvers.py
View File

@ -1,15 +1,13 @@
from ariadne import MutationType, QueryType, SubscriptionType, ScalarType
query = QueryType()
mutation = MutationType()
subscription = SubscriptionType()
datetime_scalar = ScalarType("DateTime")
@datetime_scalar.serializer
def serialize_datetime(value):
return value.isoformat()
query = QueryType()
mutation = MutationType()
subscription = SubscriptionType()
resolvers = [query, mutation, subscription, datetime_scalar]

5
migration/tables/content_items.py
View File

@ -123,12 +123,15 @@ def migrate(entry, storage):
#del shout_dict['ratings']
email = userdata.get('email')
slug = userdata.get('slug')
if not slug: raise Exception
with local_session() as session:
# c = session.query(Community).all().pop()
if email: user = session.query(User).filter(User.email == email).first()
if not user and slug: user = session.query(User).filter(User.slug == slug).first()
if not user and userdata:
try: user = User.create(**userdata)
try:
userdata['slug'] = userdata['slug'].lower().strip().replace(' ', '-')
user = User.create(**userdata)
except sqlalchemy.exc.IntegrityError:
print('[migration] user error: ' + userdata)
userdata['id'] = user.id

7
migration/tables/users.py
View File

@ -29,7 +29,7 @@ def migrate(entry):
if 'wasOnineAt' in entry: user_dict['wasOnlineAt'] = parse(entry['wasOnlineAt'])
if entry.get('profile'):
# slug
user_dict['slug'] = entry['profile'].get('path')
user_dict['slug'] = entry['profile'].get('path').lower().replace(' ', '-').strip()
user_dict['bio'] = html2text(entry.get('profile').get('bio') or '')
# userpic
@ -41,10 +41,10 @@ def migrate(entry):
# name
fn = entry['profile'].get('firstName', '')
ln = entry['profile'].get('lastName', '')
name = user_dict['slug'] if user_dict['slug'] else 'noname'
name = user_dict['slug'] if user_dict['slug'] else 'anonymous'
name = fn if fn else name
name = (name + ' ' + ln) if ln else name
name = entry['profile']['path'].lower().replace(' ', '-') if len(name) < 2 else name
name = entry['profile']['path'].lower().strip().replace(' ', '-') if len(name) < 2 else name
user_dict['name'] = name
# links
@ -63,6 +63,7 @@ def migrate(entry):
user_dict['slug'] = user_dict.get('slug', user_dict['email'].split('@')[0])
oid = user_dict['oid']
user_dict['slug'] = user_dict['slug'].lower().strip().replace(' ', '-')
try: user = User.create(**user_dict.copy())
except sqlalchemy.exc.IntegrityError:
print('[migration] cannot create user ' + user_dict['slug'])

59
orm/reaction.py
View File

@ -1,52 +1,10 @@
from datetime import datetime
from sqlalchemy import Column, String, ForeignKey, DateTime
from base.orm import Base, local_session
import enum
from base.orm import Base
from sqlalchemy import Enum
from services.stat.reacted import ReactedStorage, ReactionKind
from services.stat.viewed import ViewedStorage
class ReactionKind(enum.Enum):
AGREE = 1 # +1
DISAGREE = 2 # -1
PROOF = 3 # +1
DISPROOF = 4 # -1
ASK = 5 # +0 bookmark
PROPOSE = 6 # +0
QUOTE = 7 # +0 bookmark
COMMENT = 8 # +0
ACCEPT = 9 # +1
REJECT = 0 # -1
LIKE = 11 # +1
DISLIKE = 12 # -1
# TYPE = <reaction index> # rating diff
def kind_to_rate(kind) -> int:
if kind in [
ReactionKind.AGREE,
ReactionKind.LIKE,
ReactionKind.PROOF,
ReactionKind.ACCEPT
]: return 1
elif kind in [
ReactionKind.DISAGREE,
ReactionKind.DISLIKE,
ReactionKind.DISPROOF,
ReactionKind.REJECT
]: return -1
else: return 0
def get_bookmarked(reactions):
c = 0
for r in reactions:
c += 1 if r.kind in [ ReactionKind.QUOTE, ReactionKind.ASK] else 0
return c
def get_rating(reactions):
rating = 0
for r in reactions:
rating += kind_to_rate(r.kind)
return rating
class Reaction(Base):
__tablename__ = 'reaction'
body: str = Column(String, nullable=True, comment="Reaction Body")
@ -64,15 +22,10 @@ class Reaction(Base):
@property
async def stat(self):
reacted = []
try:
with local_session() as session:
reacted = session.query(Reaction).filter(Reaction.replyTo == self.id).all()
except Exception as e:
print(e)
rrr = await ReactedStorage.get_reaction(self.id)
print(rrr[0])
return {
"viewed": await ViewedStorage.get_reaction(self.id),
"reacted": reacted.count(),
"rating": get_rating(reacted),
"bookmarked": get_bookmarked(reacted)
"reacted": len(rrr),
"rating": await ReactedStorage.get_reaction_rating(self.id)
}

20
orm/shout.py
View File

@ -3,10 +3,10 @@ from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, Boolean
from sqlalchemy.orm import relationship
from orm.user import User
from orm.topic import Topic, ShoutTopic
from orm.reaction import Reaction, get_bookmarked
from services.stat.reacted import ReactedStorage
from orm.reaction import Reaction
from services.stat.reacted import ReactedStorage, ReactionKind
from services.stat.viewed import ViewedStorage
from base.orm import Base, local_session
from base.orm import Base
class ShoutReactionsFollower(Base):
@ -63,15 +63,9 @@ class Shout(Base):
@property
async def stat(self):
reacted = []
try:
with local_session() as session:
reacted = session.query(Reaction).where(Reaction.shout == self.slug).all()
except Exception as e:
print(e)
rrr = await ReactedStorage.get_shout(self.slug)
return {
"viewed": await ViewedStorage.get_shout(self.slug),
"reacted": await ReactedStorage.get_shout(self.slug),
"rating": await ReactedStorage.get_rating(self.slug),
"bookmarked": get_bookmarked(reacted)
}
"reacted": len(rrr),
"rating": await ReactedStorage.get_rating(self.slug)
}

4
queries/shout-by-slug.sql Normal file
View File

@ -0,0 +1,4 @@
SELECT s.*, a.*, sa.* FROM shout s
JOIN shout_author sa ON s.slug = sa.shout
JOIN user a ON a.slug = sa.user
WHERE sa.slug = a.slug AND a.slug = %s;

14
resolvers/reactions.py
View File

@ -6,6 +6,7 @@ from orm.user import User
from base.resolvers import mutation, query
from auth.authenticate import login_required
from datetime import datetime
from services.auth.users import UserStorage
from services.stat.reacted import ReactedStorage
def reactions_follow(user, slug, auto=False):
@ -103,11 +104,15 @@ async def delete_reaction(_, info, id):
return {}
@query.field("reactionsByShout")
def get_shout_reactions(_, info, slug, page, size):
async def get_shout_reactions(_, info, slug, page, size):
offset = page * size
reactions = []
with local_session() as session:
reactions = session.query(Reaction).filter(Reaction.shout == slug).limit(size).offset(offset).all()
reactions = session.query(Reaction).\
filter(Reaction.shout == slug).\
limit(size).offset(offset).all()
for r in reactions:
r.createdBy = await UserStorage.get_user(r.createdBy)
return reactions
@ -116,12 +121,13 @@ def get_all_reactions(_, info, page=1, size=10):
offset = page * size
reactions = []
with local_session() as session:
stmt = session.query(Reaction).\
# raw sql: statement = text(open('queries/reactions-all.sql', 'r').read()))
statement = session.query(Reaction).\
filter(Reaction.deletedAt == None).\
order_by(desc("createdAt")).\
offset(offset).limit(size)
reactions = []
for row in session.execute(stmt):
for row in session.execute(statement):
reaction = row.Reaction
reactions.append(reaction)
reactions.sort(key=lambda x: x.createdAt, reverse=True)

218
resolvers/zine.py
View File

@ -3,6 +3,7 @@ from orm.shout import Shout, ShoutAuthor, ShoutTopic
from orm.topic import Topic
from base.orm import local_session
from base.resolvers import mutation, query
from services.zine.shoutauthor import ShoutAuthorStorage
from services.zine.shoutscache import ShoutsCache
from services.stat.viewed import ViewedStorage
from resolvers.profile import author_follow, author_unfollow
@ -10,152 +11,171 @@ from resolvers.topics import topic_follow, topic_unfollow
from resolvers.community import community_follow, community_unfollow
from resolvers.reactions import reactions_follow, reactions_unfollow
from auth.authenticate import login_required
from sqlalchemy import select, desc, and_
from sqlalchemy.orm import selectinload, joinedload
from sqlalchemy import select, desc, and_, text
from sqlalchemy.orm import selectinload
from sqlalchemy.dialects import postgresql
@query.field("topViewed")
async def top_viewed(_, info, page, size):
async with ShoutsCache.lock:
return ShoutsCache.top_viewed[(page - 1) * size : page * size]
async with ShoutsCache.lock:
return ShoutsCache.top_viewed[(page - 1) * size : page * size]
@query.field("topMonth")
async def top_month(_, info, page, size):
async with ShoutsCache.lock:
return ShoutsCache.top_month[(page - 1) * size : page * size]
async with ShoutsCache.lock:
return ShoutsCache.top_month[(page - 1) * size : page * size]
@query.field("topOverall")
async def top_overall(_, info, page, size):
async with ShoutsCache.lock:
return ShoutsCache.top_overall[(page - 1) * size : page * size]
async with ShoutsCache.lock:
return ShoutsCache.top_overall[(page - 1) * size : page * size]
@query.field("recentPublished")
async def recent_published(_, info, page, size):
async with ShoutsCache.lock:
return ShoutsCache.recent_published[(page - 1) * size : page * size]
async with ShoutsCache.lock:
return ShoutsCache.recent_published[(page - 1) * size : page * size]
@query.field("recentAll")
async def recent_all(_, info, page, size):
async with ShoutsCache.lock:
return ShoutsCache.recent_all[(page - 1) * size : page * size]
async with ShoutsCache.lock:
return ShoutsCache.recent_all[(page - 1) * size : page * size]
@query.field("recentReacted")
async def recent_reacted(_, info, page, size):
async with ShoutsCache.lock:
return ShoutsCache.recent_reacted[(page - 1) * size : page * size]
async with ShoutsCache.lock:
return ShoutsCache.recent_reacted[(page - 1) * size : page * size]
@mutation.field("viewShout")
async def view_shout(_, info, slug):
await ViewedStorage.inc_shout(slug)
return {"error" : ""}
await ViewedStorage.inc_shout(slug)
return {"error" : ""}
@query.field("getShoutBySlug")
async def get_shout_by_slug(_, info, slug):
shout = None
# FIXME: append captions anyhow
with local_session() as session:
shout = session.query(Shout, ShoutAuthor.caption.label("author_caption")).\
options([
selectinload(Shout.topics),
selectinload(Shout.reactions),
joinedload(Shout.authors),
selectinload(ShoutAuthor.caption)
]).\
join(ShoutAuthor.shout == slug ).\
filter(Shout.slug == slug).first()
all_fields = [node.name.value for node in info.field_nodes[0].selection_set.selections]
selected_fields = set(["authors", "topics"]).intersection(all_fields)
select_options = [selectinload(getattr(Shout, field)) for field in selected_fields]
if not shout:
print(f"[resolvers.zine] error: shout with slug {slug} not exist")
return {"error" : "shout not found"}
with local_session() as session:
try: s = text(open('src/queries/shout-by-slug.sql', 'r').read() % slug)
except: pass
shout_q = session.query(Shout).\
options(select_options).\
filter(Shout.slug == slug)
return shout
print(shout_q.statement)
shout = shout_q.first()
for a in shout.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(slug, a.slug)
if not shout:
print(f"shout with slug {slug} not exist")
return {"error" : "shout not found"}
return shout
@query.field("shoutsByTopics")
async def shouts_by_topics(_, info, slugs, page, size):
page = page - 1
with local_session() as session:
shouts = session.query(Shout).\
join(ShoutTopic).\
where(and_(ShoutTopic.topic.in_(slugs), Shout.publishedAt != None)).\
order_by(desc(Shout.publishedAt)).\
limit(size).\
offset(page * size)
return shouts
page = page - 1
with local_session() as session:
shouts = session.query(Shout).\
join(ShoutTopic).\
where(and_(ShoutTopic.topic.in_(slugs), Shout.publishedAt != None)).\
order_by(desc(Shout.publishedAt)).\
limit(size).\
offset(page * size)
for s in shouts:
for a in s.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
return shouts
@query.field("shoutsByCollection")
async def shouts_by_topics(_, info, collection, page, size):
page = page - 1
with local_session() as session:
shouts = session.query(Shout).\
join(ShoutCollection, ShoutCollection.collection == collection).\
where(and_(ShoutCollection.shout == Shout.slug, Shout.publishedAt != None)).\
order_by(desc(Shout.publishedAt)).\
limit(size).\
offset(page * size)
return shouts
page = page - 1
shouts = []
with local_session() as session:
shouts = session.query(Shout).\
join(ShoutCollection, ShoutCollection.collection == collection).\
where(and_(ShoutCollection.shout == Shout.slug, Shout.publishedAt != None)).\
order_by(desc(Shout.publishedAt)).\
limit(size).\
offset(page * size)
for s in shouts:
for a in s.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
return shouts
@query.field("shoutsByAuthors")
async def shouts_by_authors(_, info, slugs, page, size):
page = page - 1
with local_session() as session:
page = page - 1
with local_session() as session:
shouts = session.query(Shout).\
join(ShoutAuthor).\
where(and_(ShoutAuthor.user.in_(slugs), Shout.publishedAt != None)).\
order_by(desc(Shout.publishedAt)).\
limit(size).\
offset(page * size)
return shouts
shouts = session.query(Shout).\
join(ShoutAuthor).\
where(and_(ShoutAuthor.user.in_(slugs), Shout.publishedAt != None)).\
order_by(desc(Shout.publishedAt)).\
limit(size).\
offset(page * size)
for s in shouts:
for a in s.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
return shouts
@query.field("shoutsByCommunities")
async def shouts_by_communities(_, info, slugs, page, size):
page = page - 1
with local_session() as session:
#TODO fix postgres high load
shouts = session.query(Shout).distinct().\
join(ShoutTopic).\
where(and_(Shout.publishedAt != None,\
ShoutTopic.topic.in_(\
select(Topic.slug).where(Topic.community.in_(slugs))\
))).\
order_by(desc(Shout.publishedAt)).\
limit(size).\
offset(page * size)
return shouts
page = page - 1
with local_session() as session:
#TODO fix postgres high load
shouts = session.query(Shout).distinct().\
join(ShoutTopic).\
where(and_(Shout.publishedAt != None,\
ShoutTopic.topic.in_(\
select(Topic.slug).where(Topic.community.in_(slugs))\
))).\
order_by(desc(Shout.publishedAt)).\
limit(size).\
offset(page * size)
for s in shouts:
for a in s.authors:
a.caption = await ShoutAuthorStorage.get_author_caption(s.slug, a.slug)
return shouts
@mutation.field("follow")
@login_required
async def follow(_, info, what, slug):
user = info.context["request"].user
try:
if what == "AUTHOR":
author_follow(user, slug)
elif what == "TOPIC":
topic_follow(user, slug)
elif what == "COMMUNITY":
community_follow(user, slug)
elif what == "REACTIONS":
reactions_follow(user, slug)
except Exception as e:
return {"error" : str(e)}
user = info.context["request"].user
try:
if what == "AUTHOR":
author_follow(user, slug)
elif what == "TOPIC":
topic_follow(user, slug)
elif what == "COMMUNITY":
community_follow(user, slug)
elif what == "REACTIONS":
reactions_follow(user, slug)
except Exception as e:
return {"error" : str(e)}
return {}
return {}
@mutation.field("unfollow")
@login_required
async def unfollow(_, info, what, slug):
user = info.context["request"].user
user = info.context["request"].user
try:
if what == "AUTHOR":
author_unfollow(user, slug)
elif what == "TOPIC":
topic_unfollow(user, slug)
elif what == "COMMUNITY":
community_unfollow(user, slug)
elif what == "REACTIONS":
reactions_unfollow(user, slug)
except Exception as e:
return {"error" : str(e)}
try:
if what == "AUTHOR":
author_unfollow(user, slug)
elif what == "TOPIC":
topic_unfollow(user, slug)
elif what == "COMMUNITY":
community_unfollow(user, slug)
elif what == "REACTIONS":
reactions_unfollow(user, slug)
except Exception as e:
return {"error" : str(e)}
return {}
return {}

2
schema.graphql
View File

@ -241,7 +241,7 @@ type Query {
# reactons
reactionsAll(page: Int!, size: Int!): [Reaction]!
reactionsByAuthor(slug: String!, page: Int!, size: Int!): [Reaction]!
reactionsByShout(slug: String!): [Reaction]!
reactionsByShout(slug: String!, page: Int!, size: Int!): [Reaction]!
# collab
inviteAuthor(slug: String!, author: String!): Result!

48
services/stat/reacted.py
View File

@ -1,12 +1,42 @@
import asyncio
from datetime import datetime
from sqlalchemy.types import Enum
from sqlalchemy import Column, DateTime, ForeignKey, Integer
from sqlalchemy import Column, DateTime, ForeignKey
# from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy import Enum
import enum
from base.orm import Base, local_session
from orm.reaction import Reaction, ReactionKind, kind_to_rate
from orm.topic import ShoutTopic
class ReactionKind(enum.Enum):
AGREE = 1 # +1
DISAGREE = 2 # -1
PROOF = 3 # +1
DISPROOF = 4 # -1
ASK = 5 # +0 bookmark
PROPOSE = 6 # +0
QUOTE = 7 # +0 bookmark
COMMENT = 8 # +0
ACCEPT = 9 # +1
REJECT = 0 # -1
LIKE = 11 # +1
DISLIKE = 12 # -1
# TYPE = <reaction index> # rating diff
def kind_to_rate(kind) -> int:
if kind in [
ReactionKind.AGREE,
ReactionKind.LIKE,
ReactionKind.PROOF,
ReactionKind.ACCEPT
]: return 1
elif kind in [
ReactionKind.DISAGREE,
ReactionKind.DISLIKE,
ReactionKind.DISPROOF,
ReactionKind.REJECT
]: return -1
else: return 0
class ReactedByDay(Base):
__tablename__ = "reacted_by_day"
@ -37,12 +67,8 @@ class ReactedStorage:
def init(session):
self = ReactedStorage
all_reactions = session.query(ReactedByDay).all()
all_reactions2 = session.query(Reaction).filter(Reaction.deletedAt == None).all()
print('[stat.reacted] %d reactions total' % len(all_reactions or all_reactions2))
rrr = (all_reactions or all_reactions2)
create = False
if not all_reactions: create = True
for reaction in rrr:
print('[stat.reacted] %d reactions total' % len(all_reactions))
for reaction in all_reactions:
shout = reaction.shout
topics = session.query(ShoutTopic.topic).where(ShoutTopic.shout == shout).all()
kind = reaction.kind
@ -65,12 +91,6 @@ class ReactedStorage:
print('[stat.reacted] %d shouts reacted' % len(self.reacted['shouts']))
print('[stat.reacted] %d reactions reacted' % len(self.reacted['reactions']))
if len(all_reactions) == 0 and len(all_reactions2) != 0:
with local_session() as session:
for r in all_reactions2:
session.add(ReactedByDay(reaction=r.id, shout=r.shout, reply=r.replyTo, kind=r.kind, day=r.createdAt.replace(hour=0, minute=0, second=0)))
session.commit()
@staticmethod
async def get_shout(shout_slug):
self = ReactedStorage

6
services/stat/topicstat.py
View File

@ -65,9 +65,9 @@ class TopicStat:
"shouts" : len(shouts),
"authors" : len(authors),
"followers" : len(followers),
"viewed": ViewedStorage.get_topic(topic),
"reacted" : ReactedStorage.get_topic(topic),
"rating" : ReactedStorage.get_topic_rating(topic),
"viewed": await ViewedStorage.get_topic(topic),
"reacted" : await ReactedStorage.get_topic(topic),
"rating" : await ReactedStorage.get_topic_rating(topic),
}
@staticmethod

21
services/zine/shoutauthor.py
View File

@ -12,14 +12,10 @@ class ShoutAuthorStorage:
@staticmethod
async def load(session):
self = ShoutAuthorStorage
authors = session.query(ShoutAuthor).all()
for author in authors:
user = author.user
shout = author.shout
if shout in self.authors_by_shout:
self.authors_by_shout[shout].append(user)
else:
self.authors_by_shout[shout] = [user]
sas = session.query(ShoutAuthor).all()
for sa in sas:
self.authors_by_shout[sa.shout] = self.authors_by_shout.get(sa.shout, [])
self.authors_by_shout[sa.shout].append([sa.user, sa.caption])
print('[zine.authors] %d shouts preprocessed' % len(self.authors_by_shout))
@staticmethod
@ -28,6 +24,15 @@ class ShoutAuthorStorage:
async with self.lock:
return self.authors_by_shout.get(shout, [])
@staticmethod
async def get_author_caption(shout, author):
self = ShoutAuthorStorage
async with self.lock:
for a in self.authors_by_shout.get(shout, []):
if author in a:
return a[1]
return { "error": "author caption not found" }
@staticmethod
async def worker():
self = ShoutAuthorStorage