From 41413242bdb2b87615b28554d42f1bd6f91778bd Mon Sep 17 00:00:00 2001 From: knst-kotov Date: Wed, 1 Sep 2021 13:28:42 +0300 Subject: [PATCH 1/3] cache top shouts --- main.py | 4 +-- resolvers/zine.py | 88 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 62 insertions(+), 30 deletions(-) diff --git a/main.py b/main.py index 3ef0979b..a473aaca 100644 --- a/main.py +++ b/main.py @@ -13,7 +13,7 @@ from auth.oauth import oauth_login, oauth_authorize from auth.email import email_authorize from redis import redis from resolvers.base import resolvers -from resolvers.zine import GitTask +from resolvers.zine import GitTask, TopShouts import asyncio @@ -28,7 +28,7 @@ middleware = [ async def start_up(): await redis.connect() git_task = asyncio.create_task(GitTask.git_task_worker()) - + top_shouts_task = asyncio.create_task(TopShouts.worker()) async def shutdown(): await redis.disconnect() diff --git a/resolvers/zine.py b/resolvers/zine.py index e8bd40a4..9f43f481 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -67,40 +67,72 @@ class GitTask: print("git task worker error = %s" % (err)) +class TopShouts: + limit = 50 + period = 60*60 #1 hour + + lock = asyncio.Lock() + + @staticmethod + async def prepare_shouts_by_rating(): + month_ago = datetime.now() - timedelta(days = 30) + with local_session() as session: + stmt = select(Shout, func.sum(ShoutRating.value).label("rating")).\ + join(ShoutRating).\ + where(ShoutRating.ts > month_ago).\ + group_by(Shout.id).\ + order_by(desc("rating")).\ + limit(TopShouts.limit) + shouts = [] + for row in session.execute(stmt): + shout = row.Shout + shout.rating = row.rating + shouts.append(shout) + async with TopShouts.lock: + TopShouts.shouts_by_rating = shouts + + @staticmethod + async def prepare_shouts_by_view(): + month_ago = datetime.now() - timedelta(days = 30) + with local_session() as session: + stmt = select(Shout, func.sum(ShoutViewByDay.value).label("view")).\ + join(ShoutViewByDay).\ + where(ShoutViewByDay.day > month_ago).\ + group_by(Shout.id).\ + order_by(desc("view")).\ + limit(TopShouts.limit) + shouts = [] + for row in session.execute(stmt): + shout = row.Shout + shout.view = row.view + shouts.append(shout) + async with TopShouts.lock: + TopShouts.shouts_by_view = shouts + + @staticmethod + async def worker(): + print("top shouts worker start") + while True: + try: + print("top shouts: update cache") + await TopShouts.prepare_shouts_by_rating() + await TopShouts.prepare_shouts_by_view() + print("top shouts: update finished") + except Exception as err: + print("top shouts worker error = %s" % (err)) + await asyncio.sleep(TopShouts.period) + + @query.field("topShoutsByView") async def top_shouts_by_view(_, info, limit): - month_ago = datetime.now() - timedelta(days = 30) - with local_session() as session: - stmt = select(Shout, func.sum(ShoutViewByDay.value).label("view")).\ - join(ShoutViewByDay).\ - where(ShoutViewByDay.day > month_ago).\ - group_by(Shout.id).\ - order_by(desc("view")).\ - limit(limit) - shouts = [] - for row in session.execute(stmt): - shout = row.Shout - shout.view = row.view - shouts.append(shout) - return shouts + async with TopShouts.lock: + return TopShouts.shouts_by_view[:limit] @query.field("topShoutsByRating") async def top_shouts(_, info, limit): - month_ago = datetime.now() - timedelta(days = 30) - with local_session() as session: - stmt = select(Shout, func.sum(ShoutRating.value).label("rating")).\ - join(ShoutRating).\ - where(ShoutRating.ts > month_ago).\ - group_by(Shout.id).\ - order_by(desc("rating")).\ - limit(limit) - shouts = [] - for row in session.execute(stmt): - shout = row.Shout - shout.rating = row.rating - shouts.append(shout) - return shouts + async with TopShouts.lock: + return TopShouts.shouts_by_rating[:limit] @mutation.field("createShout") From 6d28d2d455dbcd8da9a55f7af57c019a98067947 Mon Sep 17 00:00:00 2001 From: knst-kotov Date: Wed, 1 Sep 2021 19:03:00 +0300 Subject: [PATCH 2/3] favoritesShouts and topShoutsByAuthor --- resolvers/zine.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++- schema.graphql | 2 ++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/resolvers/zine.py b/resolvers/zine.py index 9f43f481..2a091379 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -91,6 +91,22 @@ class TopShouts: async with TopShouts.lock: TopShouts.shouts_by_rating = shouts + @staticmethod + async def prepare_favorites_shouts(): + with local_session() as session: + stmt = select(Shout, func.sum(ShoutRating.value).label("rating")).\ + join(ShoutRating).\ + group_by(Shout.id).\ + order_by(desc("rating")).\ + limit(TopShouts.limit) + shouts = [] + for row in session.execute(stmt): + shout = row.Shout + shout.rating = row.rating + shouts.append(shout) + async with TopShouts.lock: + TopShouts.favorites_shouts = shouts + @staticmethod async def prepare_shouts_by_view(): month_ago = datetime.now() - timedelta(days = 30) @@ -109,14 +125,49 @@ class TopShouts: async with TopShouts.lock: TopShouts.shouts_by_view = shouts + @staticmethod + async def prepare_shouts_by_author(): + month_ago = datetime.now() - timedelta(days = 30) + with local_session() as session: + shout_with_view = select(Shout.id, func.sum(ShoutViewByDay.value).label("view")).\ + join(ShoutViewByDay).\ + where(ShoutViewByDay.day > month_ago).\ + group_by(Shout.id).\ + order_by(desc("view")).cte() + stmt = select(ShoutAuthor.user, ShoutAuthor.shout, shout_with_view.c.view).\ + join(shout_with_view, ShoutAuthor.shout == shout_with_view.c.id) + shout_by_author = {} + for row in session.execute(stmt): + user = row.user + if user in shout_by_author: + (shout_id, view) = shout_by_author[user] + if row.view > view: + shout_by_author[user] = (row.shout, row.view) + else: + shout_by_author[user] = (row.shout, row.view) + shout_info = {} + for user, value in shout_by_author.items(): + (shout_id, view) = value + shout_info[shout_id] = (user, view) + shout_ids = shout_info.keys() + shouts = session.query(Shout).filter(Shout.id.in_(shout_ids)).all() + for shout in shouts: + (user, view) = shout_info[shout.id] + shout.view = view + async with TopShouts.lock: + TopShouts.shouts_by_author = shouts + + @staticmethod async def worker(): print("top shouts worker start") while True: try: print("top shouts: update cache") + await TopShouts.prepare_favorites_shouts() await TopShouts.prepare_shouts_by_rating() await TopShouts.prepare_shouts_by_view() + await TopShouts.prepare_shouts_by_author() print("top shouts: update finished") except Exception as err: print("top shouts worker error = %s" % (err)) @@ -130,11 +181,23 @@ async def top_shouts_by_view(_, info, limit): @query.field("topShoutsByRating") -async def top_shouts(_, info, limit): +async def top_shouts_by_rating(_, info, limit): async with TopShouts.lock: return TopShouts.shouts_by_rating[:limit] +@query.field("favoritesShouts") +async def favorites_shouts(_, info, limit): + async with TopShouts.lock: + return TopShouts.favorites_shouts[:limit] + + +@query.field("topShoutsByAuthor") +async def top_shouts_by_author(_, info, limit): + async with TopShouts.lock: + return TopShouts.shouts_by_author[:limit] + + @mutation.field("createShout") @login_required async def create_shout(_, info, input): diff --git a/schema.graphql b/schema.graphql index 93723561..a60f9e3a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -100,6 +100,8 @@ type Query { topShoutsByView(limit: Int): [Shout]! topShoutsByRating(limit: Int): [Shout]! + favoritesShouts(limit: Int): [Shout]! + topShoutsByAuthor(limit: Int): [Shout]! # getOnlineUsers: [User!]! } From db12b3f90867d24fb58479f91d9bbeaeb2e6e773 Mon Sep 17 00:00:00 2001 From: knst-kotov Date: Thu, 2 Sep 2021 14:04:58 +0300 Subject: [PATCH 3/3] add topAuthors --- resolvers/zine.py | 40 +++++++++++++++------------------------- schema.graphql | 2 +- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/resolvers/zine.py b/resolvers/zine.py index 2a091379..71e405ef 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -126,7 +126,7 @@ class TopShouts: TopShouts.shouts_by_view = shouts @staticmethod - async def prepare_shouts_by_author(): + async def prepare_top_authors(): month_ago = datetime.now() - timedelta(days = 30) with local_session() as session: shout_with_view = select(Shout.id, func.sum(ShoutViewByDay.value).label("view")).\ @@ -134,28 +134,18 @@ class TopShouts: where(ShoutViewByDay.day > month_ago).\ group_by(Shout.id).\ order_by(desc("view")).cte() - stmt = select(ShoutAuthor.user, ShoutAuthor.shout, shout_with_view.c.view).\ - join(shout_with_view, ShoutAuthor.shout == shout_with_view.c.id) - shout_by_author = {} + stmt = select(ShoutAuthor.user, func.sum(shout_with_view.c.view).label("view")).\ + join(shout_with_view, ShoutAuthor.shout == shout_with_view.c.id).\ + group_by(ShoutAuthor.user).\ + order_by(desc("view")).\ + limit(TopShouts.limit) + authors = {} for row in session.execute(stmt): - user = row.user - if user in shout_by_author: - (shout_id, view) = shout_by_author[user] - if row.view > view: - shout_by_author[user] = (row.shout, row.view) - else: - shout_by_author[user] = (row.shout, row.view) - shout_info = {} - for user, value in shout_by_author.items(): - (shout_id, view) = value - shout_info[shout_id] = (user, view) - shout_ids = shout_info.keys() - shouts = session.query(Shout).filter(Shout.id.in_(shout_ids)).all() - for shout in shouts: - (user, view) = shout_info[shout.id] - shout.view = view + authors[row.user] = row.view + authors_ids = authors.keys() + authors = session.query(User).filter(User.id.in_(authors_ids)).all() async with TopShouts.lock: - TopShouts.shouts_by_author = shouts + TopShouts.top_authors = authors @staticmethod @@ -167,7 +157,7 @@ class TopShouts: await TopShouts.prepare_favorites_shouts() await TopShouts.prepare_shouts_by_rating() await TopShouts.prepare_shouts_by_view() - await TopShouts.prepare_shouts_by_author() + await TopShouts.prepare_top_authors() print("top shouts: update finished") except Exception as err: print("top shouts worker error = %s" % (err)) @@ -192,10 +182,10 @@ async def favorites_shouts(_, info, limit): return TopShouts.favorites_shouts[:limit] -@query.field("topShoutsByAuthor") -async def top_shouts_by_author(_, info, limit): +@query.field("topAuthors") +async def top_authors(_, info, limit): async with TopShouts.lock: - return TopShouts.shouts_by_author[:limit] + return TopShouts.top_authors[:limit] @mutation.field("createShout") diff --git a/schema.graphql b/schema.graphql index a60f9e3a..8371e44a 100644 --- a/schema.graphql +++ b/schema.graphql @@ -101,7 +101,7 @@ type Query { topShoutsByView(limit: Int): [Shout]! topShoutsByRating(limit: Int): [Shout]! favoritesShouts(limit: Int): [Shout]! - topShoutsByAuthor(limit: Int): [Shout]! + topAuthors(limit: Int): [User]! # getOnlineUsers: [User!]! }