diff --git a/main.py b/main.py index edccebc2..b0aa0528 100644 --- a/main.py +++ b/main.py @@ -17,7 +17,7 @@ from resolvers.auth import confirm_email_handler from services.main import storages_init from services.stat.reacted import ReactedStorage from services.stat.topicstat import TopicStat -from services.stat.views import Stat +from services.stat.views import ViewStat from services.zine.gittask import GitTask from services.zine.shoutauthor import ShoutAuthorStorage import_module("resolvers") @@ -31,7 +31,7 @@ middleware = [ async def start_up(): await redis.connect() - views_stat_task = asyncio.create_task(Stat.worker()) + views_stat_task = asyncio.create_task(ViewStat.worker()) print(views_stat_task) reacted_storage_task = asyncio.create_task(ReactedStorage.worker()) print(reacted_storage_task) diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 8d9b64fe..d3b13fff 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -12,7 +12,7 @@ from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.user import User from orm.topic import TopicFollower from services.stat.reacted import ReactedStorage -from services.stat.views import Stat +from services.stat.views import ViewStat OLD_DATE = "2016-03-05 22:22:00.350000" ts = datetime.now() @@ -340,7 +340,7 @@ async def migrate(entry, storage): raise Exception("[migration] content_item.ratings error: \n%r" % content_rating) # shout views - await Stat.increment(shout_dict["slug"], amount=entry.get("views", 1)) + await ViewStat.increment(shout_dict["slug"], amount=entry.get("views", 1)) # del shout_dict['ratings'] shout_dict["oid"] = entry.get("_id") storage["shouts"]["by_oid"][entry["_id"]] = shout_dict diff --git a/resolvers/reactions.py b/resolvers/reactions.py index d7cd0fdd..18d8bcaa 100644 --- a/resolvers/reactions.py +++ b/resolvers/reactions.py @@ -13,7 +13,7 @@ from services.stat.reacted import ReactedStorage async def get_reaction_stat(reaction_id): return { - # "viewed": await Stat.get_reaction(reaction_id), + # "viewed": await ViewStat.get_reaction(reaction_id), "reacted": len(await ReactedStorage.get_reaction(reaction_id)), "rating": await ReactedStorage.get_reaction_rating(reaction_id), "commented": len(await ReactedStorage.get_reaction_comments(reaction_id)), diff --git a/resolvers/topics.py b/resolvers/topics.py index de167e7b..f1c7f1d1 100644 --- a/resolvers/topics.py +++ b/resolvers/topics.py @@ -9,7 +9,7 @@ from orm.topic import Topic, TopicFollower from services.zine.topics import TopicStorage from services.stat.reacted import ReactedStorage from services.stat.topicstat import TopicStat -from services.stat.views import Stat +from services.stat.views import ViewStat async def get_topic_stat(slug): @@ -17,7 +17,7 @@ async def get_topic_stat(slug): "shouts": len(TopicStat.shouts_by_topic.get(slug, {}).keys()), "authors": len(TopicStat.authors_by_topic.get(slug, {}).keys()), "followers": len(TopicStat.followers_by_topic.get(slug, {}).keys()), - "viewed": await Stat.get_topic(slug), + "viewed": await ViewStat.get_topic(slug), "reacted": len(await ReactedStorage.get_topic(slug)), "commented": len(await ReactedStorage.get_topic_comments(slug)), "rating": await ReactedStorage.get_topic_rating(slug) diff --git a/resolvers/zine.py b/resolvers/zine.py index 363ab91e..5f6bd807 100644 --- a/resolvers/zine.py +++ b/resolvers/zine.py @@ -67,7 +67,7 @@ async def load_shouts_by(_, info, by, amount=50, offset=0): q = q.group_by(Shout.id, Reaction.id).order_by( desc(by.get("order") or "createdAt") ).limit(amount).offset(offset) - + print(q) shouts = [] with local_session() as session: # post query stats and author's captions diff --git a/services/main.py b/services/main.py index bfe14663..bcd886f0 100644 --- a/services/main.py +++ b/services/main.py @@ -1,4 +1,3 @@ -from services.stat.views import Stat from services.stat.reacted import ReactedStorage from services.auth.roles import RoleStorage from services.auth.users import UserStorage @@ -10,7 +9,6 @@ from base.orm import local_session async def storages_init(): with local_session() as session: print('[main] initialize storages') - await Stat.update() ReactedStorage.init(session) RoleStorage.init(session) UserStorage.init(session) diff --git a/services/search.py b/services/search.py index 89e6abce..e11a7372 100644 --- a/services/search.py +++ b/services/search.py @@ -9,6 +9,7 @@ class SearchService: @staticmethod async def init(session): async with SearchService.lock: + print('[search.service] init') SearchService.cache = {} @staticmethod diff --git a/services/stat/reacted.py b/services/stat/reacted.py index 5d772343..ade769a0 100644 --- a/services/stat/reacted.py +++ b/services/stat/reacted.py @@ -2,7 +2,7 @@ import asyncio from base.orm import local_session from orm.reaction import ReactionKind, Reaction from services.zine.topics import TopicStorage -from services.stat.views import Stat +from services.stat.views import ViewStat def kind_to_rate(kind) -> int: @@ -36,7 +36,7 @@ class ReactedStorage: @staticmethod async def get_shout_stat(slug): return { - "viewed": await Stat.get_shout(slug), + "viewed": await ViewStat.get_shout(slug), "reacted": len(await ReactedStorage.get_shout(slug)), "commented": len(await ReactedStorage.get_comments(slug)), "rating": await ReactedStorage.get_rating(slug), diff --git a/services/stat/views.py b/services/stat/views.py index 7f999881..6d028483 100644 --- a/services/stat/views.py +++ b/services/stat/views.py @@ -1,7 +1,8 @@ from gql import gql, Client from gql.transport.aiohttp import AIOHTTPTransport import asyncio - +import json +from base.redis import redis from services.zine.topics import TopicStorage query_ackee_views = gql( @@ -33,7 +34,7 @@ query_ackee_views = gql( ) -class Stat: +class ViewStat: lock = asyncio.Lock() by_slugs = {} by_topics = {} @@ -46,9 +47,16 @@ class Stat: # TODO: when the struture of paylod will be transparent # TODO: perhaps ackee token getting here - self = Stat + self = ViewStat async with self.lock: - domains = self.client.execute(query_ackee_views) + self.by_topics = await redis.execute("GET", "views_by_topics") + if self.by_topics: + self.by_topics = json.loads(self.by_topics) + self.by_slugs = await redis.execute("GET", "views_by_shouts") + if self.by_slugs: + self.by_slugs = json.loads(self.by_slugs) + + domains = await self.client.execute_async(query_ackee_views) print("[stat.ackee] loaded domains") print(domains) @@ -56,13 +64,13 @@ class Stat: @staticmethod async def get_shout(shout_slug): - self = Stat + self = ViewStat async with self.lock: return self.by_slugs.get(shout_slug) or 0 @staticmethod async def get_topic(topic_slug): - self = Stat + self = ViewStat async with self.lock: shouts = self.by_topics.get(topic_slug) topic_views = 0 @@ -72,28 +80,39 @@ class Stat: @staticmethod async def increment(shout_slug, amount=1): - self = Stat + self = ViewStat async with self.lock: self.by_slugs[shout_slug] = self.by_slugs.get(shout_slug) or 0 self.by_slugs[shout_slug] += amount + await redis.execute( + "SET", + f"views_by_shouts/{shout_slug}", + str(self.by_slugs[shout_slug]) + ) shout_topics = await TopicStorage.get_topics_by_slugs([shout_slug, ]) for t in shout_topics: self.by_topics[t] = self.by_topics.get(t) or {} self.by_topics[t][shout_slug] = self.by_topics[t].get(shout_slug) or 0 self.by_topics[t][shout_slug] += amount + await redis.execute( + "SET", + f"views_by_topics/{t}/{shout_slug}", + str(self.by_topics[t][shout_slug]) + ) @staticmethod - async def update(): - self = Stat - async with self.lock: - self.load_views() + async def reset(): + self = ViewStat + self.by_topics = {} + self.by_slugs = {} @staticmethod async def worker(): + self = ViewStat while True: try: - await Stat.update() + await self.load_views() except Exception as err: print("[stat.ackee] : %s" % (err)) - print("[stat.ackee] renew period: %d minutes" % (Stat.period / 60)) - await asyncio.sleep(Stat.period) + print("[stat.ackee] renew period: %d minutes" % (ViewStat.period / 60)) + await asyncio.sleep(self.period)