refactored-cache-following

resolvers/__init__.py

@@ -19,7 +19,7 @@ from resolvers.reader import (get_shout, load_shouts_by, load_shouts_feed,
                               load_shouts_search, load_shouts_unrated)
 from resolvers.topic import (get_topic, get_topics_all, get_topics_by_author,
                              get_topics_by_community)
-from services.cache import events_register
+from services.triggers import events_register
 
 events_register()
 

resolvers/author.py

@@ -1,4 +1,3 @@
-import asyncio
 import json
 import time
 
@@ -9,10 +8,9 @@ from sqlalchemy_searchable import search
 from orm.author import Author, AuthorFollower
 from orm.shout import ShoutAuthor, ShoutTopic
 from orm.topic import Topic
-from resolvers.stat import (author_follows_authors, author_follows_topics,
-                            get_authors_with_stat_cached, get_with_stat)
+from resolvers.stat import author_follows_authors, author_follows_topics, get_with_stat
 from services.auth import login_required
-from services.cache import set_author_cache, update_author_followers_cache
+from services.cache import cache_author, cache_follower
 from services.db import local_session
 from services.encoders import CustomJSONEncoder
 from services.logger import root_logger as logger
@@ -55,22 +53,25 @@ async def get_author(_, _info, slug='', author_id=0):
     author = None
     author_dict = None
     try:
-        author_query = select(Author).filter(
-            or_(Author.slug == slug, Author.id == author_id)
-        )
-        result = await get_authors_with_stat_cached(author_query)
-        if not result:
-            raise ValueError('Author not found')
-        [author] = result
-        author_id = author.id
-        logger.debug(f'found @{slug} with id {author_id}')
-        if isinstance(author, Author):
-            if not author.stat:
-                [author] = get_with_stat(author_query)  # FIXME: with_rating=True)
-                if author:
-                    await set_author_cache(author.dict())
-                    logger.debug('updated author stored in cache')
-            author_dict = author.dict()
+        # lookup for cached author
+        author_query = select(Author).filter(or_(Author.slug == slug, Author.id == author_id))
+        found_author = local_session().execute(author_query).first()
+        logger.debug(f'found author id: {found_author.id}')
+        author_id = found_author.id if not found_author.id else author_id
+        cached_result = await redis.execute('GET', f'author:{author_id}')
+        author_dict = json.loads(cached_result) if cached_result else None
+
+        # update stat from db
+        if not author_dict or not author_dict.get('stat'):
+            result = get_with_stat(author_query)
+            if not result:
+                raise ValueError('Author not found')
+            [author] = result
+            # use found author
+            if isinstance(author, Author):
+                logger.debug(f'update @{author.slug} with id {author.id}')
+                author_dict = author.dict()
+                await cache_author(author_dict)
     except ValueError:
         pass
     except Exception:
@@ -95,11 +96,11 @@ async def get_author_by_user_id(user_id: str):
                 logger.debug(f'got author @{author_slug} #{author_id} cached')
                 return author
 
-        q = select(Author).filter(Author.user == user_id)
-        result = await get_authors_with_stat_cached(q)
+        author_query = select(Author).filter(Author.user == user_id)
+        result = get_with_stat(author_query)
         if result:
             [author] = result
-            await set_author_cache(author.dict())
+            await cache_author(author.dict())
     except Exception as exc:
         import traceback
 
@@ -286,36 +287,32 @@ def create_author(user_id: str, slug: str, name: str = ''):
 async def get_author_followers(_, _info, slug: str):
     logger.debug(f'getting followers for @{slug}')
     try:
-        with local_session() as session:
-            author_alias = aliased(Author)
-            result = (
-                session.query(author_alias).filter(author_alias.slug == slug).first()
-            )
-            if result:
-                [author] = result
-                author_id = author.id
-                cached = await redis.execute('GET', f'author:{author_id}:followers')
-                if not cached:
-                    author_follower_alias = aliased(AuthorFollower, name='af')
-                    q = select(Author).join(
-                        author_follower_alias,
-                        and_(
-                            author_follower_alias.author == author_id,
-                            author_follower_alias.follower == Author.id,
-                        ),
-                    )
-                    results = await get_authors_with_stat_cached(q)
-                    _ = asyncio.create_task(
-                        update_author_followers_cache(
-                            author_id, [x.dict() for x in results]
-                        )
-                    )
+        author_alias = aliased(Author)
+        author_query = select(author_alias).filter(author_alias.slug == slug)
+        result = local_session().execute(author_query).first()
+        if result:
+            [author] = result
+            author_id = author.id
+            cached = await redis.execute('GET', f'author:{author_id}:followers')
+            if not cached:
+                author_follower_alias = aliased(AuthorFollower, name='af')
+                q = select(Author).join(
+                    author_follower_alias,
+                    and_(
+                        author_follower_alias.author == author_id,
+                        author_follower_alias.follower == Author.id,
+                    ),
+                )
+                results = get_with_stat(q)
+                if isinstance(results, list):
+                    for follower in results:
+                        await cache_follower(follower, author)
                     logger.debug(f'@{slug} cache updated with {len(results)} followers')
-                    return results
-                else:
-                    logger.debug(f'@{slug} got followers cached')
-                    if isinstance(cached, str):
-                        return json.loads(cached)
+                return results
+            else:
+                logger.debug(f'@{slug} got followers cached')
+                if isinstance(cached, str):
+                    return json.loads(cached)
     except Exception as exc:
         import traceback
 
@@ -327,4 +324,4 @@ async def get_author_followers(_, _info, slug: str):
 @query.field('search_authors')
 async def search_authors(_, _info, what: str):
     q = search(select(Author), what)
-    return await get_authors_with_stat_cached(q)
+    return get_with_stat(q)

resolvers/follower.py

@@ -8,16 +8,12 @@ from sqlalchemy.sql import and_
 
 from orm.author import Author, AuthorFollower
 from orm.community import Community
-# from orm.community import Community
 from orm.reaction import Reaction
 from orm.shout import Shout, ShoutReactionsFollower
 from orm.topic import Topic, TopicFollower
-from resolvers.stat import (author_follows_authors, author_follows_topics,
-                            get_authors_with_stat_cached,
-                            get_topics_with_stat_cached)
+from resolvers.stat import author_follows_authors, author_follows_topics, get_with_stat
 from services.auth import login_required
-from services.cache import (DEFAULT_FOLLOWS, update_followers_for_author,
-                            update_follows_for_author)
+from services.cache import DEFAULT_FOLLOWS
 from services.db import local_session
 from services.logger import root_logger as logger
 from services.notify import notify_follower
@@ -33,53 +29,37 @@ async def follow(_, info, what, slug):
     user_id = info.context.get('user_id')
     if not user_id:
         return {'error': 'unauthorized'}
-    [follower] = await get_authors_with_stat_cached(
-        select(Author).select_from(Author).filter(Author.user == user_id)
-    )
+
+    follower_query = select(Author).select_from(Author).filter(Author.user == user_id)
+    [follower] = local_session().execute(follower_query)
     if not follower:
         return {'error': 'cant find follower'}
 
     if what == 'AUTHOR':
         error = author_follow(follower.id, slug)
         if not error:
-            logger.debug(f'@{follower.slug} followed @{slug}')
-            [author] = await get_authors_with_stat_cached(
-                select(Author).select_from(Author).where(Author.slug == slug)
-            )
-            if not author:
-                return {'error': 'author is not found'}
-            follows = await update_follows_for_author(
-                follower, 'author', author.dict(), True
-            )
-            _followers = await update_followers_for_author(follower, author, True)
-            await notify_follower(follower.dict(), author.id, 'unfollow')
+            author_query = select(Author).where(Author.slug == slug)
+            [author] = local_session().execute(author_query)
+            await notify_follower(follower.dict(), author.id, 'follow')
 
     elif what == 'TOPIC':
         error = topic_follow(follower.id, slug)
-        if not error:
-            [topic] = await get_topics_with_stat_cached(
-                select(Topic).where(Topic.slug == slug)
-            )
-            if not topic:
-                return {'error': 'topic is not found'}
-            follows = await update_follows_for_author(
-                follower, 'topic', topic.dict(), True
-            )
 
     elif what == 'COMMUNITY':
+        # FIXME: when more communities
         follows = local_session().execute(select(Community))
 
     elif what == 'SHOUT':
         error = reactions_follow(follower.id, slug)
-        if not error:
-            [shout] = local_session().execute(select(Shout).where(Shout.slug == slug))
-            if not shout:
-                return {'error': 'cant find shout'}
-            follows = await update_follows_for_author(
-                follower, 'shout', shout.dict(), True
-            )
 
-    return {f'{what.lower()}s': follows, 'error': error}
+    if error:
+        return {'error': error}
+
+    entity = what.lower()
+    follows_str = await redis.execute('GET', f'author:{follower.id}:follows-{entity}s')
+    if follows_str:
+        follows = json.loads(follows_str)
+    return { f'{entity}s': follows }
 
 
 @mutation.field('unfollow')
@@ -91,54 +71,33 @@ async def unfollow(_, info, what, slug):
     if not user_id:
         return {'error': 'unauthorized'}
     follower_query = select(Author).filter(Author.user == user_id)
-    [follower] = await get_authors_with_stat_cached(follower_query)
+    [follower] = local_session().execute(follower_query)
     if not follower:
         return {'error': 'follower profile is not found'}
 
     if what == 'AUTHOR':
         error = author_unfollow(follower.id, slug)
+        # NOTE: after triggers should update cached stats
         if not error:
-            logger.info(f'@{follower.slug} unfollowing @{slug}')
-            [author] = await get_authors_with_stat_cached(
-                select(Author).where(Author.slug == slug)
-            )
-            if not author:
-                return {'error': 'cant find author'}
-            _followers = await update_followers_for_author(follower, author, False)
+            logger.info(f'@{follower.slug} unfollowed @{slug}')
+            author_query = select(Author).where(Author.slug == slug)
+            [author] = local_session().execute(author_query)
             await notify_follower(follower.dict(), author.id, 'unfollow')
-            follows = await update_follows_for_author(
-                follower, 'author', author.dict(), False
-            )
 
     elif what == 'TOPIC':
         error = topic_unfollow(follower.id, slug)
-        if not error:
-            logger.info(f'@{follower.slug} unfollowing §{slug}')
-            [topic] = await get_topics_with_stat_cached(
-                select(Topic).where(Topic.slug == slug)
-            )
-            if not topic:
-                return {'error': 'cant find topic'}
-            follows = await update_follows_for_author(
-                follower, 'topic', topic.dict(), False
-            )
 
     elif what == 'COMMUNITY':
         follows = local_session().execute(select(Community))
 
     elif what == 'SHOUT':
         error = reactions_unfollow(follower.id, slug)
-        if not error:
-            logger.info(f'@{follower.slug} unfollowing §{slug}')
-            [shout] = local_session().execute(select(Shout).where(Shout.slug == slug))
-            if not shout:
-                return {'error': 'cant find shout'}
-            if not error:
-                follows = await update_follows_for_author(
-                    follower, 'shout', shout.dict(), False
-                )
 
-    return {'error': error, f'{what.lower()}s': follows}
+    entity = what.lower()
+    follows_str = await redis.execute('GET', f'author:{follower.id}:follows-{entity}s')
+    if follows_str:
+        follows = json.loads(follows_str)
+    return {'error': error, f'{entity}s': follows}
 
 
 async def get_follows_by_user_id(user_id: str):
@@ -321,7 +280,7 @@ async def get_topic_followers(_, _info, slug: str, topic_id: int) -> List[Author
         .join(Topic, Topic.id == TopicFollower.topic)
         .filter(or_(Topic.slug == slug, Topic.id == topic_id))
     )
-    return await get_authors_with_stat_cached(q)
+    return get_with_stat(q)
 
 
 @query.field('get_shout_followers')

resolvers/stat.py

@@ -1,5 +1,3 @@
-import json
-
 from sqlalchemy import and_, distinct, func, join, select
 from sqlalchemy.orm import aliased
 
@@ -7,10 +5,7 @@ from orm.author import Author, AuthorFollower
 from orm.reaction import Reaction, ReactionKind
 from orm.shout import Shout, ShoutAuthor, ShoutTopic
 from orm.topic import Topic, TopicFollower
-from resolvers.rating import add_author_rating_columns
 from services.db import local_session
-from services.logger import root_logger as logger
-from services.rediscache import redis
 
 
 def add_topic_stat_columns(q):
@@ -65,7 +60,7 @@ def add_topic_stat_columns(q):
     return q
 
 
-def add_author_stat_columns(q, with_rating=False):
+def add_author_stat_columns(q):
     aliased_shout_author = aliased(ShoutAuthor)
     aliased_authors = aliased(AuthorFollower)
     aliased_followers = aliased(AuthorFollower)
@@ -106,20 +101,17 @@ def add_author_stat_columns(q, with_rating=False):
     q = q.add_columns(sub_comments.c.comments_count)
     group_list = [Author.id, sub_comments.c.comments_count]
 
-    if with_rating:
-        q, group_list = add_author_rating_columns(q, group_list)
-
     q = q.group_by(*group_list)
 
     return q
 
 
-def get_with_stat(q, with_rating=False):
+def get_with_stat(q):
     try:
         is_author = f'{q}'.lower().startswith('select author')
         is_topic = f'{q}'.lower().startswith('select topic')
         if is_author:
-            q = add_author_stat_columns(q, with_rating)
+            q = add_author_stat_columns(q)
         elif is_topic:
             q = add_topic_stat_columns(q)
         records = []
@@ -133,11 +125,6 @@ def get_with_stat(q, with_rating=False):
                 stat['followers'] = cols[3]
                 if is_author:
                     stat['comments'] = cols[4]
-                    if with_rating:
-                        logger.debug(cols)
-                        stat['rating'] = cols[6]
-                        stat['rating_shouts'] = cols[7]
-                        stat['rating_comments'] = cols[8]
                 entity.stat = stat
                 records.append(entity)
     except Exception as exc:
@@ -148,41 +135,6 @@ def get_with_stat(q, with_rating=False):
     return records
 
 
-async def get_authors_with_stat_cached(q):
-    # logger.debug(q)
-    try:
-        records = []
-        with local_session() as session:
-            for [x] in session.execute(q):
-                stat_str = await redis.execute('GET', f'author:{x.id}')
-                x.stat = (
-                    json.loads(stat_str).get('stat')
-                    if isinstance(stat_str, str)
-                    else {}
-                )
-                records.append(x)
-    except Exception as exc:
-        raise Exception(exc)
-    return records
-
-
-async def get_topics_with_stat_cached(q):
-    try:
-        records = []
-        current = None
-        with local_session() as session:
-            for [x] in session.execute(q):
-                current = x
-                stat_str = await redis.execute('GET', f'topic:{x.id}')
-                if isinstance(stat_str, str):
-                    x.stat = json.loads(stat_str).get('stat')
-                records.append(x)
-    except Exception as exc:
-        logger.error(current)
-        raise Exception(exc)
-    return records
-
-
 def author_follows_authors(author_id: int):
     af = aliased(AuthorFollower, name='af')
     q = (