separate-stat-query

CHANGELOG.txt

@@ -2,6 +2,7 @@
 - feat: sentry integration enabled with glitchtip
 - fix: reindex on update shout
 - packages upgrade, isort
+- separated stats queries for author and topic
 
 [0.3.2]
 - redis cache for what author follows

resolvers/__init__.py

@@ -1,24 +1,52 @@
-from resolvers.author import (get_author, get_author_followers,
-                              get_author_follows, get_author_follows_authors,
-                              get_author_follows_topics, get_author_id,
-                              get_authors_all, load_authors_by, search_authors,
-                              update_author)
+from resolvers.author import (
+    get_author,
+    get_author_followers,
+    get_author_follows,
+    get_author_follows_authors,
+    get_author_follows_topics,
+    get_author_id,
+    get_authors_all,
+    load_authors_by,
+    search_authors,
+    update_author,
+)
 from resolvers.community import get_communities_all, get_community
 from resolvers.editor import create_shout, delete_shout, update_shout
-from resolvers.follower import (follow, get_shout_followers,
-                                get_topic_followers, unfollow)
-from resolvers.notifier import (load_notifications, notification_mark_seen,
-                                notifications_seen_after,
-                                notifications_seen_thread)
+from resolvers.follower import (
+    follow,
+    get_shout_followers,
+    get_topic_followers,
+    unfollow,
+)
+from resolvers.notifier import (
+    load_notifications,
+    notification_mark_seen,
+    notifications_seen_after,
+    notifications_seen_thread,
+)
 from resolvers.rating import rate_author
-from resolvers.reaction import (create_reaction, delete_reaction,
-                                load_reactions_by, load_shouts_followed,
-                                update_reaction)
-from resolvers.reader import (get_shout, load_shouts_by, load_shouts_feed,
-                              load_shouts_random_top, load_shouts_random_topic,
-                              load_shouts_search, load_shouts_unrated)
-from resolvers.topic import (get_topic, get_topics_all, get_topics_by_author,
-                             get_topics_by_community)
+from resolvers.reaction import (
+    create_reaction,
+    delete_reaction,
+    load_reactions_by,
+    load_shouts_followed,
+    update_reaction,
+)
+from resolvers.reader import (
+    get_shout,
+    load_shouts_by,
+    load_shouts_feed,
+    load_shouts_random_top,
+    load_shouts_random_topic,
+    load_shouts_search,
+    load_shouts_unrated,
+)
+from resolvers.topic import (
+    get_topic,
+    get_topics_all,
+    get_topics_by_author,
+    get_topics_by_community,
+)
 from services.triggers import events_register
 
 events_register()

resolvers/author.py

@@ -8,8 +8,7 @@ from sqlalchemy_searchable import search
 from orm.author import Author, AuthorFollower
 from orm.shout import ShoutAuthor, ShoutTopic
 from orm.topic import Topic
-from resolvers.stat import (author_follows_authors, author_follows_topics,
-                            get_with_stat)
+from resolvers.stat import author_follows_authors, author_follows_topics, get_with_stat
 from services.auth import login_required
 from services.cache import cache_author, cache_follower
 from services.db import local_session

resolvers/collab.py

@@ -139,10 +139,9 @@ async def remove_invite(_, info, invite_id: int):
 
     author_dict = info.context["author"]
     author_id = author_dict.get("id")
-    if author_id:
+    if isinstance(author_id, int):
         # Check if the user exists
         with local_session() as session:
-            author_id == int(author_id)
             # Check if the invite exists
             invite = session.query(Invite).filter(Invite.id == invite_id).first()
             if isinstance(invite, Invite):

resolvers/follower.py

@@ -11,8 +11,7 @@ from orm.community import Community
 from orm.reaction import Reaction
 from orm.shout import Shout, ShoutReactionsFollower
 from orm.topic import Topic, TopicFollower
-from resolvers.stat import (author_follows_authors, author_follows_topics,
-                            get_with_stat)
+from resolvers.stat import author_follows_authors, author_follows_topics, get_with_stat
 from services.auth import login_required
 from services.cache import DEFAULT_FOLLOWS, cache_follower
 from services.db import local_session

resolvers/notifier.py

@@ -8,8 +8,12 @@ from sqlalchemy.orm import aliased
 from sqlalchemy.sql import not_
 
 from orm.author import Author
-from orm.notification import (Notification, NotificationAction,
-                              NotificationEntity, NotificationSeen)
+from orm.notification import (
+    Notification,
+    NotificationAction,
+    NotificationEntity,
+    NotificationSeen,
+)
 from orm.shout import Shout
 from services.auth import login_required
 from services.db import local_session
@@ -142,10 +146,10 @@ def get_notifications_grouped(
 
         elif str(notification.entity) == NotificationEntity.REACTION.value:
             reaction = payload
-            if not isinstance(shout, dict):
+            if not isinstance(reaction, dict):
                 raise ValueError("reaction data is not consistent")
-            shout_id = shout.get("shout")
-            author_id = shout.get("created_by", 0)
+            shout_id = reaction.get("shout")
+            author_id = reaction.get("created_by", 0)
             if shout_id and author_id:
                 with local_session() as session:
                     author = (

resolvers/reaction.py

@@ -6,8 +6,7 @@ from sqlalchemy.orm import aliased, joinedload
 from sqlalchemy.sql import union
 
 from orm.author import Author
-from orm.rating import (PROPOSAL_REACTIONS, RATING_REACTIONS, is_negative,
-                        is_positive)
+from orm.rating import PROPOSAL_REACTIONS, RATING_REACTIONS, is_negative, is_positive
 from orm.reaction import Reaction, ReactionKind
 from orm.shout import Shout
 from resolvers.editor import handle_proposing

resolvers/reader.py

@@ -1,7 +1,6 @@
 from sqlalchemy import bindparam, distinct, or_, text
 from sqlalchemy.orm import aliased, joinedload
-from sqlalchemy.sql.expression import (and_, asc, case, desc, func, nulls_last,
-                                       select)
+from sqlalchemy.sql.expression import and_, asc, case, desc, func, nulls_last, select
 
 from orm.author import Author, AuthorFollower
 from orm.reaction import Reaction, ReactionKind

resolvers/stat.py

@@ -10,39 +10,51 @@ from services.db import local_session
 from services.logger import root_logger as logger
 
 
-def add_topic_stat_columns(q):
-    aliased_shout_topic = aliased(ShoutTopic)
-    aliased_authors = aliased(ShoutAuthor)
-    aliased_followers = aliased(TopicFollower)
-    aliased_shout = aliased(Shout)
-
-    # shouts
-    q = q.outerjoin(aliased_shout_topic, aliased_shout_topic.topic == Topic.id)
-    q = q.add_columns(
-        func.count(distinct(aliased_shout_topic.shout)).label("shouts_stat")
+def get_topic_shouts_stat(topic_id: int):
+    q = (
+        select(func.count(distinct(ShoutTopic.shout)))
+        .select_from(join(ShoutTopic, Shout, ShoutTopic.shout == Shout.id))
+        .filter(
+            and_(
+                ShoutTopic.topic == topic_id,
+                Shout.published_at.is_not(None),
+                Shout.deleted_at.is_(None),
+            )
+        )
     )
+    [shouts_stat] = local_session().execute(q)
+    return shouts_stat or 0
 
+
+def get_topic_authors_stat(topic_id: int):
     # authors
-    q = q.outerjoin(
-        aliased_shout,
-        and_(
-            aliased_shout.id == aliased_shout_topic.shout,
-            aliased_shout.published_at.is_not(None),
-            aliased_shout.deleted_at.is_(None),
-        ),
-    )
-    q = q.outerjoin(aliased_authors, aliased_shout.authors.any(id=aliased_authors.id))
-    q = q.add_columns(
-        func.count(distinct(aliased_authors.author)).label("authors_stat")
+    q = (
+        select(func.count(distinct(ShoutAuthor.author)))
+        .select_from(join(ShoutTopic, Shout, ShoutTopic.shout == Shout.id))
+        .join(ShoutAuthor, ShoutAuthor.shout == Shout.id)
+        .filter(
+            and_(
+                ShoutTopic.topic == topic_id,
+                Shout.published_at.is_not(None),
+                Shout.deleted_at.is_(None),
+            )
+        )
     )
+    [authors_stat] = local_session().execute(q)
+    return authors_stat or 0
 
-    # followers
-    q = q.outerjoin(aliased_followers, aliased_followers.topic == Topic.id)
-    q = q.add_columns(
-        func.count(distinct(aliased_followers.follower)).label("followers_stat")
-    )
 
-    # comments
+def get_topic_followers_stat(topic_id: int):
+    aliased_followers = aliased(TopicFollower)
+    q = select(func.count(distinct(aliased_followers.follower))).filter(
+        aliased_followers.topic == topic_id
+    )
+    with local_session() as session:
+        [followers_stat] = session.execute(q)
+    return followers_stat or 0
+
+
+def get_topic_comments_stat(topic_id: int):
     sub_comments = (
         select(
             Shout.id.label("shout_id"),
@@ -61,39 +73,48 @@ def add_topic_stat_columns(q):
         .group_by(Shout.id)
         .subquery()
     )
-    q = q.outerjoin(sub_comments, aliased_shout_topic.shout == sub_comments.c.shout_id)
-    q = q.add_columns(
-        func.coalesce(sub_comments.c.comments_count, 0).label("comments_stat")
+    q = select(func.coalesce(func.sum(sub_comments.c.comments_count), 0)).filter(
+        ShoutTopic.topic == topic_id
     )
-
-    group_list = [Topic.id, sub_comments.c.comments_count]
-
-    q = q.group_by(*group_list)
-    logger.debug(q)
-    return q
+    q = q.outerjoin(sub_comments, ShoutTopic.shout == sub_comments.c.shout_id)
+    [comments_stat] = local_session().execute(q)
+    return comments_stat or 0
 
 
-def add_author_stat_columns(q):
+def get_author_shouts_stat(author_id: int):
     aliased_shout_author = aliased(ShoutAuthor)
+    q = select(func.count(distinct(aliased_shout_author.shout))).filter(
+        aliased_shout_author.author == author_id
+    )
+    with local_session() as session:
+        [shouts_stat] = session.execute(q)
+    return shouts_stat or 0
+
+
+def get_author_authors_stat(author_id: int):
     aliased_authors = aliased(AuthorFollower)
+    q = select(func.count(distinct(aliased_authors.author))).filter(
+        and_(
+            aliased_authors.follower == author_id,
+            aliased_authors.author != author_id,
+        )
+    )
+    with local_session() as session:
+        [authors_stat] = session.execute(q)
+    return authors_stat or 0
+
+
+def get_author_followers_stat(author_id: int):
     aliased_followers = aliased(AuthorFollower)
-
-    q = q.outerjoin(aliased_shout_author, aliased_shout_author.author == Author.id)
-    q = q.add_columns(
-        func.count(distinct(aliased_shout_author.shout)).label("shouts_stat")
+    q = select(func.count(distinct(aliased_followers.follower))).filter(
+        aliased_followers.author == author_id
     )
+    with local_session() as session:
+        [followers_stat] = session.execute(q)
+    return followers_stat or 0
 
-    q = q.outerjoin(aliased_authors, aliased_authors.follower == Author.id)
-    q = q.add_columns(
-        func.count(distinct(aliased_authors.author)).label("authors_stat")
-    )
 
-    q = q.outerjoin(aliased_followers, aliased_followers.author == Author.id)
-    q = q.add_columns(
-        func.count(distinct(aliased_followers.follower)).label("followers_stat")
-    )
-
-    # Create a subquery for comments count
+def get_author_comments_stat(author_id: int):
     sub_comments = (
         select(
             Author.id, func.coalesce(func.count(Reaction.id)).label("comments_count")
@@ -109,37 +130,43 @@ def add_author_stat_columns(q):
         .group_by(Author.id)
         .subquery()
     )
-
-    q = q.outerjoin(sub_comments, Author.id == sub_comments.c.id)
-    q = q.add_columns(sub_comments.c.comments_count)
-    group_list = [Author.id, sub_comments.c.comments_count]
-
-    q = q.group_by(*group_list)
-
-    return q
+    q = select(sub_comments.c.comments_count).filter(sub_comments.c.id == author_id)
+    with local_session() as session:
+        [comments_stat] = session.execute(q)
+    return comments_stat or 0
 
 
 def get_with_stat(q):
     records = []
     try:
         is_author = f"{q}".lower().startswith("select author")
-        is_topic = f"{q}".lower().startswith("select topic")
-        if is_author:
-            q = add_author_stat_columns(q)
-        elif is_topic:
-            q = add_topic_stat_columns(q)
+        f"{q}".lower().startswith("select topic")
+        result = []
         with local_session() as session:
             result = session.execute(q)
-            for cols in result:
-                entity = cols[0]
-                stat = dict()
-                stat["shouts"] = cols[1]
-                stat["authors"] = cols[2]
-                stat["followers"] = cols[3]
-                if is_author:
-                    stat["comments"] = cols[4]
-                entity.stat = stat
-                records.append(entity)
+
+        for cols in result:
+            entity = cols[0]
+            stat = dict()
+            stat["shouts"] = (
+                get_author_shouts_stat(entity.id)
+                if is_author
+                else get_topic_shouts_stat(entity.id)
+            )
+            stat["authors"] = (
+                get_author_authors_stat(entity.id)
+                if is_author
+                else get_topic_authors_stat(entity.id)
+            )
+            stat["followers"] = (
+                get_author_followers_stat(entity.id)
+                if is_author
+                else get_topic_followers_stat(entity.id)
+            )
+            if is_author:
+                stat["comments"] = get_author_comments_stat(entity.id)
+            entity.stat = stat
+            records.append(entity)
     except Exception as exc:
         import traceback
 

services/auth.py

@@ -35,6 +35,7 @@ async def request_data(gql, headers=None):
     except Exception as e:
         # Handling and logging exceptions during authentication check
         import traceback
+
         logger.error(f"request_data error: {e}")
         logger.error(traceback.format_exc())
     return None

services/db.py

@@ -5,8 +5,7 @@ import traceback
 import warnings
 from typing import Any, Callable, Dict, TypeVar
 
-from sqlalchemy import (JSON, Column, Engine, Integer, create_engine, event,
-                        exc, inspect)
+from sqlalchemy import JSON, Column, Engine, Integer, create_engine, event, exc, inspect
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import Session, configure_mappers
 from sqlalchemy.sql.schema import Table

services/viewed.py

@@ -7,8 +7,12 @@ from typing import Dict
 
 # ga
 from google.analytics.data_v1beta import BetaAnalyticsDataClient
-from google.analytics.data_v1beta.types import (DateRange, Dimension, Metric,
-                                                RunReportRequest)
+from google.analytics.data_v1beta.types import (
+    DateRange,
+    Dimension,
+    Metric,
+    RunReportRequest,
+)
 
 from orm.author import Author
 from orm.shout import Shout, ShoutAuthor, ShoutTopic