From c3a6ecd3ae20ddaf26f053f2c2e87189a3eb485e Mon Sep 17 00:00:00 2001 From: Untone Date: Tue, 19 Dec 2023 11:25:06 +0300 Subject: [PATCH] precached-authors-fix --- main.py | 6 +++++ resolvers/load.py | 19 +++++++--------- resolvers/search.py | 13 +++++------ services/core.py | 53 ++++++++++++++++++++------------------------- 4 files changed, 43 insertions(+), 48 deletions(-) diff --git a/main.py b/main.py index dac7bc2..8fecc14 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ from sentry_sdk.integrations.ariadne import AriadneIntegration from sentry_sdk.integrations.redis import RedisIntegration from starlette.applications import Starlette +from services.core import get_all_authors from services.rediscache import redis from services.schema import resolvers from settings import DEV_SERVER_PID_FILE_NAME, MODE, SENTRY_DSN @@ -27,6 +28,11 @@ async def start_up(): f.write(str(os.getpid())) else: await redis.connect() + + # load authors cache + authors = await get_all_authors() + print(f"[main] {len(authors)} authors precached") + # startup sentry monitoring services try: import sentry_sdk diff --git a/resolvers/load.py b/resolvers/load.py index 5350560..dda7b17 100644 --- a/resolvers/load.py +++ b/resolvers/load.py @@ -6,7 +6,7 @@ from models.chat import ChatPayload, Message from models.member import ChatMember from resolvers.chats import create_chat from services.auth import login_required -from services.core import get_all_authors, get_my_followed +from services.core import authors_by_id, get_my_followed from services.rediscache import redis from services.schema import query @@ -65,8 +65,6 @@ async def load_chats(_, info, limit: int = 50, offset: int = 0) -> Dict[str, Uni r = await create_chat(None, info, members=[1]) # member with id = 1 is discours print(f"[resolvers.load] created chat: {r['chat_id']}") cids.append(r["chat"]["id"]) - all_authors: List[ChatMember] = await get_all_authors() - authors = {a["id"]: a for a in all_authors} for cid in cids: async with lock: chat_str: str = await redis.execute("GET", f"chats/{cid}") @@ -78,7 +76,7 @@ async def load_chats(_, info, limit: int = 50, offset: int = 0) -> Dict[str, Uni member_ids = c["members"].copy() c["members"] = [] for member_id in member_ids: - a = authors.get(member_id) + a = authors_by_id.get(member_id) if a: a["online"] = a.get("id") in members_online c["members"].append(a) @@ -118,14 +116,13 @@ async def load_recipients(_, _info, limit=50, offset=0): """load possible chat participants""" onliners: List[int] = (await redis.execute("SMEMBERS", "authors-online")) or [] r = [] - all_authors: List[ChatMember] = await get_all_authors(limit, offset) my_followings: List[ChatMember] = await get_my_followed() - if all_authors: - if len(my_followings) < limit: - my_followings = my_followings + list(all_authors)[offset : limit - len(my_followings)] - for a in my_followings: - a["online"] = bool(a["id"] in list(onliners)) - r.append(a) + if len(my_followings) < limit: + my_followings = my_followings + list(authors_by_id.values())[offset : limit - len(my_followings)] + my_followings = list(set(my_followings)) + for a in my_followings: + a["online"] = bool(a["id"] in list(onliners)) + r.append(a) # NOTE: maybe sort members here diff --git a/resolvers/search.py b/resolvers/search.py index 636990c..8258965 100644 --- a/resolvers/search.py +++ b/resolvers/search.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List, Union from resolvers.load import load_messages from services.auth import login_required -from services.core import get_all_authors +from services.core import authors_by_id from services.rediscache import redis from services.schema import query @@ -12,27 +12,24 @@ from services.schema import query @query.field("search_recipients") @login_required async def search_recipients(_, info, text: str, limit: int = 50, offset: int = 0): - result = [] + result = set([]) # TODO: maybe redis scan? author_id = info.context["author_id"] existed_chats = await redis.execute("SMEMBERS", f"/chats_by_author/{author_id}") - authors = await get_all_authors() - members = {a["id"]: a for a in authors} if existed_chats: for chat_id in list(json.loads(existed_chats))[offset : (offset + limit)]: members_ids = await redis.execute("GET", f"/chats/{chat_id}/members") for member_id in members_ids: - author = members.get(member_id) + author = authors_by_id.get(member_id) if author: if author["name"].startswith(text): - if author not in result: - result.append(author) + result.add(author) more_amount = limit - len(result) if more_amount > 0: - result += authors[0:more_amount] + result.update(authors_by_id.values()[0:more_amount]) return {"members": list(result), "error": None} diff --git a/services/core.py b/services/core.py index 854dc07..0b9438b 100644 --- a/services/core.py +++ b/services/core.py @@ -1,15 +1,16 @@ from typing import Any, List -import aiohttp -from aiohttp import ClientResponse, ClientSession +from aiohttp import ClientSession from models.member import ChatMember from settings import API_BASE headers = {"Content-Type": "application/json"} +authors_by_user = {} +authors_by_id = {} async def _request_endpoint(query_name, body) -> Any: - async with aiohttp.ClientSession() as session: + async with ClientSession() as session: async with session.post(API_BASE, headers=headers, json=body) as response: print(f"[services.core] {query_name} response: <{response.status}> {await response.text()}") if response.status == 200: @@ -19,21 +20,27 @@ async def _request_endpoint(query_name, body) -> Any: return [] -async def get_all_authors(limit: int = 50, offset: int = 0) -> List[ChatMember]: - query_name = "load_authors_all" +async def get_all_authors() -> List[ChatMember]: + if len(authors_by_user.keys()) == 0: + query_name = "get_authors_all" - gql = { - "query": "query { " - + query_name - + "(limit: " - + str(limit) - + ", offset: " - + str(offset) - + ") { id slug pic name } }", - "variables": None, - } + # Check if authors are already cached + if "authors" in authors_by_user: + return authors_by_user["authors"] - return await _request_endpoint(query_name, gql) + gql = { + "query": "query { " + query_name + "{ id slug pic name } }", + "variables": None, + } + + # Make a request to load authors + authors = await _request_endpoint(query_name, gql) + + for a in authors: + authors_by_user[a.user] = a + authors_by_id[a.id] = a + + return authors_by_id.values() async def get_my_followed() -> List[ChatMember]: @@ -48,16 +55,4 @@ async def get_my_followed() -> List[ChatMember]: async def get_author(user: str = ""): - query_name = "get_author_id" - operation = "GetAuthorId" - - gql = { - "query": f"query {operation}($user: String!) {{ {query_name}(user: $user) {{ id slug pic name }} }}", - "operationName": operation, - "variables": {"user": user}, - } - - r = await _request_endpoint(query_name, gql) - if r and isinstance(r, list): - r = r.pop() - return r or None + return authors_by_user.get(user)