From e6f2ed6f53dc8c7479728ff28961dd56ca8395ed Mon Sep 17 00:00:00 2001 From: Untone Date: Fri, 26 Jan 2024 03:28:21 +0300 Subject: [PATCH 1/9] 0.2.22-granian+precommit --- CHANGELOG.txt | 4 ++ Dockerfile | 2 +- pyproject.toml | 106 ++++++++++++++++++++++++++----------------------- server.py | 59 --------------------------- 4 files changed, 62 insertions(+), 109 deletions(-) delete mode 100644 server.py diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 7075c4c..7bced8c 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,7 @@ +[0.2.22] +- precommit installed +- granian asgi added + [0.2.19] - versioning sync with core, inbox, presence - fix: auth connection user_id trimming diff --git a/Dockerfile b/Dockerfile index 9ce2d47..6401033 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,4 +19,4 @@ RUN apt-get update && apt-get install -y gcc curl && \ poetry install --no-dev # Run server.py when the container launches -CMD ["python", "server.py"] \ No newline at end of file +CMD granian --no-ws --host 0.0.0.0 --port 80 --interface asgi main:app diff --git a/pyproject.toml b/pyproject.toml index 0a9a21b..56ffb23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "discoursio-notifier" -version = "0.2.19" +version = "0.2.22" description = "notifier server for discours.io" authors = ["discours.io devteam"] @@ -13,53 +13,75 @@ python = "^3.12" SQLAlchemy = "^2.0.22" psycopg2-binary = "^2.9.9" redis = {extras = ["hiredis"], version = "^5.0.1"} -uvicorn = "^0.24.0" -strawberry-graphql = {extras = ["asgi", "debug-server"], version = "^0.216.1" } -strawberry-sqlalchemy-mapper = "^0.4.0" -sentry-sdk = "^1.37.1" +strawberry-graphql = {extras = ["asgi", "debug-server"], version = "^0.219.0" } +strawberry-sqlalchemy-mapper = "^0.4.2" +sentry-sdk = "^1.39.2" aiohttp = "^3.9.1" +pre-commit = "^3.6.0" +granian = "^1.0.1" [tool.poetry.group.dev.dependencies] setuptools = "^69.0.2" +pyright = "^1.1.341" pytest = "^7.4.2" black = { version = "^23.12.0", python = ">=3.12" } ruff = { version = "^0.1.8", python = ">=3.12" } -mypy = { version = "^1.7", python = ">=3.12" } isort = "^5.13.2" -pyright = "^1.1.341" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.ruff] +line-length = 120 +extend-select = [ + # E and F are enabled by default + 'B', # flake8-bugbear + 'C4', # flake8-comprehensions + 'C90', # mccabe + 'I', # isort + 'N', # pep8-naming + 'Q', # flake8-quotes + 'S', # flake8-bandit + 'W', # pycodestyle +] +extend-ignore = [ + 'B008', # function calls in args defaults are fine + 'B009', # getattr with constants is fine + 'B034', # re.split won't confuse us + 'B904', # rising without from is fine + 'E501', # leave line length to black + 'N818', # leave to us exceptions naming + 'S101', # assert is fine + 'E712', # allow == True +] +flake8-quotes = { inline-quotes = 'single', multiline-quotes = 'double' } +mccabe = { max-complexity = 13 } +target-version = "py312" + +[tool.ruff.format] +quote-style = 'single' [tool.black] -line-length = 120 -target-version = ['py312'] -include = '\.pyi?$' -exclude = ''' +skip-string-normalization = true -( - /( - \.eggs # exclude a few common directories in the - | \.git # root of the project - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - )/ - | foo.py # also separately exclude a file named foo.py in - # the root of the project -) -''' +[tool.ruff.isort] +combine-as-imports = true +lines-after-imports = 2 +known-first-party = ['resolvers', 'services', 'orm', 'tests'] -[tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -line_length = 120 +[tool.ruff.per-file-ignores] +'tests/**' = ['B018', 'S110', 'S501'] +[tool.mypy] +python_version = "3.12" +warn_return_any = true +warn_unused_configs = true +ignore_missing_imports = true +exclude = ["nb"] + +[tool.pytest.ini_options] +asyncio_mode = 'auto' [tool.pyright] venvPath = "." @@ -87,17 +109,3 @@ logLevel = "Information" pluginSearchPaths = [] typings = {} mergeTypeStubPackages = false - -[tool.mypy] -python_version = "3.12" -warn_unused_configs = true -plugins = ["mypy_sqlalchemy.plugin", "strawberry.ext.mypy_plugin"] - -[tool.ruff] -# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. -# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or -# McCabe complexity (`C901`) by default. -select = ["E4", "E7", "E9", "F"] -ignore = [] -line-length = 120 -target-version = "py312" diff --git a/server.py b/server.py deleted file mode 100644 index 45a093b..0000000 --- a/server.py +++ /dev/null @@ -1,59 +0,0 @@ -import sys - -import uvicorn -from uvicorn.main import logger - -from settings import PORT - -log_settings = { - "version": 1, - "disable_existing_loggers": True, - "formatters": { - "default": { - "()": "uvicorn.logging.DefaultFormatter", - "fmt": "%(levelprefix)s %(message)s", - "use_colors": None, - }, - "access": { - "()": "uvicorn.logging.AccessFormatter", - "fmt": '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s', - }, - }, - "handlers": { - "default": { - "formatter": "default", - "class": "logging.StreamHandler", - "stream": "ext://sys.stderr", - }, - "access": { - "formatter": "access", - "class": "logging.StreamHandler", - "stream": "ext://sys.stdout", - }, - }, - "loggers": { - "uvicorn": {"handlers": ["default"], "level": "INFO"}, - "uvicorn.error": {"level": "INFO", "handlers": ["default"], "propagate": True}, - "uvicorn.access": {"handlers": ["access"], "level": "INFO", "propagate": False}, - }, -} - -local_headers = [ - ("Access-Control-Allow-Methods", "GET, POST, OPTIONS, HEAD"), - ("Access-Control-Allow-Origin", "https://localhost:3000"), - ( - "Access-Control-Allow-Headers", - "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization", - ), - ("Access-Control-Expose-Headers", "Content-Length,Content-Range"), - ("Access-Control-Allow-Credentials", "true"), -] - - -def exception_handler(_et, exc, _tb): - logger.error(..., exc_info=(type(exc), exc, exc.__traceback__)) - - -if __name__ == "__main__": - sys.excepthook = exception_handler - uvicorn.run("main:app", host="0.0.0.0", port=PORT, proxy_headers=True, server_header=True) From 7beddea5b165e8c30f6f0b8afc118d771db67528 Mon Sep 17 00:00:00 2001 From: Untone Date: Fri, 26 Jan 2024 03:29:35 +0300 Subject: [PATCH 2/9] ymlfix --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 56ffb23..1af9476 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,3 @@ -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" - [tool.poetry] name = "discoursio-notifier" version = "0.2.22" From 59a1f8c902b2e46f5c278cb391ef641e2e55b8c7 Mon Sep 17 00:00:00 2001 From: Untone Date: Fri, 26 Jan 2024 03:40:49 +0300 Subject: [PATCH 3/9] fmt2 --- .pre-commit-config.yaml | 21 +++++++ main.py | 12 ++-- orm/author.py | 29 +++++---- orm/notification.py | 37 ++++++------ resolvers/listener.py | 22 +++---- resolvers/load.py | 130 ++++++++++++++++++++++++---------------- resolvers/model.py | 5 +- resolvers/schema.py | 4 +- resolvers/seen.py | 42 ++++++------- services/auth.py | 53 ++++++++-------- services/core.py | 24 ++++---- services/db.py | 13 ++-- services/rediscache.py | 28 +++++---- settings.py | 17 +++--- 14 files changed, 249 insertions(+), 188 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..43fa826 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +fail_fast: true + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-added-large-files + - id: detect-private-key + - id: double-quote-string-fixer + - id: check-ast + - id: check-merge-conflict + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.13 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format diff --git a/main.py b/main.py index 9b7f841..440c7d3 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ import asyncio +import logging import os from os.path import exists @@ -13,11 +14,10 @@ from resolvers.listener import notifications_worker from resolvers.schema import schema from services.rediscache import redis from settings import DEV_SERVER_PID_FILE_NAME, MODE, SENTRY_DSN -import logging logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger("\t[main]\t") +logger = logging.getLogger('\t[main]\t') logger.setLevel(logging.DEBUG) @@ -27,9 +27,9 @@ async def start_up(): task = asyncio.create_task(notifications_worker()) logger.info(task) - if MODE == "dev": + if MODE == 'dev': if exists(DEV_SERVER_PID_FILE_NAME): - with open(DEV_SERVER_PID_FILE_NAME, "w", encoding="utf-8") as f: + with open(DEV_SERVER_PID_FILE_NAME, 'w', encoding='utf-8') as f: f.write(str(os.getpid())) else: try: @@ -46,7 +46,7 @@ async def start_up(): ], ) except Exception as e: - logger.error("sentry init error", e) + logger.error('sentry init error', e) async def shutdown(): @@ -54,4 +54,4 @@ async def shutdown(): app = Starlette(debug=True, on_startup=[start_up], on_shutdown=[shutdown]) -app.mount("/", GraphQL(schema, debug=True)) +app.mount('/', GraphQL(schema, debug=True)) diff --git a/orm/author.py b/orm/author.py index ba9fcc5..49bf6d1 100644 --- a/orm/author.py +++ b/orm/author.py @@ -1,46 +1,45 @@ import time -from sqlalchemy import JSON as JSONType -from sqlalchemy import Boolean, Column, ForeignKey, Integer, String +from sqlalchemy import JSON, Boolean, Column, ForeignKey, Integer, String from sqlalchemy.orm import relationship from services.db import Base class AuthorRating(Base): - __tablename__ = "author_rating" + __tablename__ = 'author_rating' id = None # type: ignore - rater = Column(ForeignKey("author.id"), primary_key=True, index=True) - author = Column(ForeignKey("author.id"), primary_key=True, index=True) + rater = Column(ForeignKey('author.id'), primary_key=True, index=True) + author = Column(ForeignKey('author.id'), primary_key=True, index=True) plus = Column(Boolean) class AuthorFollower(Base): - __tablename__ = "author_follower" + __tablename__ = 'author_follower' id = None # type: ignore - follower = Column(ForeignKey("author.id"), primary_key=True, index=True) - author = Column(ForeignKey("author.id"), primary_key=True, index=True) + follower = Column(ForeignKey('author.id'), primary_key=True, index=True) + author = Column(ForeignKey('author.id'), primary_key=True, index=True) created_at = Column(Integer, nullable=False, default=lambda: int(time.time())) auto = Column(Boolean, nullable=False, default=False) class Author(Base): - __tablename__ = "author" + __tablename__ = 'author' user = Column(String, unique=True) # unbounded link with authorizer's User type - name = Column(String, nullable=True, comment="Display name") + name = Column(String, nullable=True, comment='Display name') slug = Column(String, unique=True, comment="Author's slug") - bio = Column(String, nullable=True, comment="Bio") # status description - about = Column(String, nullable=True, comment="About") # long and formatted - pic = Column(String, nullable=True, comment="Picture") - links = Column(JSONType, nullable=True, comment="Links") + bio = Column(String, nullable=True, comment='Bio') # status description + about = Column(String, nullable=True, comment='About') # long and formatted + pic = Column(String, nullable=True, comment='Picture') + links = Column(JSON, nullable=True, comment='Links') ratings = relationship(AuthorRating, foreign_keys=AuthorRating.author) created_at = Column(Integer, nullable=False, default=lambda: int(time.time())) last_seen = Column(Integer, nullable=False, default=lambda: int(time.time())) updated_at = Column(Integer, nullable=False, default=lambda: int(time.time())) - deleted_at = Column(Integer, nullable=True, comment="Deleted at") + deleted_at = Column(Integer, nullable=True, comment='Deleted at') diff --git a/orm/notification.py b/orm/notification.py index 0622ad4..2b09ea1 100644 --- a/orm/notification.py +++ b/orm/notification.py @@ -1,42 +1,41 @@ +import time from enum import Enum as Enumeration -from sqlalchemy import JSON as JSONType, func, cast -from sqlalchemy import Column, Enum, ForeignKey, Integer, String +from sqlalchemy import JSON, Column, ForeignKey, Integer, String from sqlalchemy.orm import relationship -from sqlalchemy.orm.session import engine from orm.author import Author from services.db import Base -import time + class NotificationEntity(Enumeration): - REACTION = "reaction" - SHOUT = "shout" - FOLLOWER = "follower" + REACTION = 'reaction' + SHOUT = 'shout' + FOLLOWER = 'follower' class NotificationAction(Enumeration): - CREATE = "create" - UPDATE = "update" - DELETE = "delete" - SEEN = "seen" - FOLLOW = "follow" - UNFOLLOW = "unfollow" + CREATE = 'create' + UPDATE = 'update' + DELETE = 'delete' + SEEN = 'seen' + FOLLOW = 'follow' + UNFOLLOW = 'unfollow' class NotificationSeen(Base): - __tablename__ = "notification_seen" + __tablename__ = 'notification_seen' - viewer = Column(ForeignKey("author.id")) - notification = Column(ForeignKey("notification.id")) + viewer = Column(ForeignKey('author.id')) + notification = Column(ForeignKey('notification.id')) class Notification(Base): - __tablename__ = "notification" + __tablename__ = 'notification' created_at = Column(Integer, server_default=str(int(time.time()))) entity = Column(String, nullable=False) action = Column(String, nullable=False) - payload = Column(JSONType, nullable=True) + payload = Column(JSON, nullable=True) - seen = relationship(lambda: Author, secondary="notification_seen") + seen = relationship(lambda: Author, secondary='notification_seen') diff --git a/resolvers/listener.py b/resolvers/listener.py index 39d3ee3..95b762c 100644 --- a/resolvers/listener.py +++ b/resolvers/listener.py @@ -1,11 +1,13 @@ -from orm.notification import Notification, NotificationAction, NotificationEntity -from resolvers.model import NotificationReaction, NotificationAuthor, NotificationShout -from services.db import local_session -from services.rediscache import redis import asyncio import logging -logger = logging.getLogger(f"[listener.listen_task] ") +from orm.notification import Notification +from resolvers.model import NotificationAuthor, NotificationReaction, NotificationShout +from services.db import local_session +from services.rediscache import redis + + +logger = logging.getLogger('[listener.listen_task] ') logger.setLevel(logging.DEBUG) @@ -19,8 +21,8 @@ async def handle_notification(n: ServiceMessage, channel: str): """создаеёт новое хранимое уведомление""" with local_session() as session: try: - if channel.startswith("follower:"): - author_id = int(channel.split(":")[1]) + if channel.startswith('follower:'): + author_id = int(channel.split(':')[1]) if isinstance(n.payload, NotificationAuthor): n.payload.following_id = author_id n = Notification(action=n.action, entity=n.entity, payload=n.payload) @@ -28,7 +30,7 @@ async def handle_notification(n: ServiceMessage, channel: str): session.commit() except Exception as e: session.rollback() - logger.error(f"[listener.handle_reaction] error: {str(e)}") + logger.error(f'[listener.handle_reaction] error: {str(e)}') async def listen_task(pattern): @@ -38,9 +40,9 @@ async def listen_task(pattern): notification_message = ServiceMessage(**message_data) await handle_notification(notification_message, str(channel)) except Exception as e: - logger.error(f"Error processing notification: {str(e)}") + logger.error(f'Error processing notification: {str(e)}') async def notifications_worker(): # Use asyncio.gather to run tasks concurrently - await asyncio.gather(listen_task("follower:*"), listen_task("reaction"), listen_task("shout")) + await asyncio.gather(listen_task('follower:*'), listen_task('reaction'), listen_task('shout')) diff --git a/resolvers/load.py b/resolvers/load.py index 65f3b1d..fce5671 100644 --- a/resolvers/load.py +++ b/resolvers/load.py @@ -1,27 +1,36 @@ +import json +import logging +import time +from typing import Dict, List +import strawberry +from sqlalchemy import and_, select +from sqlalchemy.orm import aliased from sqlalchemy.sql import not_ -from services.db import local_session + +from orm.notification import ( + Notification, + NotificationAction, + NotificationEntity, + NotificationSeen, +) from resolvers.model import ( - NotificationReaction, - NotificationGroup, - NotificationShout, NotificationAuthor, + NotificationGroup, + NotificationReaction, + NotificationShout, NotificationsResult, ) -from orm.notification import NotificationAction, NotificationEntity, NotificationSeen, Notification -from typing import Dict, List -import time, json -import strawberry -from sqlalchemy.orm import aliased -from sqlalchemy.sql.expression import or_ -from sqlalchemy import select, and_ -import logging +from services.db import local_session -logger = logging.getLogger("[resolvers.schema] ") + +logger = logging.getLogger('[resolvers.schema] ') logger.setLevel(logging.DEBUG) -async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = 10, offset: int = 0): +async def get_notifications_grouped( # noqa: C901 + author_id: int, after: int = 0, limit: int = 10, offset: int = 0 +): """ Retrieves notifications for a given author. @@ -47,10 +56,13 @@ async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = authors: List[NotificationAuthor], # List of authors involved in the thread. } """ - NotificationSeenAlias = aliased(NotificationSeen) - query = select(Notification, NotificationSeenAlias.viewer.label("seen")).outerjoin( + seen_alias = aliased(NotificationSeen) + query = select(Notification, seen_alias.viewer.label('seen')).outerjoin( NotificationSeen, - and_(NotificationSeen.viewer == author_id, NotificationSeen.notification == Notification.id), + and_( + NotificationSeen.viewer == author_id, + NotificationSeen.notification == Notification.id, + ), ) if after: query = query.filter(Notification.created_at > after) @@ -62,23 +74,36 @@ async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = notifications_by_thread: Dict[str, List[Notification]] = {} groups_by_thread: Dict[str, NotificationGroup] = {} with local_session() as session: - total = session.query(Notification).filter(and_(Notification.action == NotificationAction.CREATE.value, Notification.created_at > after)).count() - unread = session.query(Notification).filter( - and_( - Notification.action == NotificationAction.CREATE.value, - Notification.created_at > after, - not_(Notification.seen) + total = ( + session.query(Notification) + .filter( + and_( + Notification.action == NotificationAction.CREATE.value, + Notification.created_at > after, + ) ) - ).count() + .count() + ) + unread = ( + session.query(Notification) + .filter( + and_( + Notification.action == NotificationAction.CREATE.value, + Notification.created_at > after, + not_(Notification.seen), + ) + ) + .count() + ) notifications_result = session.execute(query) - for n, seen in notifications_result: - thread_id = "" + for n, _seen in notifications_result: + thread_id = '' payload = json.loads(n.payload) - logger.debug(f"[resolvers.schema] {n.action} {n.entity}: {payload}") - if n.entity == "shout" and n.action == "create": + logger.debug(f'[resolvers.schema] {n.action} {n.entity}: {payload}') + if n.entity == 'shout' and n.action == 'create': shout: NotificationShout = payload - thread_id += f"{shout.id}" - logger.debug(f"create shout: {shout}") + thread_id += f'{shout.id}' + logger.debug(f'create shout: {shout}') group = groups_by_thread.get(thread_id) or NotificationGroup( id=thread_id, entity=n.entity, @@ -86,8 +111,8 @@ async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = authors=shout.authors, updated_at=shout.created_at, reactions=[], - action="create", - seen=author_id in n.seen + action='create', + seen=author_id in n.seen, ) # store group in result groups_by_thread[thread_id] = group @@ -99,11 +124,11 @@ async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = elif n.entity == NotificationEntity.REACTION.value and n.action == NotificationAction.CREATE.value: reaction: NotificationReaction = payload shout: NotificationShout = reaction.shout - thread_id += f"{reaction.shout}" - if reaction.kind == "LIKE" or reaction.kind == "DISLIKE": + thread_id += f'{reaction.shout}' + if not bool(reaction.reply_to) and (reaction.kind == 'LIKE' or reaction.kind == 'DISLIKE'): # TODO: making published reaction vote announce pass - elif reaction.kind == "COMMENT": + elif reaction.kind == 'COMMENT': if reaction.reply_to: thread_id += f"{'::' + str(reaction.reply_to)}" group: NotificationGroup | None = groups_by_thread.get(thread_id) @@ -128,8 +153,9 @@ async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = break else: # init notification group - reactions = [] - reactions.append(reaction.id) + reactions = [ + reaction.id, + ] group = NotificationGroup( id=thread_id, action=n.action, @@ -140,7 +166,7 @@ async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = authors=[ reaction.created_by, ], - seen=author_id in n.seen + seen=author_id in n.seen, ) # store group in result groups_by_thread[thread_id] = group @@ -149,20 +175,22 @@ async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = notifications.append(n) notifications_by_thread[thread_id] = notifications - elif n.entity == "follower": - thread_id = "followers" + elif n.entity == 'follower': + thread_id = 'followers' follower: NotificationAuthor = payload group = groups_by_thread.get(thread_id) or NotificationGroup( - id=thread_id, - authors=[follower], - updated_at=int(time.time()), - shout=None, - reactions=[], - entity="follower", - action="follow", - seen=author_id in n.seen - ) - group.authors = [follower, ] + id=thread_id, + authors=[follower], + updated_at=int(time.time()), + shout=None, + reactions=[], + entity='follower', + action='follow', + seen=author_id in n.seen, + ) + group.authors = [ + follower, + ] group.updated_at = int(time.time()) # store group in result groups_by_thread[thread_id] = group @@ -182,7 +210,7 @@ async def get_notifications_grouped(author_id: int, after: int = 0, limit: int = class Query: @strawberry.field async def load_notifications(self, info, after: int, limit: int = 50, offset: int = 0) -> NotificationsResult: - author_id = info.context.get("author_id") + author_id = info.context.get('author_id') groups: Dict[str, NotificationGroup] = {} if author_id: groups, notifications, total, unread = await get_notifications_grouped(author_id, after, limit, offset) diff --git a/resolvers/model.py b/resolvers/model.py index 357f2bf..0ccc9b4 100644 --- a/resolvers/model.py +++ b/resolvers/model.py @@ -1,8 +1,11 @@ -import strawberry from typing import List, Optional + +import strawberry from strawberry_sqlalchemy_mapper import StrawberrySQLAlchemyMapper + from orm.notification import Notification as NotificationMessage + strawberry_sqlalchemy_mapper = StrawberrySQLAlchemyMapper() diff --git a/resolvers/schema.py b/resolvers/schema.py index 7e87145..da45ad1 100644 --- a/resolvers/schema.py +++ b/resolvers/schema.py @@ -1,12 +1,12 @@ - import strawberry from strawberry.schema.config import StrawberryConfig -from services.auth import LoginRequiredMiddleware from resolvers.load import Query from resolvers.seen import Mutation +from services.auth import LoginRequiredMiddleware from services.db import Base, engine + schema = strawberry.Schema( query=Query, mutation=Mutation, config=StrawberryConfig(auto_camel_case=False), extensions=[LoginRequiredMiddleware] ) diff --git a/resolvers/seen.py b/resolvers/seen.py index bfbc260..d1cce6f 100644 --- a/resolvers/seen.py +++ b/resolvers/seen.py @@ -1,14 +1,14 @@ -from sqlalchemy import and_ -from orm.notification import NotificationSeen -from services.db import local_session -from resolvers.model import Notification, NotificationSeenResult, NotificationReaction +import json +import logging import strawberry -import logging -import json - +from sqlalchemy import and_ from sqlalchemy.exc import SQLAlchemyError +from orm.notification import NotificationSeen +from resolvers.model import Notification, NotificationReaction, NotificationSeenResult +from services.db import local_session + logger = logging.getLogger(__name__) @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) class Mutation: @strawberry.mutation async def mark_seen(self, info, notification_id: int) -> NotificationSeenResult: - author_id = info.context.get("author_id") + author_id = info.context.get('author_id') if author_id: with local_session() as session: try: @@ -27,9 +27,9 @@ class Mutation: except SQLAlchemyError as e: session.rollback() logger.error( - f"[mark_notification_as_read] Ошибка при обновлении статуса прочтения уведомления: {str(e)}" + f'[mark_notification_as_read] Ошибка при обновлении статуса прочтения уведомления: {str(e)}' ) - return NotificationSeenResult(error="cant mark as read") + return NotificationSeenResult(error='cant mark as read') return NotificationSeenResult(error=None) @strawberry.mutation @@ -37,7 +37,7 @@ class Mutation: # TODO: use latest loaded notification_id as input offset parameter error = None try: - author_id = info.context.get("author_id") + author_id = info.context.get('author_id') if author_id: with local_session() as session: nnn = session.query(Notification).filter(and_(Notification.created_at > after)).all() @@ -46,26 +46,26 @@ class Mutation: ns = NotificationSeen(notification=n.id, viewer=author_id) session.add(ns) session.commit() - except SQLAlchemyError as e: + except SQLAlchemyError: session.rollback() except Exception as e: print(e) - error = "cant mark as read" + error = 'cant mark as read' return NotificationSeenResult(error=error) @strawberry.mutation async def mark_seen_thread(self, info, thread: str, after: int) -> NotificationSeenResult: error = None - author_id = info.context.get("author_id") + author_id = info.context.get('author_id') if author_id: - [shout_id, reply_to_id] = thread.split("::") + [shout_id, reply_to_id] = thread.split('::') with local_session() as session: # TODO: handle new follower and new shout notifications new_reaction_notifications = ( session.query(Notification) .filter( - Notification.action == "create", - Notification.entity == "reaction", + Notification.action == 'create', + Notification.entity == 'reaction', Notification.created_at > after, ) .all() @@ -73,13 +73,13 @@ class Mutation: removed_reaction_notifications = ( session.query(Notification) .filter( - Notification.action == "delete", - Notification.entity == "reaction", + Notification.action == 'delete', + Notification.entity == 'reaction', Notification.created_at > after, ) .all() ) - exclude = set([]) + exclude = set() for nr in removed_reaction_notifications: reaction: NotificationReaction = json.loads(nr.payload) exclude.add(reaction.id) @@ -97,5 +97,5 @@ class Mutation: except Exception: session.rollback() else: - error = "You are not logged in" + error = 'You are not logged in' return NotificationSeenResult(error=error) diff --git a/services/auth.py b/services/auth.py index 6912382..b84a9bd 100644 --- a/services/auth.py +++ b/services/auth.py @@ -1,57 +1,60 @@ +import logging + from aiohttp import ClientSession from strawberry.extensions import Extension -from settings import AUTH_URL -from services.db import local_session from orm.author import Author +from services.db import local_session +from settings import AUTH_URL -import logging -logger = logging.getLogger("\t[services.auth]\t") +logger = logging.getLogger('\t[services.auth]\t') logger.setLevel(logging.DEBUG) + async def check_auth(req) -> str | None: - token = req.headers.get("Authorization") - user_id = "" + token = req.headers.get('Authorization') + user_id = '' if token: - query_name = "validate_jwt_token" - operation = "ValidateToken" + query_name = 'validate_jwt_token' + operation = 'ValidateToken' headers = { - "Content-Type": "application/json", + 'Content-Type': 'application/json', } variables = { - "params": { - "token_type": "access_token", - "token": token, + 'params': { + 'token_type': 'access_token', + 'token': token, } } gql = { - "query": f"query {operation}($params: ValidateJWTTokenInput!) {{ {query_name}(params: $params) {{ is_valid claims }} }}", - "variables": variables, - "operationName": operation, + 'query': f'query {operation}($params: ValidateJWTTokenInput!) {{ {query_name}(params: $params) {{ is_valid claims }} }}', + 'variables': variables, + 'operationName': operation, } try: # Asynchronous HTTP request to the authentication server async with ClientSession() as session: async with session.post(AUTH_URL, json=gql, headers=headers) as response: - print(f"[services.auth] HTTP Response {response.status} {await response.text()}") + print(f'[services.auth] HTTP Response {response.status} {await response.text()}') if response.status == 200: data = await response.json() - errors = data.get("errors") + errors = data.get('errors') if errors: - print(f"[services.auth] errors: {errors}") + print(f'[services.auth] errors: {errors}') else: - user_id = data.get("data", {}).get(query_name, {}).get("claims", {}).get("sub") + user_id = data.get('data', {}).get(query_name, {}).get('claims', {}).get('sub') if user_id: - print(f"[services.auth] got user_id: {user_id}") + print(f'[services.auth] got user_id: {user_id}') return user_id except Exception as e: import traceback + traceback.print_exc() # Handling and logging exceptions during authentication check - print(f"[services.auth] Error {e}") + print(f'[services.auth] Error {e}') return None @@ -59,14 +62,14 @@ async def check_auth(req) -> str | None: class LoginRequiredMiddleware(Extension): async def on_request_start(self): context = self.execution_context.context - req = context.get("request") + req = context.get('request') user_id = await check_auth(req) if user_id: - context["user_id"] = user_id.strip() + context['user_id'] = user_id.strip() with local_session() as session: author = session.query(Author).filter(Author.user == user_id).first() if author: - context["author_id"] = author.id - context["user_id"] = user_id or None + context['author_id'] = author.id + context['user_id'] = user_id or None self.execution_context.context = context diff --git a/services/core.py b/services/core.py index 8957f02..148b393 100644 --- a/services/core.py +++ b/services/core.py @@ -4,47 +4,49 @@ import aiohttp from settings import API_BASE -headers = {"Content-Type": "application/json"} + +headers = {'Content-Type': 'application/json'} # TODO: rewrite to orm usage? + async def _request_endpoint(query_name, body) -> Any: async with aiohttp.ClientSession() as session: async with session.post(API_BASE, headers=headers, json=body) as response: - print(f"[services.core] {query_name} HTTP Response {response.status} {await response.text()}") + print(f'[services.core] {query_name} HTTP Response {response.status} {await response.text()}') if response.status == 200: r = await response.json() if r: - return r.get("data", {}).get(query_name, {}) + return r.get('data', {}).get(query_name, {}) return [] async def get_followed_shouts(author_id: int): - query_name = "load_shouts_followed" - operation = "GetFollowedShouts" + query_name = 'load_shouts_followed' + operation = 'GetFollowedShouts' query = f"""query {operation}($author_id: Int!, limit: Int, offset: Int) {{ {query_name}(author_id: $author_id, limit: $limit, offset: $offset) {{ id slug title }} }}""" gql = { - "query": query, - "operationName": operation, - "variables": {"author_id": author_id, "limit": 1000, "offset": 0}, # FIXME: too big limit + 'query': query, + 'operationName': operation, + 'variables': {'author_id': author_id, 'limit': 1000, 'offset': 0}, # FIXME: too big limit } return await _request_endpoint(query_name, gql) async def get_shout(shout_id): - query_name = "get_shout" - operation = "GetShout" + query_name = 'get_shout' + operation = 'GetShout' query = f"""query {operation}($slug: String, $shout_id: Int) {{ {query_name}(slug: $slug, shout_id: $shout_id) {{ id slug title authors {{ id slug name pic }} }} }}""" - gql = {"query": query, "operationName": operation, "variables": {"slug": None, "shout_id": shout_id}} + gql = {'query': query, 'operationName': operation, 'variables': {'slug': None, 'shout_id': shout_id}} return await _request_endpoint(query_name, gql) diff --git a/services/db.py b/services/db.py index e9b15d6..c1e3bd6 100644 --- a/services/db.py +++ b/services/db.py @@ -9,15 +9,16 @@ from sqlalchemy.sql.schema import Table from settings import DB_URL + engine = create_engine(DB_URL, echo=False, pool_size=10, max_overflow=20) -T = TypeVar("T") +T = TypeVar('T') REGISTRY: Dict[str, type] = {} # @contextmanager -def local_session(src=""): +def local_session(src=''): return Session(bind=engine, expire_on_commit=False) # try: @@ -45,7 +46,7 @@ class Base(declarative_base()): __init__: Callable __allow_unmapped__ = True __abstract__ = True - __table_args__ = {"extend_existing": True} + __table_args__ = {'extend_existing': True} id = Column(Integer, primary_key=True) @@ -54,12 +55,12 @@ class Base(declarative_base()): def dict(self) -> Dict[str, Any]: column_names = self.__table__.columns.keys() - if "_sa_instance_state" in column_names: - column_names.remove("_sa_instance_state") + if '_sa_instance_state' in column_names: + column_names.remove('_sa_instance_state') try: return {c: getattr(self, c) for c in column_names} except Exception as e: - print(f"[services.db] Error dict: {e}") + print(f'[services.db] Error dict: {e}') return {} def update(self, values: Dict[str, Any]) -> None: diff --git a/services/rediscache.py b/services/rediscache.py index 3609a99..15a89b3 100644 --- a/services/rediscache.py +++ b/services/rediscache.py @@ -1,14 +1,16 @@ -import json - -import redis.asyncio as aredis import asyncio -from settings import REDIS_URL - +import json import logging -logger = logging.getLogger("\t[services.redis]\t") +import redis.asyncio as aredis + +from settings import REDIS_URL + + +logger = logging.getLogger('\t[services.redis]\t') logger.setLevel(logging.DEBUG) + class RedisCache: def __init__(self, uri=REDIS_URL): self._uri: str = uri @@ -25,11 +27,11 @@ class RedisCache: async def execute(self, command, *args, **kwargs): if self._client: try: - logger.debug(command + " " + " ".join(args)) + logger.debug(command + ' ' + ' '.join(args)) r = await self._client.execute_command(command, *args, **kwargs) return r except Exception as e: - logger.error(f"{e}") + logger.error(f'{e}') return None async def subscribe(self, *channels): @@ -59,15 +61,15 @@ class RedisCache: while True: message = await pubsub.get_message() - if message and isinstance(message["data"], (str, bytes, bytearray)): - logger.debug("pubsub got msg") + if message and isinstance(message['data'], (str, bytes, bytearray)): + logger.debug('pubsub got msg') try: - yield json.loads(message["data"]), message.get("channel") + yield json.loads(message['data']), message.get('channel') except Exception as e: - logger.error(f"{e}") + logger.error(f'{e}') await asyncio.sleep(1) redis = RedisCache() -__all__ = ["redis"] +__all__ = ['redis'] diff --git a/settings.py b/settings.py index eaae333..3eae789 100644 --- a/settings.py +++ b/settings.py @@ -1,13 +1,14 @@ from os import environ + PORT = 80 DB_URL = ( - environ.get("DATABASE_URL", environ.get("DB_URL", "")).replace("postgres://", "postgresql://") - or "postgresql://postgres@localhost:5432/discoursio" + environ.get('DATABASE_URL', environ.get('DB_URL', '')).replace('postgres://', 'postgresql://') + or 'postgresql://postgres@localhost:5432/discoursio' ) -REDIS_URL = environ.get("REDIS_URL") or "redis://127.0.0.1" -API_BASE = environ.get("API_BASE") or "https://core.discours.io" -AUTH_URL = environ.get("AUTH_URL") or "https://auth.discours.io" -MODE = environ.get("MODE") or "production" -SENTRY_DSN = environ.get("SENTRY_DSN") -DEV_SERVER_PID_FILE_NAME = "dev-server.pid" +REDIS_URL = environ.get('REDIS_URL') or 'redis://127.0.0.1' +API_BASE = environ.get('API_BASE') or 'https://core.discours.io' +AUTH_URL = environ.get('AUTH_URL') or 'https://auth.discours.io' +MODE = environ.get('MODE') or 'production' +SENTRY_DSN = environ.get('SENTRY_DSN') +DEV_SERVER_PID_FILE_NAME = 'dev-server.pid' From 15fd6ec76503fa5bbf26c43d6f5cf1c02deb4873 Mon Sep 17 00:00:00 2001 From: Untone Date: Sat, 3 Feb 2024 18:36:33 +0300 Subject: [PATCH 4/9] check-deploy --- .pre-commit-config.yaml | 3 +-- Dockerfile | 4 ++-- pyproject.toml | 10 ++++++++-- server.py | 17 +++++++++++++++++ services/auth.py | 17 +++++++++++++---- 5 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 server.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43fa826..757f660 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,8 +14,7 @@ repos: - id: check-merge-conflict - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.13 + rev: v0.1.15 hooks: - id: ruff args: [--fix] - - id: ruff-format diff --git a/Dockerfile b/Dockerfile index 6401033..350acd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM python:slim WORKDIR /app # Add metadata to the image to describe that the container is listening on port 80 -EXPOSE 80 +EXPOSE 8000 # Copy the current directory contents into the container at /app COPY . /app @@ -19,4 +19,4 @@ RUN apt-get update && apt-get install -y gcc curl && \ poetry install --no-dev # Run server.py when the container launches -CMD granian --no-ws --host 0.0.0.0 --port 80 --interface asgi main:app +CMD python server.py diff --git a/pyproject.toml b/pyproject.toml index 1af9476..33420bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ + [tool.poetry] name = "discoursio-notifier" -version = "0.2.22" +version = "0.3.0" description = "notifier server for discours.io" authors = ["discours.io devteam"] @@ -15,19 +16,24 @@ sentry-sdk = "^1.39.2" aiohttp = "^3.9.1" pre-commit = "^3.6.0" granian = "^1.0.1" +discoursio-core = { git = "https://dev.discours.io/discours.io/core.git", branch = "feature/core" } [tool.poetry.group.dev.dependencies] setuptools = "^69.0.2" pyright = "^1.1.341" pytest = "^7.4.2" black = { version = "^23.12.0", python = ">=3.12" } -ruff = { version = "^0.1.8", python = ">=3.12" } +ruff = { version = "^0.1.15", python = ">=3.12" } isort = "^5.13.2" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" +[tool.setuptools.dynamic] +version = {attr = "notifier.__version__"} +readme = {file = "README.md"} + [tool.ruff] line-length = 120 extend-select = [ diff --git a/server.py b/server.py new file mode 100644 index 0000000..1301ba5 --- /dev/null +++ b/server.py @@ -0,0 +1,17 @@ +from granian.constants import Interfaces +from granian.server import Granian + + +if __name__ == '__main__': + print('[server] started') + + granian_instance = Granian( + 'main:app', + address='0.0.0.0', # noqa S104 + port=8000, + workers=2, + threads=2, + websockets=False, + interface=Interfaces.ASGI, + ) + granian_instance.serve() diff --git a/services/auth.py b/services/auth.py index b84a9bd..f26b37a 100644 --- a/services/auth.py +++ b/services/auth.py @@ -37,15 +37,24 @@ async def check_auth(req) -> str | None: try: # Asynchronous HTTP request to the authentication server async with ClientSession() as session: - async with session.post(AUTH_URL, json=gql, headers=headers) as response: - print(f'[services.auth] HTTP Response {response.status} {await response.text()}') + async with session.post( + AUTH_URL, json=gql, headers=headers + ) as response: + print( + f'[services.auth] HTTP Response {response.status} {await response.text()}' + ) if response.status == 200: data = await response.json() errors = data.get('errors') if errors: - print(f'[services.auth] errors: {errors}') + print(f'errors: {errors}') else: - user_id = data.get('data', {}).get(query_name, {}).get('claims', {}).get('sub') + user_id = ( + data.get('data', {}) + .get(query_name, {}) + .get('claims', {}) + .get('sub') + ) if user_id: print(f'[services.auth] got user_id: {user_id}') return user_id From 3c3e93767c8c5ee13b3041fb133fd875d4768252 Mon Sep 17 00:00:00 2001 From: Untone Date: Sat, 3 Feb 2024 19:22:34 +0300 Subject: [PATCH 5/9] deploy-fix --- .gitea/workflows/main.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/main.yml b/.gitea/workflows/main.yml index c88393d..bace4bb 100644 --- a/.gitea/workflows/main.yml +++ b/.gitea/workflows/main.yml @@ -1,4 +1,4 @@ -name: 'deploy' +name: "Deploy to notifier" on: [push] jobs: @@ -14,9 +14,13 @@ jobs: id: repo_name run: echo "::set-output name=repo::$(echo ${GITHUB_REPOSITORY##*/})" + - name: Get Branch Name + id: branch_name + run: echo "::set-output name=branch::$(echo ${GITHUB_REF##*/})" + - name: Push to dokku uses: dokku/github-action@master with: - branch: 'main' - git_remote_url: 'ssh://dokku@staging.discours.io:22/${{ steps.repo_name.outputs.repo }}' - ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }} \ No newline at end of file + branch: "feature/core" + git_remote_url: "ssh://dokku@v2.discours.io:22/notifier" + ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }} From 9f7143d756e1e552a3853195c5ed365d113e98a9 Mon Sep 17 00:00:00 2001 From: Untone Date: Sat, 3 Feb 2024 19:24:35 +0300 Subject: [PATCH 6/9] deploy-fix-2 --- .gitea/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/main.yml b/.gitea/workflows/main.yml index bace4bb..6837b66 100644 --- a/.gitea/workflows/main.yml +++ b/.gitea/workflows/main.yml @@ -21,6 +21,6 @@ jobs: - name: Push to dokku uses: dokku/github-action@master with: - branch: "feature/core" + branch: "main" git_remote_url: "ssh://dokku@v2.discours.io:22/notifier" ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }} From 3e0f7b05215d083618857f785d1a90f6ab56c955 Mon Sep 17 00:00:00 2001 From: Untone Date: Sat, 3 Feb 2024 19:36:17 +0300 Subject: [PATCH 7/9] graphiql-fix --- main.py | 3 +-- services/auth.py | 15 +++------------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/main.py b/main.py index 440c7d3..5abe0b9 100644 --- a/main.py +++ b/main.py @@ -16,7 +16,6 @@ from services.rediscache import redis from settings import DEV_SERVER_PID_FILE_NAME, MODE, SENTRY_DSN -logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger('\t[main]\t') logger.setLevel(logging.DEBUG) @@ -54,4 +53,4 @@ async def shutdown(): app = Starlette(debug=True, on_startup=[start_up], on_shutdown=[shutdown]) -app.mount('/', GraphQL(schema, debug=True)) +app.mount('/', GraphQL(schema, graphiql=True, debug=True)) diff --git a/services/auth.py b/services/auth.py index f26b37a..72d589b 100644 --- a/services/auth.py +++ b/services/auth.py @@ -37,24 +37,15 @@ async def check_auth(req) -> str | None: try: # Asynchronous HTTP request to the authentication server async with ClientSession() as session: - async with session.post( - AUTH_URL, json=gql, headers=headers - ) as response: - print( - f'[services.auth] HTTP Response {response.status} {await response.text()}' - ) + async with session.post(AUTH_URL, json=gql, headers=headers) as response: + print(f'[services.auth] HTTP Response {response.status} {await response.text()}') if response.status == 200: data = await response.json() errors = data.get('errors') if errors: print(f'errors: {errors}') else: - user_id = ( - data.get('data', {}) - .get(query_name, {}) - .get('claims', {}) - .get('sub') - ) + user_id = data.get('data', {}).get(query_name, {}).get('claims', {}).get('sub') if user_id: print(f'[services.auth] got user_id: {user_id}') return user_id From d62e229f63b968be51b72636f6a6d8c0f6819fea Mon Sep 17 00:00:00 2001 From: Untone Date: Sat, 3 Feb 2024 20:02:51 +0300 Subject: [PATCH 8/9] debug-runtime --- resolvers/listener.py | 1 + 1 file changed, 1 insertion(+) diff --git a/resolvers/listener.py b/resolvers/listener.py index 95b762c..00f826a 100644 --- a/resolvers/listener.py +++ b/resolvers/listener.py @@ -34,6 +34,7 @@ async def handle_notification(n: ServiceMessage, channel: str): async def listen_task(pattern): + logger.info(f' listening {pattern} ...') async for message_data, channel in redis.listen(pattern): try: if message_data: From 1635976edf4e7b65ac60da7682f44a2b4c4c2c34 Mon Sep 17 00:00:00 2001 From: Untone Date: Sat, 3 Feb 2024 22:43:04 +0300 Subject: [PATCH 9/9] debug-4 --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index 5abe0b9..1a9703b 100644 --- a/main.py +++ b/main.py @@ -21,6 +21,7 @@ logger.setLevel(logging.DEBUG) async def start_up(): + logger.info('[main] starting...') await redis.connect() task = asyncio.create_task(notifications_worker()) @@ -31,6 +32,7 @@ async def start_up(): with open(DEV_SERVER_PID_FILE_NAME, 'w', encoding='utf-8') as f: f.write(str(os.getpid())) else: + logger.info('[main] production mode') try: import sentry_sdk