diff --git a/migration/__init__.py b/migration/__init__.py index a998ad07..92a97549 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -10,7 +10,8 @@ from migration.tables.comments import migrate_2stage as migrateComment_2stage from migration.tables.content_items import get_shout_slug from migration.tables.content_items import migrate as migrateShout from migration.tables.topics import migrate as migrateTopic -from migration.tables.users import migrate as migrateUser +from migration.tables.users import migrate as migrateUser, post_migrate as users_post_migrate +from migration.tables.remarks import migrate as migrateRemark from migration.tables.users import migrate_2stage as migrateUser_2stage from orm.reaction import Reaction from orm import init_tables @@ -41,6 +42,7 @@ async def users_handle(storage): ce = 0 for entry in storage["users"]["data"]: ce += migrateUser_2stage(entry, id_map) + users_post_migrate() async def topics_handle(storage): @@ -135,6 +137,15 @@ async def shouts_handle(storage, args): print("[migration] " + str(anonymous_author) + " authored by @anonymous") +async def remarks_handle(storage): + print("[migration] comments") + c = 0 + for entry_remark in storage["remarks"]["data"]: + remark = await migrateRemark(entry_remark, storage) + c += 1 + print("[migration] " + str(c) + " remarks migrated") + + async def comments_handle(storage): print("[migration] comments") id_map = {} @@ -170,6 +181,8 @@ async def all_handle(storage, args): await topics_handle(storage) print("[migration] users and topics are migrated") await shouts_handle(storage, args) + # print("[migration] remarks...") + # await remarks_handle(storage) print("[migration] migrating comments") await comments_handle(storage) # export_email_subscriptions() @@ -190,6 +203,7 @@ def data_load(): "cats": [], "tags": [], }, + "remarks": {"data": []}, "users": {"by_oid": {}, "by_slug": {}, "data": []}, "replacements": json.loads(open("migration/tables/replacements.json").read()), } @@ -210,6 +224,11 @@ def data_load(): content_data = json.loads(open("migration/data/content_items.json").read()) storage["shouts"]["data"] = content_data print("[migration.load] " + str(len(content_data)) + " content items ") + + remarks_data = json.loads(open("migration/data/remarks.json").read()) + storage["remarks"]["data"] = remarks_data + print("[migration.load] " + str(len(remarks_data)) + " remarks data ") + # fill out storage for x in users_data: storage["users"]["by_oid"][x["_id"]] = x diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index f386e484..d44b19dc 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -176,7 +176,7 @@ async def migrate(entry, storage): await content_ratings_to_reactions(entry, shout_dict["slug"]) # shout views - await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1)) + await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1), viewer='old-discours') # del shout_dict['ratings'] storage["shouts"]["by_oid"][entry["_id"]] = shout_dict diff --git a/migration/tables/remarks.py b/migration/tables/remarks.py new file mode 100644 index 00000000..78f52c92 --- /dev/null +++ b/migration/tables/remarks.py @@ -0,0 +1,31 @@ +from base.orm import local_session +from migration.extract import extract_md +from migration.html2text import html2text +from orm.remark import Remark + + +def migrate(entry, storage): + post_oid = entry['contentItem'] + print(post_oid) + shout_dict = storage['shouts']['by_oid'].get(post_oid) + remark = { + "shout": shout_dict['id'], + "body": extract_md( + html2text(entry['body']), + entry['_id'] + ), + "desc": extract_md( + html2text( + entry['textAfter'] or '' + \ + entry['textBefore'] or '' + \ + entry['textSelected'] or '' + ), + entry["_id"] + ) + } + + with local_session() as session: + rmrk = Remark.create(**remark) + session.commit() + del rmrk["_sa_instance_state"] + return rmrk diff --git a/migration/tables/users.py b/migration/tables/users.py index 9dc45518..3404dd7c 100644 --- a/migration/tables/users.py +++ b/migration/tables/users.py @@ -1,7 +1,9 @@ +import re + +from bs4 import BeautifulSoup from dateutil.parser import parse from sqlalchemy.exc import IntegrityError -from bs4 import BeautifulSoup -import re + from base.orm import local_session from orm.user import AuthorFollower, User, UserRating @@ -108,6 +110,20 @@ def migrate(entry): return user_dict +def post_migrate(): + old_discours_dict = { + "slug": "old-discours", + "username": "old-discours", + "email": "old@discours.io", + "name": "Просмотры на старой версии сайта" + } + + with local_session() as session: + old_discours_user = User.create(**old_discours_dict) + session.add(old_discours_user) + session.commit() + + def migrate_2stage(entry, id_map): ce = 0 for rating_entry in entry.get("ratings", []): diff --git a/orm/collab.py b/orm/collab.py deleted file mode 100644 index aa71004c..00000000 --- a/orm/collab.py +++ /dev/null @@ -1,27 +0,0 @@ -from datetime import datetime - -from sqlalchemy import Column, ForeignKey, DateTime, String -from sqlalchemy.orm import relationship -from base.orm import Base -from orm.user import User - - -class CollabAuthor(Base): - __tablename__ = "collab_author" - - id = None # type: ignore - collab = Column(ForeignKey("collab.id"), primary_key=True) - author = Column(ForeignKey("user.id"), primary_key=True) - # accepted = Column(Boolean, default=False) - - -class Collab(Base): - __tablename__ = "collab" - - title = Column(String, nullable=True, comment="Title") - body = Column(String, nullable=True, comment="Body") - pic = Column(String, nullable=True, comment="Picture") - authors = relationship(lambda: User, secondary=CollabAuthor.__tablename__) - # invites = relationship(lambda: User, secondary=CollabInvited.__tablename__) - createdAt = Column(DateTime, default=datetime.now, comment="Created At") - chat = Column(String, unique=True, nullable=False) diff --git a/orm/draft.py b/orm/draft.py new file mode 100644 index 00000000..6b97e5c3 --- /dev/null +++ b/orm/draft.py @@ -0,0 +1,40 @@ +from datetime import datetime + +from sqlalchemy import Boolean, Column, ForeignKey, DateTime, String +from sqlalchemy.orm import relationship +from base.orm import Base +from orm.user import User +from orm.topic import Topic + + +class DraftTopic(Base): + __tablename__ = "draft_topic" + + id = None # type: ignore + collab = Column(ForeignKey("draft_collab.id"), primary_key=True) + topic = Column(ForeignKey("topic.id"), primary_key=True) + + +class DraftAuthor(Base): + __tablename__ = "draft_author" + + id = None # type: ignore + collab = Column(ForeignKey("draft_collab.id"), primary_key=True) + author = Column(ForeignKey("user.id"), primary_key=True) + accepted = Column(Boolean, default=False) + + +class DraftCollab(Base): + __tablename__ = "draft_collab" + + slug = Column(String, nullable=True, comment="Slug") + title = Column(String, nullable=True, comment="Title") + subtitle = Column(String, nullable=True, comment="Subtitle") + layout = Column(String, nullable=True, comment="Layout format") + body = Column(String, nullable=True, comment="Body") + cover = Column(String, nullable=True, comment="Cover") + authors = relationship(lambda: User, secondary=DraftAuthor.__tablename__) + topics = relationship(lambda: Topic, secondary=DraftTopic.__tablename__) + createdAt = Column(DateTime, default=datetime.now, comment="Created At") + updatedAt = Column(DateTime, default=datetime.now, comment="Updated At") + chat = Column(String, unique=True, nullable=True) diff --git a/orm/remark.py b/orm/remark.py new file mode 100644 index 00000000..9432a3f5 --- /dev/null +++ b/orm/remark.py @@ -0,0 +1,15 @@ +from datetime import datetime +from enum import Enum as Enumeration + +from sqlalchemy import Column, DateTime, Enum, ForeignKey, String + +from base.orm import Base + + +class Remark(Base): + + __tablename__ = "remark" + + body = Column(String, nullable=False) + desc = Column(String, default='') + shout = Column(ForeignKey("shout.id"), nullable=True, index=True, comment="Shout") diff --git a/orm/shout.py b/orm/shout.py index 55435d5f..04b0102b 100644 --- a/orm/shout.py +++ b/orm/shout.py @@ -50,7 +50,7 @@ class Shout(Base): subtitle = Column(String, nullable=True) layout = Column(String, nullable=True) mainTopic = Column(ForeignKey("topic.slug"), nullable=True) - cover = Column(String, nullable=True) + cover = Column(String, nullable=True, comment="Cover") authors = relationship(lambda: User, secondary=ShoutAuthor.__tablename__) topics = relationship(lambda: Topic, secondary=ShoutTopic.__tablename__) reactions = relationship(lambda: Reaction) diff --git a/resolvers/__init__.py b/resolvers/__init__.py index e35f8ba4..5837f767 100644 --- a/resolvers/__init__.py +++ b/resolvers/__init__.py @@ -8,7 +8,8 @@ from resolvers.auth import ( get_current_user, ) -from resolvers.create.collab import remove_coauthor, invite_coauthor +from resolvers.create.collab import load_drafts, create_draft, update_draft, delete_draft,\ + accept_coauthor, invite_coauthor from resolvers.create.migrate import markdown_body from resolvers.create.editor import create_shout, delete_shout, update_shout @@ -93,8 +94,12 @@ __all__ = [ # create.migrate "markdown_body", # create.collab + "load_drafts", + "create_draft", + "update_draft", + "delete_draft", "invite_coauthor", - "remove_coauthor", + "accept_coauthor", # zine.topics "topics_all", "topics_by_community", diff --git a/resolvers/create/collab.py b/resolvers/create/collab.py index c51c9be2..f8b1c8a7 100644 --- a/resolvers/create/collab.py +++ b/resolvers/create/collab.py @@ -3,84 +3,130 @@ from auth.credentials import AuthCredentials from base.orm import local_session from base.resolvers import query, mutation from base.exceptions import ObjectNotExist, BaseHttpException -from orm.collab import Collab, CollabAuthor +from orm.draft import DraftCollab, DraftAuthor, DraftTopic from orm.shout import Shout from orm.user import User -@query.field("getCollabs") +@query.field("loadDrafts") @login_required -async def get_collabs(_, info): +async def load_drafts(_, info): + auth: AuthCredentials = info.context["request"].auth + drafts = [] + with local_session() as session: + drafts = session.query(DraftCollab).filter(auth.user_id in DraftCollab.authors) + return drafts + + +@mutation.field("createDraft") # TODO +@login_required +async def create_draft(_, info, draft_input): auth: AuthCredentials = info.context["request"].auth with local_session() as session: - collabs = session.query(Collab).filter(auth.user_id in Collab.authors) - return collabs + collab = DraftCollab.create(**draft_input) + session.add(collab) + session.commit() - -@mutation.field("inviteCoauthor") -@login_required -async def invite_coauthor(_, info, author: int = 0, shout: int = 0): - auth: AuthCredentials = info.context["request"].auth - - with local_session() as session: - s = session.query(Shout).where(Shout.id == shout).one() - if not s: - raise ObjectNotExist("invalid shout id") - else: - c = session.query(Collab).where(Collab.shout == shout).one() - if auth.user_id not in c.authors: - raise BaseHttpException("you are not in authors list") - else: - invited_user = session.query(User).where(User.id == author).one() - c.invites.append(invited_user) - session.add(c) - session.commit() - - # TODO: email notify + # TODO: email notify to all authors return {} -@mutation.field("removeCoauthor") +@mutation.field("deleteDraft") # TODO @login_required -async def remove_coauthor(_, info, author: int = 0, shout: int = 0): +async def delete_draft(_, info, draft: int = 0): auth: AuthCredentials = info.context["request"].auth with local_session() as session: - s = session.query(Shout).where(Shout.id == shout).one() # raises Error when not found + collab = session.query(DraftCollab).where(DraftCollab.id == draft_input.id).one() if auth.user_id not in s.authors: - raise BaseHttpException("only owner can remove coauthors") + # raise BaseHttpException("only owner can remove coauthors") + return { + "error": "Only authors can update a draft" + } + elif not collab: + return { + "error": "There is no draft with this id" + } else: - c = session.query(Collab).where(Collab.shout == shout).one() - ca = session.query(CollabAuthor).join(User).where(c.shout == shout, User.id == author).one() - session.remve(ca) - c.invites = filter(lambda x: x.id == author, c.invites) - c.authors = filter(lambda x: x.id == author, c.authors) - session.add(c) + session.delete(collab) + session.commit() + return {} + + +@mutation.field("updateDraft") # TODO: draft input type +@login_required +async def update_draft(_, info, draft_input): + auth: AuthCredentials = info.context["request"].auth + + with local_session() as session: + collab = session.query(DraftCollab).where(DraftCollab.id == draft_input.id).one() # raises Error when not found + if auth.user_id not in s.authors: + # raise BaseHttpException("only owner can remove coauthors") + return { + "error": "Only authors can update draft" + } + elif not s: + return { + "error": "There is no draft with this id" + } + else: + draft_input["updatedAt"] = datetime.now(tz=timezone.utc) + collab.update(draft_input) session.commit() # TODO: email notify return {} - -@mutation.field("acceptCoauthor") +@mutation.field("inviteAuthor") @login_required -async def accept_coauthor(_, info, shout: int): +async def invite_coauthor(_, info, author: int = 0, draft: int = 0): auth: AuthCredentials = info.context["request"].auth with local_session() as session: - s = session.query(Shout).where(Shout.id == shout).one() - if not s: - raise ObjectNotExist("invalid shout id") + c = session.query(DraftCollab).where(DraftCollab.id == draft).one() + if auth.user_id not in c.authors: + # raise BaseHttpException("you are not in authors list") + return { + "error": "You are not in authors list" + } + elif c.id: + invited_user = session.query(User).where(User.id == author).one() + da = DraftAuthor.create({ + "accepted": False, + "collab": c.id, + "author": invited_user.id + }) + session.add(da) + session.commit() else: - c = session.query(Collab).where(Collab.shout == shout).one() - accepted = filter(lambda x: x.id == auth.user_id, c.invites).pop() - if accepted: - c.authors.append(accepted) - s.authors.append(accepted) - session.add(s) - session.add(c) - session.commit() - return {} - else: - raise BaseHttpException("only invited can accept") + return { + "error": "Draft is not found" + } + + # TODO: email notify + return {} + + +@mutation.field("inviteAccept") +@login_required +async def accept_coauthor(_, info, draft: int): + auth: AuthCredentials = info.context["request"].auth + + with local_session() as session: + # c = session.query(DraftCollab).where(DraftCollab.id == draft).one() + a = session.query(DraftAuthor).where(DraftAuthor.collab == draft).filter(DraftAuthor.author == auth.user_id).one() + if not a.accepted: + a.accepted = True + session.commit() + # TODO: email notify + return {} + elif a.accepted == True: + return { + "error": "You have accepted invite before" + } + else: + # raise BaseHttpException("only invited can accept") + return { + "error": "You don't have an invitation yet" + } diff --git a/resolvers/create/editor.py b/resolvers/create/editor.py index 84d744a4..a2b321ca 100644 --- a/resolvers/create/editor.py +++ b/resolvers/create/editor.py @@ -14,7 +14,7 @@ from resolvers.zine.reactions import reactions_follow, reactions_unfollow from services.zine.gittask import GitTask from resolvers.inbox.chats import create_chat from services.inbox.storage import MessagesStorage -from orm.collab import Collab +from orm.draft import DraftCollab @mutation.field("createShout") diff --git a/resolvers/zine/profile.py b/resolvers/zine/profile.py index 3b6e630f..c70106df 100644 --- a/resolvers/zine/profile.py +++ b/resolvers/zine/profile.py @@ -218,10 +218,13 @@ def author_unfollow(user_id, slug): ).first() ) if not flw: - raise Exception("[resolvers.profile] follower not exist, cant unfollow") + return { + "error": "Follower is not exist, cant unfollow" + } else: session.delete(flw) session.commit() + return {} @query.field("authorsAll") diff --git a/resolvers/zine/reactions.py b/resolvers/zine/reactions.py index 92b86935..f935920c 100644 --- a/resolvers/zine/reactions.py +++ b/resolvers/zine/reactions.py @@ -196,7 +196,7 @@ async def update_reaction(_, info, reaction={}): if not r: return {"error": "invalid reaction id"} - if r.createdBy != user.slug: + if r.createdBy != user.id: return {"error": "access denied"} r.body = reaction["body"] diff --git a/resolvers/zine/remark.py b/resolvers/zine/remark.py new file mode 100644 index 00000000..6f5f9d48 --- /dev/null +++ b/resolvers/zine/remark.py @@ -0,0 +1,48 @@ + +from datetime import datetime, timedelta, timezone +from sqlalchemy.orm import joinedload, aliased +from sqlalchemy.sql.expression import desc, asc, select, func +from base.orm import local_session +from base.resolvers import query, mutation +from base.exceptions import ObjectNotExist +from orm.remark import Remark + + +@mutation.field("createRemark") +@login_required +async def create_remark(_, info, slug, body): + auth = info.context["request"].auth + user_id = auth.user_id + with local_session() as session: + tt = Remark.create(slug=slug, body=body) + session.commit() + return + +@mutation.field("updateRemark") +@login_required +async def update_remark(_, info, slug, body = ''): + auth = info.context["request"].auth + user_id = auth.user_id + with local_session() as session: + rmrk = session.query(Remark).where(Remark.slug == slug).one() + if body: + tt.body = body + session.add(rmrk) + session.commit() + return + +@mutation.field("deleteRemark") +@login_required +async def delete_remark(_, info, slug): + auth = info.context["request"].auth + user_id = auth.user_id + with local_session() as session: + rmrk = session.query(Remark).where(Remark.slug == slug).one() + rmrk.remove() + session.commit() + return + +@query.field("loadRemark") +@login_required +async def load_remark(_, info, slug): + pass diff --git a/schema.graphql b/schema.graphql index 015f2b02..080cb371 100644 --- a/schema.graphql +++ b/schema.graphql @@ -69,6 +69,7 @@ type Result { members: [ChatMember] shout: Shout shouts: [Shout] + drafts: [DraftCollab] author: Author authors: [Author] reaction: Reaction @@ -128,6 +129,17 @@ input TopicInput { # parents: [String] } +input DraftInput { + slug: String + topics: [Int] + authors: [Int] + title: String + subtitle: String + body: String + cover: String + +} + input ReactionInput { kind: ReactionKind! shout: Int! @@ -189,10 +201,12 @@ type Mutation { updateReaction(reaction: ReactionInput!): Result! deleteReaction(reaction: Int!): Result! - # collab - inviteCoauthor(author: String!, shout: Int!): Result! - removeCoauthor(author: String!, shout: Int!): Result! - acceptCoauthor(shout: Int!): Result! + # draft / collab + createDraft(draft: DraftInput!): Result! + updateDraft(draft: DraftInput!): Result! + deleteDraft(draft: Int!): Result! + inviteAccept(draft: Int!): Result! + inviteAuthor(draft: Int!, author: Int!): Result! # following follow(what: FollowingEntity!, slug: String!): Result! @@ -289,8 +303,8 @@ type Query { authorsAll: [Author]! getAuthor(slug: String!): User - # collab - getCollabs: [Collab]! + # draft/collab + loadDrafts: [DraftCollab]! # migrate markdownBody(body: String!): String! @@ -525,10 +539,16 @@ type Chat { private: Boolean } -type Collab { - authors: [String]! - invites: [String] - shout: Shout +type DraftCollab { + slug: String + title: String + subtitle: String + body: String + cover: String + layout: String + authors: [Int]! + topics: [String] chat: Chat createdAt: Int! + updatedAt: Int } diff --git a/services/stat/viewed.py b/services/stat/viewed.py index e2168942..e1ef0a58 100644 --- a/services/stat/viewed.py +++ b/services/stat/viewed.py @@ -1,16 +1,17 @@ import asyncio import time from datetime import timedelta, timezone, datetime +from os import environ, path +from ssl import create_default_context + from gql import Client, gql from gql.transport.aiohttp import AIOHTTPTransport -from base.orm import local_session from sqlalchemy import func +from base.orm import local_session from orm import User, Topic from orm.shout import ShoutTopic, Shout from orm.viewed import ViewedEntry -from ssl import create_default_context -from os import environ, path load_facts = gql(""" query getDomains { @@ -64,7 +65,7 @@ class ViewedStorage: views = None pages = None domains = None - period = 24 * 60 * 60 # one time a day + period = 60 * 60 # every hour client = None auth_result = None disabled = False @@ -98,8 +99,8 @@ class ViewedStorage: p = page["value"].split("?")[0] slug = p.split('discours.io/')[-1] shouts[slug] = page["count"] - for slug, v in shouts: - await ViewedStorage.increment(slug, v) + for slug in shouts.keys(): + await ViewedStorage.increment(slug, shouts[slug]) except Exception: pass print("[stat.viewed] ⎪ %d pages collected " % len(shouts.keys())) @@ -164,15 +165,31 @@ class ViewedStorage: self = ViewedStorage async with self.lock: with local_session() as session: - shout = session.query(Shout).where(Shout.slug == shout_slug).one() - viewer = session.query(User).where(User.slug == viewer).one() + # TODO: user slug -> id + viewed = session.query( + ViewedEntry + ).join( + Shout + ).join( + User + ).filter( + User.slug == viewer, + Shout.slug == shout_slug + ).first() + + if viewed: + viewed.amount = amount + print("amount: %d" % amount) + else: + shout = session.query(Shout).where(Shout.slug == shout_slug).one() + viewer = session.query(User).where(User.slug == viewer).one() + new_viewed = ViewedEntry.create(**{ + "viewer": viewer.id, + "shout": shout.id, + "amount": amount + }) + session.add(new_viewed) - viewed = ViewedEntry.create(**{ - "viewer": viewer.id, - "shout": shout.id, - "amount": amount - }) - session.add(viewed) session.commit() self.by_shouts[shout_slug] = self.by_shouts.get(shout_slug, 0) + amount self.update_topics(session, shout_slug) @@ -184,25 +201,25 @@ class ViewedStorage: self = ViewedStorage if self.disabled: return - async with self.lock: - while True: - try: - print("[stat.viewed] - updating views...") - await self.update_pages() - failed = 0 - except Exception: - failed += 1 - print("[stat.viewed] - update failed #%d, wait 10 seconds" % failed) - if failed > 3: - print("[stat.viewed] - not trying to update anymore") - break - if failed == 0: - when = datetime.now(timezone.utc) + timedelta(seconds=self.period) - t = format(when.astimezone().isoformat()) - print("[stat.viewed] ⎩ next update: %s" % ( - t.split("T")[0] + " " + t.split("T")[1].split(".")[0] - )) - await asyncio.sleep(self.period) - else: - await asyncio.sleep(10) - print("[stat.viewed] - trying to update data again") + + while True: + try: + print("[stat.viewed] - updating views...") + await self.update_pages() + failed = 0 + except Exception: + failed += 1 + print("[stat.viewed] - update failed #%d, wait 10 seconds" % failed) + if failed > 3: + print("[stat.viewed] - not trying to update anymore") + break + if failed == 0: + when = datetime.now(timezone.utc) + timedelta(seconds=self.period) + t = format(when.astimezone().isoformat()) + print("[stat.viewed] ⎩ next update: %s" % ( + t.split("T")[0] + " " + t.split("T")[1].split(".")[0] + )) + await asyncio.sleep(self.period) + else: + await asyncio.sleep(10) + print("[stat.viewed] - trying to update data again")