discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

remove shout id

Browse Source
This commit is contained in:
knst-kotov 2021-12-13 10:50:33 +03:00
parent 1c0e977a60
commit a985e800a2
7 changed files with 79 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
migrate.py
View File

@ -123,13 +123,6 @@ def shouts(content_data, shouts_by_slug, shouts_by_oid):
try: limit = int(sys.argv[2]) if len(sys.argv) > 2 else len(content_data)
except ValueError: limit = len(content_data)
if not topics_by_cat:
with local_session() as session:
topics = session.query(Topic).all()
print("loaded %s topics" % len(topics))
for topic in topics:
topics_by_cat[topic.cat_id] = topic
for entry in content_data[:limit]:
try:
shout = migrateShout(entry, users_by_oid, topics_by_cat)
@ -202,7 +195,9 @@ def export_slug(slug, export_articles, export_authors, content_dict):
def comments(comments_data):
id_map = {}
for comment in comments_data:
comment = migrateComment(comment)
comment = migrateComment(comment, shouts_by_oid)
if not comment:
continue
id = comment.get('id')
old_id = comment.get('old_id')
id_map[old_id] = id

16
migration/tables/comments.py
View File

@ -6,7 +6,7 @@ from orm import Shout, Comment, CommentRating, User
from orm.base import local_session
from migration.html2text import html2text
def migrate(entry):
def migrate(entry, shouts_by_oid):
'''
{
"_id": "hdtwS8fSyFLxXCgSC",
@ -40,15 +40,19 @@ def migrate(entry):
views: Int
}
'''
shout_old_id = entry['contentItem']
if not shout_old_id in shouts_by_oid:
return
shout = shouts_by_oid[shout_old_id]
with local_session() as session:
shout = session.query(Shout).filter(Shout.old_id == entry['contentItem']).first()
if not shout: shout = session.query(Shout).first()
author = session.query(User).filter(User.old_id == entry['createdBy']).first()
comment_dict = {
'author': author.id if author else 0,
'createdAt': date_parse(entry['createdAt']),
'body': html2text(entry['body']),
'shout': shout.id
'shout': shout["slug"]
}
if entry.get('deleted'):
comment_dict['deletedAt'] = date_parse(entry['updatedAt'])
@ -86,7 +90,9 @@ def migrate_2stage(entry, id_map):
if not old_reply_to:
return
old_id = entry['_id']
id = id_map.get(old_id)
if not old_id in id_map:
return
id = id_map[old_id]
with local_session() as session:
comment = session.query(Comment).filter(Comment.id == id).first()
reply_to = id_map.get(old_reply_to)

9
migration/tables/content_items.py
View File

@ -133,7 +133,6 @@ def migrate(entry, users_by_oid, topics_by_oid):
body_html = str(BeautifulSoup(body_orig, features="html.parser"))
r['body'] = body_html # html2text(body_html)
body = r.get('body', '')
r['old_id'] = entry.get('_id')
# get author data
userdata = {}
@ -212,8 +211,8 @@ def migrate(entry, users_by_oid, topics_by_oid):
if rater:
shout_rating_dict = {
'value': shout_rating_old['value'],
'rater_id': rater.id,
'shout_id': s.id
'rater': rater.id,
'shout': s.slug
}
cts = shout_rating_old.get('createdAt')
if cts: shout_rating_dict['rater_id'] = date_parse(cts)
@ -221,14 +220,14 @@ def migrate(entry, users_by_oid, topics_by_oid):
except sqlalchemy.exc.IntegrityError: pass
shout_dict['ratings'].append(shout_rating_dict)
# shout topics
shout_dict['id'] = s.id
shout_dict['topics'] = []
for topic in r['topics']:
ShoutTopic.create(**{ 'shout': s.id, 'topic': topic['slug'] })
ShoutTopic.create(**{ 'shout': s.slug, 'topic': topic['slug'] })
shout_dict['topics'].append(topic['slug'])
except Exception as e:
raise e
except Exception as e:
if not shout_dict['body']: r['body'] = 'body moved'
raise e
shout_dict['old_id'] = entry.get('_id')
return shout_dict # for json

2
orm/comment.py
View File

@ -25,7 +25,7 @@ class Comment(Base):
updatedBy = Column(ForeignKey("user.id"), nullable=True, comment="Last Editor")
deletedAt = Column(DateTime, nullable=True, comment="Deleted at")
deletedBy = Column(ForeignKey("user.id"), nullable=True, comment="Deleted by")
shout: int = Column(ForeignKey("shout.id"), nullable=False, comment="Shout ID")
shout: int = Column(ForeignKey("shout.slug"), nullable=False)
replyTo: int = Column(ForeignKey("comment.id"), nullable=True, comment="comment ID")
ratings = relationship(CommentRating, foreign_keys=CommentRating.comment_id)

49
orm/shout.py
View File

@ -15,29 +15,29 @@ class ShoutAuthor(Base):
__tablename__ = "shout_author"
id = None
shout = Column(ForeignKey('shout.id'), primary_key = True)
shout = Column(ForeignKey('shout.slug'), primary_key = True)
user = Column(ForeignKey('user.id'), primary_key = True)
class ShoutViewer(Base):
__tablename__ = "shout_viewer"
id = None
shout = Column(ForeignKey('shout.id'), primary_key = True)
shout = Column(ForeignKey('shout.slug'), primary_key = True)
user = Column(ForeignKey('user.id'), primary_key = True)
class ShoutTopic(Base):
__tablename__ = 'shout_topic'
id = None
shout = Column(ForeignKey('shout.id'), primary_key = True)
shout = Column(ForeignKey('shout.slug'), primary_key = True)
topic = Column(ForeignKey('topic.slug'), primary_key = True)
class ShoutRating(Base):
__tablename__ = "shout_rating"
id = None
rater_id = Column(ForeignKey('user.id'), primary_key = True)
shout_id = Column(ForeignKey('shout.id'), primary_key = True)
rater = Column(ForeignKey('user.id'), primary_key = True)
shout = Column(ForeignKey('shout.slug'), primary_key = True)
ts = Column(DateTime, nullable=False, default = datetime.now, comment="Timestamp")
value = Column(Integer)
@ -52,16 +52,16 @@ class ShoutRatingStorage:
ShoutRatingStorage.ratings = session.query(ShoutRating).all()
@staticmethod
async def get_rating(shout_id):
async def get_rating(shout_slug):
async with ShoutRatingStorage.lock:
shout_ratings = list(filter(lambda x: x.shout_id == shout_id, ShoutRatingStorage.ratings))
shout_ratings = list(filter(lambda x: x.shout == shout_slug, ShoutRatingStorage.ratings))
return reduce((lambda x, y: x + y.value), shout_ratings, 0)
@staticmethod
async def update_rating(new_rating):
async with ShoutRatingStorage.lock:
rating = next((x for x in ShoutRatingStorage.ratings \
if x.rater_id == new_rating.rater_id and x.shout_id == new_rating.shout_id), None)
if x.rater == new_rating.rater and x.shout == new_rating.shout), None)
if rating:
rating.value = new_rating.value
rating.ts = new_rating.ts
@ -73,7 +73,7 @@ class ShoutViewByDay(Base):
__tablename__ = "shout_view_by_day"
id = None
shout_id = Column(ForeignKey('shout.id'), primary_key = True)
shout = Column(ForeignKey('shout.slug'), primary_key = True)
day = Column(DateTime, primary_key = True, default = datetime.now)
value = Column(Integer)
@ -91,28 +91,28 @@ class ShoutViewStorage:
self = ShoutViewStorage
self.views = session.query(ShoutViewByDay).all()
for view in self.views:
shout_id = view.shout_id
if not shout_id in self.this_day_views:
self.this_day_views[shout_id] = view
this_day_view = self.this_day_views[shout_id]
shout_slug = view.shout
if not shout_slug in self.this_day_views:
self.this_day_views[shout_slug] = view
this_day_view = self.this_day_views[shout_slug]
if this_day_view.day < view.day:
self.this_day_views[shout_id] = view
self.this_day_views[shout_slug] = view
@staticmethod
async def get_view(shout_id):
async def get_view(shout_slug):
async with ShoutViewStorage.lock:
shout_views = list(filter(lambda x: x.shout_id == shout_id, ShoutViewStorage.views))
shout_views = list(filter(lambda x: x.shout == shout_slug, ShoutViewStorage.views))
return reduce((lambda x, y: x + y.value), shout_views, 0)
@staticmethod
async def inc_view(shout_id):
async def inc_view(shout_slug):
self = ShoutViewStorage
async with ShoutViewStorage.lock:
this_day_view = self.this_day_views.get(shout_id)
this_day_view = self.this_day_views.get(shout_slug)
day_start = datetime.now().replace(hour = 0, minute = 0, second = 0)
if not this_day_view or this_day_view.day < day_start:
this_day_view = ShoutViewByDay.create(shout_id = shout_id, value = 1)
self.this_day_views[shout_id] = this_day_view
this_day_view = ShoutViewByDay.create(shout = shout_slug, value = 1)
self.this_day_views[shout_slug] = this_day_view
self.views.append(this_day_view)
else:
this_day_view.value = this_day_view.value + 1
@ -144,15 +144,15 @@ class ShoutViewStorage:
class Shout(Base):
__tablename__ = 'shout'
# NOTE: automatic ID here
id = None
slug: str = Column(String, nullable=False, unique=True)
slug: str = Column(String, primary_key=True)
community: int = Column(Integer, ForeignKey("community.id"), nullable=True, comment="Community")
body: str = Column(String, nullable=False, comment="Body")
createdAt: str = Column(DateTime, nullable=False, default = datetime.now, comment="Created at")
updatedAt: str = Column(DateTime, nullable=True, comment="Updated at")
replyTo: int = Column(ForeignKey("shout.id"), nullable=True)
versionOf: int = Column(ForeignKey("shout.id"), nullable=True)
replyTo: int = Column(ForeignKey("shout.slug"), nullable=True)
versionOf: int = Column(ForeignKey("shout.slug"), nullable=True)
tags: str = Column(String, nullable=True)
publishedBy: bool = Column(ForeignKey("user.id"), nullable=True)
publishedAt: str = Column(DateTime, nullable=True)
@ -164,4 +164,3 @@ class Shout(Base):
authors = relationship(lambda: User, secondary=ShoutAuthor.__tablename__) # NOTE: multiple authors
topics = relationship(lambda: Topic, secondary=ShoutTopic.__tablename__)
visibleFor = relationship(lambda: User, secondary=ShoutViewer.__tablename__)
old_id: str = Column(String, nullable = True)

58
resolvers/zine.py
View File

@ -87,8 +87,8 @@ class ShoutsCache:
shouts = []
for row in session.execute(stmt):
shout = row.Shout
shout.rating = await ShoutRatingStorage.get_rating(shout.id)
shout.views = await ShoutViewStorage.get_view(shout.id)
shout.rating = await ShoutRatingStorage.get_rating(shout.slug)
shout.views = await ShoutViewStorage.get_view(shout.slug)
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.recent_shouts = shouts
@ -101,14 +101,14 @@ class ShoutsCache:
options(selectinload(Shout.authors), selectinload(Shout.topics)).\
join(ShoutRating).\
where(Shout.publishedAt != None).\
group_by(Shout.id).\
group_by(Shout.slug).\
order_by(desc("rating")).\
limit(ShoutsCache.limit)
shouts = []
for row in session.execute(stmt):
shout = row.Shout
shout.rating = row.rating
shout.views = await ShoutViewStorage.get_view(shout.id)
shout.views = await ShoutViewStorage.get_view(shout.slug)
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.top_overall = shouts
@ -121,14 +121,14 @@ class ShoutsCache:
options(selectinload(Shout.authors), selectinload(Shout.topics)).\
join(ShoutRating).\
where(and_(Shout.createdAt > month_ago, Shout.publishedAt != None)).\
group_by(Shout.id).\
group_by(Shout.slug).\
order_by(desc("rating")).\
limit(ShoutsCache.limit)
shouts = []
for row in session.execute(stmt):
shout = row.Shout
shout.rating = row.rating
shout.views = await ShoutViewStorage.get_view(shout.id)
shout.views = await ShoutViewStorage.get_view(shout.slug)
shouts.append(shout)
async with ShoutsCache.lock:
ShoutsCache.top_month = shouts
@ -141,13 +141,13 @@ class ShoutsCache:
options(selectinload(Shout.authors), selectinload(Shout.topics)).\
join(ShoutViewByDay).\
where(and_(ShoutViewByDay.day > month_ago, Shout.publishedAt != None)).\
group_by(Shout.id).\
group_by(Shout.slug).\
order_by(desc("views")).\
limit(ShoutsCache.limit)
shouts = []
for row in session.execute(stmt):
shout = row.Shout
shout.rating = await ShoutRatingStorage.get_rating(shout.id)
shout.rating = await ShoutRatingStorage.get_rating(shout.slug)
shout.views = row.views
shouts.append(shout)
async with ShoutsCache.lock:
@ -157,13 +157,13 @@ class ShoutsCache:
async def prepare_top_authors():
month_ago = datetime.now() - timedelta(days = 30)
with local_session() as session:
shout_with_view = select(Shout.id, func.sum(ShoutViewByDay.value).label("view")).\
shout_with_view = select(Shout.slug, func.sum(ShoutViewByDay.value).label("view")).\
join(ShoutViewByDay).\
where(and_(ShoutViewByDay.day > month_ago, Shout.publishedAt != None)).\
group_by(Shout.id).\
group_by(Shout.slug).\
order_by(desc("view")).cte()
stmt = select(ShoutAuthor.user, func.sum(shout_with_view.c.view).label("view")).\
join(shout_with_view, ShoutAuthor.shout == shout_with_view.c.id).\
join(shout_with_view, ShoutAuthor.shout == shout_with_view.c.slug).\
group_by(ShoutAuthor.user).\
order_by(desc("view")).\
limit(ShoutsCache.limit)
@ -254,12 +254,12 @@ async def create_shout(_, info, input):
new_shout = Shout.create(**input)
ShoutAuthor.create(
shout = new_shout.id,
shout = new_shout.slug,
user = user_id)
for slug in topic_slugs:
topic = ShoutTopic.create(
shout = new_shout.id,
shout = new_shout.slug,
topic = slug)
new_shout.topic_slugs = topic_slugs
@ -278,13 +278,15 @@ async def create_shout(_, info, input):
@mutation.field("updateShout")
@login_required
async def update_shout(_, info, id, input):
async def update_shout(_, info, input):
auth = info.context["request"].auth
user_id = auth.user_id
slug = input["slug"]
session = local_session()
user = session.query(User).filter(User.id == user_id).first()
shout = session.query(Shout).filter(Shout.id == id).first()
shout = session.query(Shout).filter(Shout.slug == slug).first()
if not shout:
return {
@ -305,16 +307,16 @@ async def update_shout(_, info, id, input):
session.commit()
session.close()
for topic in input.get("topic_slugs"):
for topic in input.get("topic_slugs", []):
ShoutTopic.create(
shout = shout.id,
shout = slug,
topic = topic)
task = GitTask(
input,
user.username,
user.email,
"update shout %s" % (shout.slug)
"update shout %s" % (slug)
)
return {
@ -323,21 +325,21 @@ async def update_shout(_, info, id, input):
@mutation.field("rateShout")
@login_required
async def rate_shout(_, info, shout_id, value):
async def rate_shout(_, info, slug, value):
auth = info.context["request"].auth
user_id = auth.user_id
with local_session() as session:
rating = session.query(ShoutRating).\
filter(and_(ShoutRating.rater_id == user_id, ShoutRating.shout_id == shout_id)).first()
filter(and_(ShoutRating.rater == user_id, ShoutRating.shout == slug)).first()
if rating:
rating.value = value;
rating.ts = datetime.now()
session.commit()
else:
rating = ShoutRating.create(
rater_id = user_id,
shout_id = shout_id,
rater = user_id,
shout = slug,
value = value
)
@ -346,8 +348,8 @@ async def rate_shout(_, info, shout_id, value):
return {"error" : ""}
@mutation.field("viewShout")
async def view_shout(_, info, shout_id):
await ShoutViewStorage.inc_view(shout_id)
async def view_shout(_, info, slug):
await ShoutViewStorage.inc_view(slug)
return {"error" : ""}
@query.field("getShoutBySlug")
@ -360,16 +362,16 @@ async def get_shout_by_slug(_, info, slug):
shout = session.query(Shout).\
options(select_options).\
filter(Shout.slug == slug).first()
shout.rating = await ShoutRatingStorage.get_rating(shout.id)
shout.views = await ShoutViewStorage.get_view(shout.id)
shout.rating = await ShoutRatingStorage.get_rating(slug)
shout.views = await ShoutViewStorage.get_view(slug)
return shout
@query.field("getShoutComments")
async def get_shout_comments(_, info, shout_id):
async def get_shout_comments(_, info, slug):
with local_session() as session:
comments = session.query(Comment).\
options(selectinload(Comment.ratings)).\
filter(Comment.shout == shout_id).\
filter(Comment.shout == slug).\
group_by(Comment.id).all()
for comment in comments:
comment.author = await UserStorage.get_user(comment.author)

14
schema.graphql
View File

@ -109,10 +109,10 @@ type Mutation {
# shout
createShout(input: ShoutInput!): ShoutResult!
updateShout(id: Int!, input: ShoutInput!): ShoutResult!
deleteShout(id: Int!): Result!
rateShout(shout_id: Int!, value: Int!): Result!
viewShout(shout_id: Int!): Result!
updateShout(input: ShoutInput!): ShoutResult!
deleteShout(slug: String!): Result!
rateShout(slug: String!, value: Int!): Result!
viewShout(slug: String!): Result!
# user profile
# rateUser(value: Int!): Result!
@ -125,7 +125,7 @@ type Mutation {
topicSubscribe(slug: String!): Result!
topicUnsubscribe(slug: String!): Result!
createComment(body: String!, shout: Int!, replyTo: Int): CommentResult!
createComment(body: String!, shout: String!, replyTo: Int): CommentResult!
updateComment(id: Int!, body: String!): CommentResult!
deleteComment(id: Int!): Result!
rateComment(id: Int!, value: Int!): Result!
@ -157,7 +157,7 @@ type Query {
shoutsByTopic(topic: String!, limit: Int!): [Shout]!
shoutsByAuthor(author: String!, limit: Int!): [Shout]!
shoutsByCommunity(community: String!, limit: Int!): [Shout]!
getShoutComments(shout_id: Int!): [Comment]!
getShoutComments(slug: String!): [Comment]!
# mainpage
topViewed(limit: Int): [Shout]!
@ -299,7 +299,6 @@ type CommentRating {
# is publication
type Shout {
id: Int!
slug: String!
body: String!
createdAt: DateTime!
@ -323,7 +322,6 @@ type Shout {
deletedBy: Int
publishedBy: Int # if there is no published field - it is not published
publishedAt: DateTime
old_id: String
}
type Community {