discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

fix-migration-replyto

Browse Source
This commit is contained in:
tonyrewin 2022-11-30 22:47:34 +03:00
parent 44bd4f6ede
commit 3ed1857f11
5 changed files with 292 additions and 307 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
migration/__init__.py
View File

@ -96,16 +96,16 @@ async def shouts_handle(storage, args):
continue continue
# migrate # migrate
shout = await migrateShout(entry, storage) shout_dict = await migrateShout(entry, storage)
if shout: if shout_dict:
storage["shouts"]["by_oid"][entry["_id"]] = shout storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
storage["shouts"]["by_slug"][shout["slug"]] = shout storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict
# shouts.topics # shouts.topics
if not shout["topics"]: if not shout_dict["topics"]:
print("[migration] no topics!") print("[migration] no topics!")
# with author # with author
author: str = shout["authors"][0].dict() author = shout_dict["authors"][0]
if author["slug"] == "discours": if author["slug"] == "discours":
discours_author += 1 discours_author += 1
if author["slug"] == "anonymous": if author["slug"] == "anonymous":
@ -114,18 +114,20 @@ async def shouts_handle(storage, args):
if entry.get("published"): if entry.get("published"):
if "mdx" in args: if "mdx" in args:
export_mdx(shout) export_mdx(shout_dict)
pub_counter += 1 pub_counter += 1
# print main counter # print main counter
counter += 1 counter += 1
print('[migration] shouts_handle %d: %s @%s' % ((counter + 1), shout["slug"], author["slug"])) print('[migration] shouts_handle %d: %s @%s' % (
(counter + 1), shout_dict["slug"], author["slug"]
))
b = bs4.BeautifulSoup(shout["body"], "html.parser") b = bs4.BeautifulSoup(shout_dict["body"], "html.parser")
texts = [shout["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")]
texts = texts + b.findAll(text=True) texts = texts + b.findAll(text=True)
topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts]))
topics_dataset_tlist.append(shout["topics"]) topics_dataset_tlist.append(shout_dict["topics"])
else: else:
ignored += 1 ignored += 1
@ -133,9 +135,7 @@ async def shouts_handle(storage, args):
# ', fmt='%s') # ', fmt='%s')
print("[migration] " + str(counter) + " content items were migrated") print("[migration] " + str(counter) + " content items were migrated")
print("[migration] " + str(ignored) + " content items were ignored")
print("[migration] " + str(pub_counter) + " have been published") print("[migration] " + str(pub_counter) + " have been published")
print("[migration] " + str(discours_author) + " authored by @discours")
print("[migration] " + str(anonymous_author) + " authored by @anonymous") print("[migration] " + str(anonymous_author) + " authored by @anonymous")

286
migration/tables/comments.py
View File

@ -5,13 +5,93 @@ from dateutil.parser import parse as date_parse
from base.orm import local_session from base.orm import local_session
from migration.html2text import html2text from migration.html2text import html2text
from orm.reaction import Reaction, ReactionKind from orm.reaction import Reaction, ReactionKind
from orm.shout import ShoutReactionsFollower, Shout from orm.shout import ShoutReactionsFollower
from orm.topic import TopicFollower, Topic from orm.topic import TopicFollower
from orm.user import User from orm.user import User
from orm.shout import Shout
# from services.stat.reacted import ReactedStorage
ts = datetime.now(tz=timezone.utc) ts = datetime.now(tz=timezone.utc)
def auto_followers(session, topics, reaction_dict):
# creating shout's reactions following for reaction author
following1 = session.query(
ShoutReactionsFollower
).where(
ShoutReactionsFollower.follower == reaction_dict["createdBy"]
).filter(
ShoutReactionsFollower.shout == reaction_dict["shout"]
).first()
if not following1:
following1 = ShoutReactionsFollower.create(
follower=reaction_dict["createdBy"],
shout=reaction_dict["shout"],
auto=True
)
session.add(following1)
# creating topics followings for reaction author
for t in topics:
tf = session.query(
TopicFollower
).where(
TopicFollower.follower == reaction_dict["createdBy"]
).filter(
TopicFollower.topic == t['id']
).first()
if not tf:
topic_following = TopicFollower.create(
follower=reaction_dict["createdBy"],
topic=t['id'],
auto=True
)
session.add(topic_following)
def migrate_ratings(session, entry, reaction_dict):
for comment_rating_old in entry.get("ratings", []):
rater = (
session.query(User)
.filter(User.oid == comment_rating_old["createdBy"])
.first()
)
re_reaction_dict = {
"shout": reaction_dict["shout"],
"replyTo": reaction_dict["id"],
"kind": ReactionKind.LIKE
if comment_rating_old["value"] > 0
else ReactionKind.DISLIKE,
"createdBy": rater.id if rater else 1,
}
cts = comment_rating_old.get("createdAt")
if cts:
re_reaction_dict["createdAt"] = date_parse(cts)
try:
# creating reaction from old rating
rr = Reaction.create(**re_reaction_dict)
following2 = session.query(
ShoutReactionsFollower
).where(
ShoutReactionsFollower.follower == re_reaction_dict['createdBy']
).filter(
ShoutReactionsFollower.shout == rr.shout
).first()
if not following2:
following2 = ShoutReactionsFollower.create(
follower=re_reaction_dict['createdBy'],
shout=rr.shout,
auto=True
)
session.add(following2)
session.add(rr)
# await ReactedStorage.react(rr)
except Exception as e:
print("[migration] comment rating error: %r" % re_reaction_dict)
raise e
session.commit()
async def migrate(entry, storage): async def migrate(entry, storage):
""" """
{ {
@ -29,9 +109,7 @@ async def migrate(entry, storage):
"updatedAt": "2020-05-27 19:22:57.091000+00:00", "updatedAt": "2020-05-27 19:22:57.091000+00:00",
"updatedBy": "0" "updatedBy": "0"
} }
-> ->
type Reaction { type Reaction {
id: Int! id: Int!
shout: Shout! shout: Shout!
@ -49,10 +127,9 @@ async def migrate(entry, storage):
old_thread: String old_thread: String
} }
""" """
old_ts = entry.get("createdAt")
reaction_dict = { reaction_dict = {
"createdAt": ( "createdAt": (ts if not old_ts else date_parse(old_ts)),
ts if not entry.get("createdAt") else date_parse(entry.get("createdAt"))
),
"body": html2text(entry.get("body", "")), "body": html2text(entry.get("body", "")),
"oid": entry["_id"], "oid": entry["_id"],
} }
@ -65,140 +142,71 @@ async def migrate(entry, storage):
raise Exception raise Exception
return return
else: else:
stage = "started"
reaction = None
with local_session() as session: with local_session() as session:
author = session.query(User).filter(User.oid == entry["createdBy"]).first() author = session.query(User).filter(User.oid == entry["createdBy"]).first()
shout_dict = storage["shouts"]["by_oid"][shout_oid] old_shout = storage["shouts"]["by_oid"].get(shout_oid)
if shout_dict: if not old_shout:
shout = session.query( raise Exception("no old shout in storage")
Shout
).where(Shout.slug == shout_dict["slug"]).one()
reaction_dict["shout"] = shout.id
reaction_dict["createdBy"] = author.id if author else 1
reaction_dict["kind"] = ReactionKind.COMMENT
# creating reaction from old comment
reaction = Reaction.create(**reaction_dict)
session.add(reaction)
# creating shout's reactions following for reaction author
following1 = session.query(
ShoutReactionsFollower
).join(
User
).join(
Shout
).where(
User.id == reaction_dict["createdBy"]
).filter(
ShoutReactionsFollower.shout == reaction.shout
).first()
if not following1:
following1 = ShoutReactionsFollower.create(
follower=reaction_dict["createdBy"],
shout=reaction.shout,
auto=True
)
session.add(following1)
# creating topics followings for reaction author
for t in shout_dict["topics"]:
tf = session.query(
TopicFollower
).join(
Topic
).where(
TopicFollower.follower == reaction_dict["createdBy"]
).filter(
Topic.slug == t
).first()
if not tf:
topic = session.query(
Topic
).where(Topic.slug == t).one()
topic_following = TopicFollower.create(
follower=reaction_dict["createdBy"],
topic=topic.id,
auto=True
)
session.add(topic_following)
reaction_dict["id"] = reaction.id
for comment_rating_old in entry.get("ratings", []):
rater = (
session.query(User)
.filter(User.oid == comment_rating_old["createdBy"])
.first()
)
re_reaction_dict = {
"shout": reaction_dict["shout"],
"replyTo": reaction.id,
"kind": ReactionKind.LIKE
if comment_rating_old["value"] > 0
else ReactionKind.DISLIKE,
"createdBy": rater.id if rater else 1,
}
cts = comment_rating_old.get("createdAt")
if cts:
re_reaction_dict["createdAt"] = date_parse(cts)
try:
# creating reaction from old rating
rr = Reaction.create(**re_reaction_dict)
following2 = session.query(
ShoutReactionsFollower
).where(
ShoutReactionsFollower.follower == re_reaction_dict['createdBy']
).filter(
ShoutReactionsFollower.shout == rr.shout
).first()
if not following2:
following2 = ShoutReactionsFollower.create(
follower=re_reaction_dict['createdBy'],
shout=rr.shout,
auto=True
)
session.add(following2)
session.add(rr)
except Exception as e:
print("[migration] comment rating error: %r" % re_reaction_dict)
raise e
session.commit()
else: else:
print( stage = "author and old id found"
"[migration] error: cannot find shout for comment %r" try:
% reaction_dict shout = session.query(
) Shout
return reaction ).where(Shout.slug == old_shout["slug"]).one()
if shout:
reaction_dict["shout"] = shout.id
reaction_dict["createdBy"] = author.id if author else 1
reaction_dict["kind"] = ReactionKind.COMMENT
# creating reaction from old comment
reaction = Reaction.create(**reaction_dict)
session.add(reaction)
# session.commit()
stage = "new reaction commited"
reaction_dict = reaction.dict()
topics = [t.dict() for t in shout.topics]
auto_followers(session, topics, reaction_dict)
migrate_ratings(session, entry, reaction_dict)
return reaction
except Exception as e:
print(e)
print(reaction)
raise Exception(stage)
return
def migrate_2stage(rr, old_new_id): def migrate_2stage(old_comment, idmap):
reply_oid = rr.get("replyTo") if old_comment.get('body'):
if not reply_oid: new_id = idmap.get(old_comment.get('oid'))
return if new_id:
new_replyto_id = None
new_id = old_new_id.get(rr.get("oid")) old_replyto_id = old_comment.get("replyTo")
if not new_id: if old_replyto_id:
return new_replyto_id = int(idmap.get(old_replyto_id, "0"))
with local_session() as session:
with local_session() as session: comment = session.query(Reaction).where(Reaction.id == new_id).first()
comment = session.query(Reaction).filter(Reaction.id == new_id).first() try:
comment.replyTo = old_new_id.get(reply_oid) if new_replyto_id:
session.add(comment) new_reply = session.query(Reaction).where(Reaction.id == new_replyto_id).first()
if not new_reply:
srf = session.query(ShoutReactionsFollower).where( print(new_replyto_id)
ShoutReactionsFollower.shout == comment.shout raise Exception("cannot find reply by id!")
).filter( comment.replyTo = new_reply.id
ShoutReactionsFollower.follower == comment.createdBy session.add(comment)
).first() srf = session.query(ShoutReactionsFollower).where(
ShoutReactionsFollower.shout == comment.shout
if not srf: ).filter(
srf = ShoutReactionsFollower.create(shout=comment.shout, follower=comment.createdBy, auto=True) ShoutReactionsFollower.follower == comment.createdBy
session.add(srf) ).first()
if not srf:
session.commit() srf = ShoutReactionsFollower.create(
if not rr["body"]: shout=comment.shout, follower=comment.createdBy, auto=True
raise Exception(rr) )
session.add(srf)
session.commit()
except Exception:
raise Exception("cannot find a comment by oldid")

273
migration/tables/content_items.py
View File

@ -9,7 +9,9 @@ from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower from orm.shout import Shout, ShoutTopic, ShoutReactionsFollower
from orm.user import User from orm.user import User
from orm.topic import TopicFollower, Topic from orm.topic import TopicFollower, Topic
# from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedStorage from services.stat.viewed import ViewedStorage
import re
OLD_DATE = "2016-03-05 22:22:00.350000" OLD_DATE = "2016-03-05 22:22:00.350000"
ts = datetime.now(tz=timezone.utc) ts = datetime.now(tz=timezone.utc)
@ -21,6 +23,8 @@ type2layout = {
"Image": "image", "Image": "image",
} }
anondict = {"slug": "anonymous", "id": 1, "name": "Аноним"}
def get_shout_slug(entry): def get_shout_slug(entry):
slug = entry.get("slug", "") slug = entry.get("slug", "")
@ -29,6 +33,7 @@ def get_shout_slug(entry):
slug = friend.get("slug", "") slug = friend.get("slug", "")
if slug: if slug:
break break
slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
return slug return slug
@ -39,13 +44,8 @@ def create_author_from_app(app):
user = session.query(User).where(User.email == app['email']).first() user = session.query(User).where(User.email == app['email']).first()
if not user: if not user:
name = app.get('name') name = app.get('name')
slug = ( slug = translit(name, "ru", reversed=True).lower()
translit(name, "ru", reversed=True) slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
.replace(" ", "-")
.replace("'", "")
.replace(".", "-")
.lower()
)
# check if nameslug is used # check if nameslug is used
user = session.query(User).where(User.slug == slug).first() user = session.query(User).where(User.slug == slug).first()
# get slug from email # get slug from email
@ -74,122 +74,99 @@ def create_author_from_app(app):
session.commit() session.commit()
userdata = user.dict() userdata = user.dict()
if not userdata: if not userdata:
userdata = User.default_user.dict() userdata = User.default_user.dict() # anonymous
except Exception as e: except Exception as e:
print(app) print(app)
raise e raise e
return userdata return userdata
async def create_shout(shout_dict, userslug): async def create_shout(shout_dict, user):
s = Shout.create(**shout_dict) s = Shout.create(**shout_dict)
with local_session() as session: with local_session() as session:
follower = session.query(User).where(User.slug == userslug).one() srf = session.query(ShoutReactionsFollower).where(
srf = session.query(
ShoutReactionsFollower
).join(
User
).where(
ShoutReactionsFollower.shout == s.id ShoutReactionsFollower.shout == s.id
).filter( ).filter(
User.slug == userslug ShoutReactionsFollower.follower == user.id
).first() ).first()
if not srf: if not srf:
srf = ShoutReactionsFollower.create(shout=s.id, follower=follower.id, auto=True) srf = ShoutReactionsFollower.create(shout=s.id, follower=user.id, auto=True)
session.add(srf) session.add(srf)
session.commit() session.commit()
return s
def get_userdata(entry, storage):
user_oid = entry.get("createdBy", "")
userdata = None
app = entry.get("application")
if app:
userdata = create_author_from_app(app) or anondict
else:
userdata = storage["users"]["by_oid"].get(user_oid) or anondict
slug = userdata.get("slug")
slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
userdata["slug"] = slug
return userdata, user_oid
async def migrate(entry, storage): async def migrate(entry, storage):
# init, set title and layout userdata, user_oid = get_userdata(entry, storage)
user = await get_user(userdata, storage, user_oid)
r = { r = {
"layout": type2layout[entry["type"]], "layout": type2layout[entry["type"]],
"title": entry["title"], "title": entry["title"],
"authors": [], "authors": [userdata["slug"], ],
"topics": set([]) "slug": get_shout_slug(entry),
"cover": (
"https://assets.discours.io/unsafe/1600x/" +
entry["thumborId"] if entry.get("thumborId") else entry.get("image", {}).get("url")
),
"visibility": "public" if entry.get("published") else "authors",
"publishedAt": date_parse(entry.get("publishedAt")) if entry.get("published") else None,
"deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None,
"createdAt": date_parse(entry.get("createdAt", OLD_DATE)),
"updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts,
"topics": await add_topics_follower(entry, storage, user),
"body": extract_html(entry)
} }
# author # main topic patch
users_by_oid = storage["users"]["by_oid"] r['mainTopic'] = r['topics'][0]
user_oid = entry.get("createdBy", "")
userdata = users_by_oid.get(user_oid)
user = None
if not userdata:
app = entry.get("application")
if app:
userdata = create_author_from_app(app)
if userdata:
userslug = userdata.get('slug')
else:
userslug = "anonymous" # bad old id slug was found
r["authors"] = [userslug, ]
# slug # published author auto-confirm
slug = get_shout_slug(entry)
if slug:
r["slug"] = slug
else:
raise Exception
# cover
c = ""
if entry.get("thumborId"):
c = "https://assets.discours.io/unsafe/1600x/" + entry["thumborId"]
else:
c = entry.get("image", {}).get("url")
if not c or "cloudinary" in c:
c = ""
r["cover"] = c
# timestamps
r["createdAt"] = date_parse(entry.get("createdAt", OLD_DATE))
r["updatedAt"] = date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts
# visibility
if entry.get("published"): if entry.get("published"):
r["publishedAt"] = date_parse(entry.get("publishedAt", OLD_DATE))
r["visibility"] = "public"
with local_session() as session: with local_session() as session:
# update user.emailConfirmed if published # update user.emailConfirmed if published
author = session.query(User).where(User.slug == userslug).first() author = session.query(User).where(User.slug == userdata["slug"]).first()
author.emailConfirmed = True author.emailConfirmed = True
session.add(author) session.add(author)
session.commit() session.commit()
else:
r["visibility"] = "authors"
if "deletedAt" in entry: # media
r["deletedAt"] = date_parse(entry["deletedAt"])
# topics
r['topics'] = await add_topics_follower(entry, storage, userslug)
r['mainTopic'] = r['topics'][0]
entry["topics"] = r["topics"]
entry["cover"] = r["cover"]
# body
r["body"] = extract_html(entry)
media = extract_media(entry) media = extract_media(entry)
if media: r["media"] = json.dumps(media, ensure_ascii=True) if media else None
r["media"] = json.dumps(media, ensure_ascii=True)
# ----------------------------------- copy
shout_dict = r.copy() shout_dict = r.copy()
# user # user
user = await get_user(userslug, userdata, storage, user_oid)
shout_dict["authors"] = [user, ] shout_dict["authors"] = [user, ]
del shout_dict["topics"] del shout_dict["topics"]
try: try:
# save shout to db # save shout to db
await create_shout(shout_dict, userslug) shout_dict["oid"] = entry.get("_id", "")
shout = await create_shout(shout_dict, user)
except IntegrityError as e: except IntegrityError as e:
print(e) print('[migration] create_shout integrity error', e)
await resolve_create_shout(shout_dict, userslug) shout = await resolve_create_shout(shout_dict, userdata["slug"])
except Exception as e: except Exception as e:
raise Exception(e) raise Exception(e)
# udpate data
shout_dict = shout.dict()
shout_dict["authors"] = [user.dict(), ]
# shout topics aftermath # shout topics aftermath
shout_dict["topics"] = await topics_aftermath(r, storage) shout_dict["topics"] = await topics_aftermath(r, storage)
@ -200,13 +177,12 @@ async def migrate(entry, storage):
await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1)) await ViewedStorage.increment(shout_dict["slug"], amount=entry.get("views", 1))
# del shout_dict['ratings'] # del shout_dict['ratings']
shout_dict["oid"] = entry.get("_id", "")
storage["shouts"]["by_oid"][entry["_id"]] = shout_dict storage["shouts"]["by_oid"][entry["_id"]] = shout_dict
storage["shouts"]["by_slug"][slug] = shout_dict storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict
return shout_dict return shout_dict
async def add_topics_follower(entry, storage, userslug): async def add_topics_follower(entry, storage, user):
topics = set([]) topics = set([])
category = entry.get("category") category = entry.get("category")
topics_by_oid = storage["topics"]["by_oid"] topics_by_oid = storage["topics"]["by_oid"]
@ -218,29 +194,26 @@ async def add_topics_follower(entry, storage, userslug):
ttt = list(topics) ttt = list(topics)
# add author as TopicFollower # add author as TopicFollower
with local_session() as session: with local_session() as session:
for tpc in topics: for tpcslug in topics:
try: try:
topic = session.query(Topic).where(Topic.slug == tpc).one() tpc = session.query(Topic).where(Topic.slug == tpcslug).first()
follower = session.query(User).where(User.slug == userslug).one()
tf = session.query( tf = session.query(
TopicFollower TopicFollower
).where( ).where(
TopicFollower.follower == follower.id TopicFollower.follower == user.id
).filter( ).filter(
TopicFollower.topic == topic.id TopicFollower.topic == tpc.id
).first() ).first()
if not tf: if not tf:
tf = TopicFollower.create( tf = TopicFollower.create(
topic=topic.id, topic=tpc.id,
follower=follower.id, follower=user.id,
auto=True auto=True
) )
session.add(tf) session.add(tf)
session.commit() session.commit()
except IntegrityError: except IntegrityError:
print('[migration.shout] hidden by topic ' + tpc) print('[migration.shout] hidden by topic ' + tpc.slug)
# main topic # main topic
maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug")) maintopic = storage["replacements"].get(topics_by_oid.get(category, {}).get("slug"))
if maintopic in ttt: if maintopic in ttt:
@ -249,23 +222,28 @@ async def add_topics_follower(entry, storage, userslug):
return ttt return ttt
async def get_user(userslug, userdata, storage, oid): async def get_user(userdata, storage, oid):
user = None user = None
with local_session() as session: with local_session() as session:
if not user and userslug: uid = userdata.get("id")
user = session.query(User).filter(User.slug == userslug).first() if uid:
if not user and userdata: user = session.query(User).filter(User.id == uid).first()
elif userdata:
try: try:
userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-") slug = userdata["slug"].lower().strip()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
userdata["slug"] = slug
user = User.create(**userdata) user = User.create(**userdata)
session.add(user) session.add(user)
session.commit() session.commit()
except IntegrityError: except IntegrityError:
print("[migration] user error: " + userdata) print("[migration] user creating with slug %s" % userdata["slug"])
userdata["id"] = user.id print("[migration] from userdata: %r" % userdata)
userdata["createdAt"] = user.createdAt raise Exception("[migration] cannot create user in content_items.get_user()")
storage["users"]["by_slug"][userdata["slug"]] = userdata userdata["id"] = user.id
storage["users"]["by_oid"][oid] = userdata userdata["createdAt"] = user.createdAt
storage["users"]["by_slug"][userdata["slug"]] = userdata
storage["users"]["by_oid"][oid] = userdata
if not user: if not user:
raise Exception("could not get a user") raise Exception("could not get a user")
return user return user
@ -303,6 +281,7 @@ async def resolve_create_shout(shout_dict, userslug):
print("[migration] something went wrong with shout: \n%r" % shout_dict) print("[migration] something went wrong with shout: \n%r" % shout_dict)
raise Exception("") raise Exception("")
session.commit() session.commit()
return s
async def topics_aftermath(entry, storage): async def topics_aftermath(entry, storage):
@ -318,22 +297,22 @@ async def topics_aftermath(entry, storage):
shout_topic_old = ( shout_topic_old = (
session.query(ShoutTopic) session.query(ShoutTopic)
.join(Shout) .join(Shout)
.join(Topic) .join(Topic)
.filter(Shout.slug == entry["slug"]) .filter(Shout.slug == entry["slug"])
.filter(Topic.slug == oldslug) .filter(Topic.slug == oldslug)
.first() .first()
) )
if shout_topic_old: if shout_topic_old:
shout_topic_old.update({"topic": new_topic.id}) shout_topic_old.update({"topic": new_topic.id})
else: else:
shout_topic_new = ( shout_topic_new = (
session.query(ShoutTopic) session.query(ShoutTopic)
.join(Shout) .join(Shout)
.join(Topic) .join(Topic)
.filter(Shout.slug == entry["slug"]) .filter(Shout.slug == entry["slug"])
.filter(Topic.slug == newslug) .filter(Topic.slug == newslug)
.first() .first()
) )
if not shout_topic_new: if not shout_topic_new:
try: try:
@ -357,45 +336,37 @@ async def content_ratings_to_reactions(entry, slug):
for content_rating in entry.get("ratings", []): for content_rating in entry.get("ratings", []):
rater = ( rater = (
session.query(User) session.query(User)
.filter(User.oid == content_rating["createdBy"]) .filter(User.oid == content_rating["createdBy"])
.first() .first()
) or User.default_user
shout = session.query(Shout).where(Shout.slug == slug).first()
cts = content_rating.get("createdAt")
reaction_dict = {
"createdAt": date_parse(cts) if cts else None,
"kind": ReactionKind.LIKE
if content_rating["value"] > 0
else ReactionKind.DISLIKE,
"createdBy": rater.id,
"shout": shout.id
}
reaction = (
session.query(Reaction)
.filter(Reaction.shout == reaction_dict["shout"])
.filter(Reaction.createdBy == reaction_dict["createdBy"])
.filter(Reaction.kind == reaction_dict["kind"])
.first()
) )
reactedBy = ( if reaction:
rater k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE
if rater reaction_dict["kind"] = k
else session.query(User).filter(User.slug == "anonymous").first() reaction.update(reaction_dict)
) session.add(reaction)
if rater: else:
shout = session.query(Shout).where(Shout.slug == slug).one() rea = Reaction.create(**reaction_dict)
session.add(rea)
reaction_dict = { # await ReactedStorage.react(rea)
"kind": ReactionKind.LIKE # shout_dict['ratings'].append(reaction_dict)
if content_rating["value"] > 0
else ReactionKind.DISLIKE,
"createdBy": reactedBy.id,
"shout": shout.id,
}
cts = content_rating.get("createdAt")
if cts:
reaction_dict["createdAt"] = date_parse(cts)
reaction = (
session.query(Reaction).filter(
Reaction.shout == reaction_dict["shout"]
).filter(
Reaction.createdBy == reaction_dict["createdBy"]
).filter(
Reaction.kind == reaction_dict["kind"]
).first()
)
if reaction:
k = ReactionKind.AGREE if content_rating["value"] > 0 else ReactionKind.DISAGREE
reaction_dict["kind"] = k
reaction.update(reaction_dict)
else:
rea = Reaction.create(**reaction_dict)
session.add(rea)
# shout_dict['ratings'].append(reaction_dict)
session.commit() session.commit()
except Exception: except Exception:
raise Exception("[migration] content_item.ratings error: \n%r" % content_rating) print("[migration] content_item.ratings error: \n%r" % content_rating)

8
migration/tables/users.py
View File

@ -1,7 +1,7 @@
from dateutil.parser import parse from dateutil.parser import parse
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import re
from base.orm import local_session from base.orm import local_session
from orm.user import AuthorFollower, User, UserRating from orm.user import AuthorFollower, User, UserRating
@ -32,9 +32,9 @@ def migrate(entry):
user_dict["lastSeen"] = parse(entry["wasOnlineAt"]) user_dict["lastSeen"] = parse(entry["wasOnlineAt"])
if entry.get("profile"): if entry.get("profile"):
# slug # slug
user_dict["slug"] = ( slug = entry["profile"].get("path").lower()
entry["profile"].get("path").lower().replace(" ", "-").strip() slug = re.sub('[^0-9a-zA-Z]+', '-', slug).strip()
) user_dict["slug"] = slug
bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text bio = BeautifulSoup(entry.get("profile").get("bio") or "", features="lxml").text
if bio.startswith('<'): if bio.startswith('<'):
print('[migration] bio! ' + bio) print('[migration] bio! ' + bio)

6
schema.graphql
View File

@ -371,6 +371,12 @@ type User {
oid: String oid: String
} }
type Draft {
title: String
body: String
createdBy: Int
}
type Collab { type Collab {
authors: [String]! authors: [String]!
invites: [String] invites: [String]