discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

fix shout migration

Browse Source
This commit is contained in:
tonyrewin 2022-12-13 15:01:42 +03:00
parent 3ed8d57bd6
commit 00eb239fff
1 changed files with 86 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
migration/tables/content_items.py
View File

@ -23,6 +23,7 @@ type2layout = {
} }
anondict = {"slug": "anonymous", "id": 1, "name": "Аноним"} anondict = {"slug": "anonymous", "id": 1, "name": "Аноним"}
discours = {"slug": "discours", "id": 2, "name": "Дискурс"}
def get_shout_slug(entry): def get_shout_slug(entry):
@ -37,85 +38,90 @@ def get_shout_slug(entry):
def create_author_from_app(app): def create_author_from_app(app):
try: userdata = None
with local_session() as session: if app:
# check if email is used try:
user = session.query(User).where(User.email == app['email']).first() with local_session() as session:
if not user: # check if email is used
name = app.get('name') user = session.query(User).where(User.email == app['email']).first()
slug = translit(name, "ru", reversed=True).lower()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
# check if nameslug is used
user = session.query(User).where(User.slug == slug).first()
# get slug from email
if user:
slug = app['email'].split('@')[0]
user = session.query(User).where(User.slug == slug).first()
# one more try
if user:
slug += '-author'
user = session.query(User).where(User.slug == slug).first()
# create user with application data
if not user: if not user:
userdata = { print('[migration] creating user...')
"username": app["email"], name = app.get('name')
"email": app["email"], slug = translit(name, "ru", reversed=True).lower()
"name": app.get("name", ""), slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
"bio": app.get("bio", ""), # check if nameslug is used
"emailConfirmed": False, user = session.query(User).where(User.slug == slug).first()
"slug": slug, # get slug from email
"createdAt": ts, if user:
"lastSeen": ts, slug = app['email'].split('@')[0]
} user = session.query(User).where(User.slug == slug).first()
user = User.create(**userdata) # one more try
session.add(user) if user:
session.commit() slug += '-author'
userdata = user.dict() user = session.query(User).where(User.slug == slug).first()
if not userdata:
userdata = User.default_user.dict() # anonymous # create user with application data
except Exception as e: if not user:
print(app) userdata = {
raise e "username": app["email"],
return userdata "email": app["email"],
"name": app.get("name", ""),
"bio": app.get("bio", ""),
"emailConfirmed": False,
"slug": slug,
"createdAt": ts,
"lastSeen": ts,
}
user = User.create(**userdata)
session.add(user)
session.commit()
userdata = user.dict()
except Exception as e:
print(app)
raise e
return userdata
async def create_shout(shout_dict, user): async def create_shout(shout_dict):
s = Shout.create(**shout_dict) s = Shout.create(**shout_dict)
author = shout_dict['authors'][0]
with local_session() as session: with local_session() as session:
srf = session.query(ShoutReactionsFollower).where( srf = session.query(ShoutReactionsFollower).where(
ShoutReactionsFollower.shout == s.id ShoutReactionsFollower.shout == s.id
).filter( ).filter(
ShoutReactionsFollower.follower == user.id ShoutReactionsFollower.follower == author.id
).first() ).first()
if not srf: if not srf:
srf = ShoutReactionsFollower.create(shout=s.id, follower=user.id, auto=True) srf = ShoutReactionsFollower.create(shout=s.id, follower=author.id, auto=True)
session.add(srf) session.add(srf)
session.commit() session.commit()
return s return s
def get_userdata(entry, storage): async def get_user(entry, storage):
user_oid = entry.get("createdBy", "") user_oid = entry.get("createdBy")
userdata = None userdata = storage["users"]["by_oid"].get(user_oid)
app = entry.get("application") if not userdata:
if app: userdata = create_author_from_app(entry.get("application"))
userdata = create_author_from_app(app) or anondict print("[migration] from user_oid")
else: if not userdata:
userdata = storage["users"]["by_oid"].get(user_oid) or anondict print("[migration] no app no user_oid")
slug = userdata.get("slug") userdata = anondict
# cleanup slug
slug = userdata.get("slug", "")
slug = re.sub('[^0-9a-zA-Z]+', '-', slug) slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
userdata["slug"] = slug userdata["slug"] = slug
return userdata, user_oid
user = await process_user(userdata, storage, user_oid)
return user, user_oid
async def migrate(entry, storage): async def migrate(entry, storage):
userdata, user_oid = get_userdata(entry, storage) author, user_oid = await get_user(entry, storage)
user = await get_user(userdata, storage, user_oid)
r = { r = {
"layout": type2layout[entry["type"]], "layout": type2layout[entry["type"]],
"title": entry["title"], "title": entry["title"],
"authors": [userdata["slug"], ], "authors": [author, ],
"slug": get_shout_slug(entry), "slug": get_shout_slug(entry),
"cover": ( "cover": (
"https://assets.discours.io/unsafe/1600x/" + "https://assets.discours.io/unsafe/1600x/" +
@ -126,7 +132,7 @@ async def migrate(entry, storage):
"deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None, "deletedAt": date_parse(entry.get("deletedAt")) if entry.get("deletedAt") else None,
"createdAt": date_parse(entry.get("createdAt", OLD_DATE)), "createdAt": date_parse(entry.get("createdAt", OLD_DATE)),
"updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts, "updatedAt": date_parse(entry["updatedAt"]) if "updatedAt" in entry else ts,
"topics": await add_topics_follower(entry, storage, user), "topics": await add_topics_follower(entry, storage, author),
"body": extract_html(entry) "body": extract_html(entry)
} }
@ -137,7 +143,6 @@ async def migrate(entry, storage):
if entry.get("published"): if entry.get("published"):
with local_session() as session: with local_session() as session:
# update user.emailConfirmed if published # update user.emailConfirmed if published
author = session.query(User).where(User.slug == userdata["slug"]).first()
author.emailConfirmed = True author.emailConfirmed = True
session.add(author) session.add(author)
session.commit() session.commit()
@ -148,23 +153,21 @@ async def migrate(entry, storage):
# ----------------------------------- copy # ----------------------------------- copy
shout_dict = r.copy() shout_dict = r.copy()
# user
shout_dict["authors"] = [user, ]
del shout_dict["topics"] del shout_dict["topics"]
try: try:
# save shout to db # save shout to db
shout_dict["oid"] = entry.get("_id", "") shout_dict["oid"] = entry.get("_id", "")
shout = await create_shout(shout_dict, user) shout = await create_shout(shout_dict)
except IntegrityError as e: except IntegrityError as e:
print('[migration] create_shout integrity error', e) print('[migration] create_shout integrity error', e)
shout = await resolve_create_shout(shout_dict, userdata["slug"]) shout = await resolve_create_shout(shout_dict)
except Exception as e: except Exception as e:
raise Exception(e) raise Exception(e)
# udpate data # udpate data
shout_dict = shout.dict() shout_dict = shout.dict()
shout_dict["authors"] = [user.dict(), ] shout_dict["authors"] = [author.dict(), ]
# shout topics aftermath # shout topics aftermath
shout_dict["topics"] = await topics_aftermath(r, storage) shout_dict["topics"] = await topics_aftermath(r, storage)
@ -222,13 +225,11 @@ async def add_topics_follower(entry, storage, user):
return ttt return ttt
async def get_user(userdata, storage, oid): async def process_user(userdata, storage, oid):
user = None
with local_session() as session: with local_session() as session:
uid = userdata.get("id") uid = userdata.get("id", 1) # anonymous as
if uid: user = session.query(User).filter(User.id == uid).first()
user = session.query(User).filter(User.id == uid).first() if not user:
elif userdata:
try: try:
slug = userdata["slug"].lower().strip() slug = userdata["slug"].lower().strip()
slug = re.sub('[^0-9a-zA-Z]+', '-', slug) slug = re.sub('[^0-9a-zA-Z]+', '-', slug)
@ -237,28 +238,33 @@ async def get_user(userdata, storage, oid):
session.add(user) session.add(user)
session.commit() session.commit()
except IntegrityError: except IntegrityError:
print("[migration] user creating with slug %s" % userdata["slug"]) print(f"[migration] user creating with slug {userdata['slug']}")
print("[migration] from userdata: %r" % userdata) print("[migration] from userdata")
print(userdata)
raise Exception("[migration] cannot create user in content_items.get_user()") raise Exception("[migration] cannot create user in content_items.get_user()")
if user.id == 946:
print("[migration] ***************** ALPINA")
if user.id == 2:
print("[migration] +++++++++++++++++ DISCOURS")
userdata["id"] = user.id userdata["id"] = user.id
userdata["createdAt"] = user.createdAt userdata["createdAt"] = user.createdAt
storage["users"]["by_slug"][userdata["slug"]] = userdata storage["users"]["by_slug"][userdata["slug"]] = userdata
storage["users"]["by_oid"][oid] = userdata storage["users"]["by_oid"][oid] = userdata
if not user: if not user:
raise Exception("could not get a user") raise Exception("could not get a user")
return user return user
async def resolve_create_shout(shout_dict, userslug): async def resolve_create_shout(shout_dict):
with local_session() as session: with local_session() as session:
s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first() s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
bump = False bump = False
if s: if s:
if s.authors[0] != userslug: if s.createdAt != shout_dict['createdAt']:
# create new with different slug # create new with different slug
shout_dict["slug"] += '-' + shout_dict["layout"] shout_dict["slug"] += '-' + shout_dict["layout"]
try: try:
await create_shout(shout_dict, userslug) await create_shout(shout_dict)
except IntegrityError as e: except IntegrityError as e:
print(e) print(e)
bump = True bump = True
@ -292,8 +298,8 @@ async def topics_aftermath(entry, storage):
if newslug: if newslug:
with local_session() as session: with local_session() as session:
shout = session.query(Shout).where(Shout.slug == entry["slug"]).one() shout = session.query(Shout).where(Shout.slug == entry["slug"]).first()
new_topic = session.query(Topic).where(Topic.slug == newslug).one() new_topic = session.query(Topic).where(Topic.slug == newslug).first()
shout_topic_old = ( shout_topic_old = (
session.query(ShoutTopic) session.query(ShoutTopic)