discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

migration, auth, refactoring, formatting

Browse Source
This commit is contained in:
tonyrewin
2022-09-17 21:12:14 +03:00
parent 6b4c00d9e7
commit 3136eecd7e
68 changed files with 968 additions and 930 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
migration/tables/comments.py
View File

@@ -1,8 +1,10 @@
from datetime import datetime
from dateutil.parser import parse as date_parse
from orm import Reaction, User
from base.orm import local_session
from migration.html2text import html2text
from orm import Reaction, User
from orm.reaction import ReactionKind
from services.stat.reacted import ReactedStorage
@@ -46,16 +48,13 @@ async def migrate(entry, storage):
old_thread: String
}
"""
reaction_dict = {}
reaction_dict["createdAt"] = (
ts if not entry.get("createdAt") else date_parse(entry.get("createdAt"))
)
print("[migration] reaction original date %r" % entry.get("createdAt"))
# print('[migration] comment date %r ' % comment_dict['createdAt'])
reaction_dict["body"] = html2text(entry.get("body", ""))
reaction_dict["oid"] = entry["_id"]
if entry.get("createdAt"):
reaction_dict["createdAt"] = date_parse(entry.get("createdAt"))
reaction_dict = {
"createdAt": (
ts if not entry.get("createdAt") else date_parse(entry.get("createdAt"))
),
"body": html2text(entry.get("body", "")),
"oid": entry["_id"],
}
shout_oid = entry.get("contentItem")
if shout_oid not in storage["shouts"]["by_oid"]:
if len(storage["shouts"]["by_oid"]) > 0:
@@ -126,7 +125,7 @@ def migrate_2stage(rr, old_new_id):
with local_session() as session:
comment = session.query(Reaction).filter(Reaction.id == new_id).first()
comment.replyTo = old_new_id.get(reply_oid)
comment.save()
session.add(comment)
session.commit()
if not rr["body"]:
raise Exception(rr)

51
migration/tables/content_items.py
View File

@@ -1,14 +1,18 @@
from dateutil.parser import parse as date_parse
import sqlalchemy
from orm.shout import Shout, ShoutTopic, User
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedByDay
from transliterate import translit
from datetime import datetime
from dateutil.parser import parse as date_parse
from sqlalchemy.exc import IntegrityError
from transliterate import translit
from base.orm import local_session
from migration.extract import prepare_html_body
from orm.community import Community
from orm.reaction import Reaction, ReactionKind
from orm.shout import Shout, ShoutTopic, User
from orm.topic import TopicFollower
from services.stat.reacted import ReactedStorage
from services.stat.viewed import ViewedByDay
from services.zine.topics import TopicStorage
OLD_DATE = "2016-03-05 22:22:00.350000"
ts = datetime.now()
@@ -72,7 +76,10 @@ async def migrate(entry, storage):
}
else:
userdata = User.default_user.dict()
assert userdata, "no user found for %s from %d" % [oid, len(users_by_oid.keys())]
if not userdata:
raise Exception(
"no user found for %s from %d" % [oid, len(users_by_oid.keys())]
)
r["authors"] = [
userdata,
]
@@ -139,32 +146,40 @@ async def migrate(entry, storage):
# del shout_dict['rating'] # NOTE: TypeError: 'rating' is an invalid keyword argument for Shout
# del shout_dict['ratings']
email = userdata.get("email")
slug = userdata.get("slug")
if not slug:
userslug = userdata.get("slug")
if not userslug:
raise Exception
with local_session() as session:
# c = session.query(Community).all().pop()
if email:
user = session.query(User).filter(User.email == email).first()
if not user and slug:
user = session.query(User).filter(User.slug == slug).first()
if not user and userslug:
user = session.query(User).filter(User.slug == userslug).first()
if not user and userdata:
try:
userdata["slug"] = userdata["slug"].lower().strip().replace(" ", "-")
user = User.create(**userdata)
except sqlalchemy.exc.IntegrityError:
except IntegrityError:
print("[migration] user error: " + userdata)
userdata["id"] = user.id
userdata["createdAt"] = user.createdAt
storage["users"]["by_slug"][userdata["slug"]] = userdata
storage["users"]["by_oid"][entry["_id"]] = userdata
assert user, "could not get a user"
shout_dict["authors"] = [user, ]
if not user:
raise Exception("could not get a user")
shout_dict["authors"] = [
user,
]
# TODO: subscribe shout user on shout topics
try:
s = Shout.create(**shout_dict)
except sqlalchemy.exc.IntegrityError as e:
with local_session() as session:
topics = session.query(ShoutTopic).where(ShoutTopic.shout == s.slug).all()
for tpc in topics:
TopicFollower.create(topic=tpc.slug, follower=userslug)
await TopicStorage.update_topic(tpc.slug)
except IntegrityError as e:
with local_session() as session:
s = session.query(Shout).filter(Shout.slug == shout_dict["slug"]).first()
bump = False
@@ -267,9 +282,9 @@ async def migrate(entry, storage):
)
reaction.update(reaction_dict)
else:
reaction_dict["day"] = (
reaction_dict.get("createdAt") or ts
).replace(hour=0, minute=0, second=0, microsecond=0)
# day = (
# reaction_dict.get("createdAt") or ts
# ).replace(hour=0, minute=0, second=0, microsecond=0)
rea = Reaction.create(**reaction_dict)
await ReactedStorage.react(rea)
# shout_dict['ratings'].append(reaction_dict)

36
migration/tables/replacements.json
View File

@@ -764,5 +764,37 @@
"blocked-in-russia": "blocked-in-russia",
"kavarga": "kavarga",
"galereya-anna-nova": "gallery-anna-nova",
"derrida": "derrida"
}
"derrida": "derrida",
"dinozavry": "dinosaurs",
"beecake": "beecake",
"literaturnyykaver": "literature-cover",
"dialog": "dialogue",
"dozhd": "rain",
"pomosch": "help",
"igra": "game",
"reportazh-1": "reportage",
"armiya-1": "army",
"ukraina-2": "ukraine",
"nasilie-1": "violence",
"smert-1": "death",
"dnevnik-1": "dairy",
"voyna-na-ukraine": "war-in-ukraine",
"zabota": "care",
"ango": "ango",
"hayku": "haiku",
"utrata": "loss",
"pokoy": "peace",
"kladbische": "cemetery",
"lomonosov": "lomonosov",
"istoriya-nauki": "history-of-sceince",
"sud": "court",
"russkaya-toska": "russian-toska",
"duh": "spirit",
"devyanostye": "90s",
"seksualnoe-nasilie": "sexual-violence",
"gruziya-2": "georgia",
"dokumentalnaya-poeziya": "documentary-poetry",
"kriptovalyuty": "cryptocurrencies",
"magiya": "magic",
"yazychestvo": "paganism"
}

2
migration/tables/topics.py
View File

@@ -1,5 +1,5 @@
from migration.extract import extract_md, html2text
from base.orm import local_session
from migration.extract import extract_md, html2text
from orm import Topic, Community

36
migration/tables/users.py
View File

@@ -1,8 +1,9 @@
import sqlalchemy
from dateutil.parser import parse
from sqlalchemy.exc import IntegrityError
from base.orm import local_session
from migration.html2text import html2text
from orm import User, UserRating
from dateutil.parser import parse
from base.orm import local_session
def migrate(entry):
@@ -21,9 +22,6 @@ def migrate(entry):
"muted": False, # amnesty
"bio": entry["profile"].get("bio", ""),
"notifications": [],
"createdAt": parse(entry["createdAt"]),
"roles": [], # entry['roles'] # roles by community
"ratings": [], # entry['ratings']
"links": [],
"name": "anonymous",
}
@@ -86,7 +84,7 @@ def migrate(entry):
user_dict["slug"] = user_dict["slug"].lower().strip().replace(" ", "-")
try:
user = User.create(**user_dict.copy())
except sqlalchemy.exc.IntegrityError:
except IntegrityError:
print("[migration] cannot create user " + user_dict["slug"])
with local_session() as session:
old_user = (
@@ -120,28 +118,10 @@ def migrate_2stage(entry, id_map):
with local_session() as session:
try:
user_rating = UserRating.create(**user_rating_dict)
except sqlalchemy.exc.IntegrityError:
old_rating = (
session.query(UserRating)
.filter(UserRating.rater == rater_slug)
.first()
)
print(
"[migration] cannot create "
+ author_slug
+ "`s rate from "
+ rater_slug
)
print(
"[migration] concat rating value %d+%d=%d"
% (
old_rating.value,
rating_entry["value"],
old_rating.value + rating_entry["value"],
)
)
old_rating.update({"value": old_rating.value + rating_entry["value"]})
session.add(user_rating)
session.commit()
except IntegrityError:
print("[migration] cannot rate " + author_slug + "`s by " + rater_slug)
except Exception as e:
print(e)
return ce