Revert "Feature/lint"
This commit is contained in:
@@ -1,25 +1,24 @@
|
||||
""" cmd managed migration """
|
||||
import asyncio
|
||||
import gc
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import bs4
|
||||
|
||||
from migration.export import export_mdx
|
||||
from migration.tables.comments import migrate as migrateComment
|
||||
from migration.tables.comments import migrate_2stage as migrateComment_2stage
|
||||
from migration.tables.content_items import get_shout_slug
|
||||
from migration.tables.content_items import migrate as migrateShout
|
||||
|
||||
# from migration.tables.remarks import migrate as migrateRemark
|
||||
from migration.tables.remarks import migrate as migrateRemark
|
||||
from migration.tables.topics import migrate as migrateTopic
|
||||
from migration.tables.users import migrate as migrateUser
|
||||
from migration.tables.users import migrate as migrateUser, post_migrate as users_post_migrate
|
||||
from migration.tables.users import migrate_2stage as migrateUser_2stage
|
||||
from migration.tables.users import post_migrate as users_post_migrate
|
||||
from orm import init_tables
|
||||
from orm.reaction import Reaction
|
||||
|
||||
import asyncio
|
||||
import bs4
|
||||
import gc
|
||||
import json
|
||||
import sys
|
||||
|
||||
TODAY = datetime.strftime(datetime.now(tz=timezone.utc), "%Y%m%d")
|
||||
OLD_DATE = "2016-03-05 22:22:00.350000"
|
||||
|
||||
@@ -64,8 +63,16 @@ async def topics_handle(storage):
|
||||
del storage["topics"]["by_slug"][oldslug]
|
||||
storage["topics"]["by_oid"][oid] = storage["topics"]["by_slug"][newslug]
|
||||
print("[migration] " + str(counter) + " topics migrated")
|
||||
print("[migration] " + str(len(storage["topics"]["by_oid"].values())) + " topics by oid")
|
||||
print("[migration] " + str(len(storage["topics"]["by_slug"].values())) + " topics by slug")
|
||||
print(
|
||||
"[migration] "
|
||||
+ str(len(storage["topics"]["by_oid"].values()))
|
||||
+ " topics by oid"
|
||||
)
|
||||
print(
|
||||
"[migration] "
|
||||
+ str(len(storage["topics"]["by_slug"].values()))
|
||||
+ " topics by slug"
|
||||
)
|
||||
|
||||
|
||||
async def shouts_handle(storage, args):
|
||||
@@ -110,10 +117,9 @@ async def shouts_handle(storage, args):
|
||||
|
||||
# print main counter
|
||||
counter += 1
|
||||
print(
|
||||
"[migration] shouts_handle %d: %s @%s"
|
||||
% ((counter + 1), shout_dict["slug"], author["slug"])
|
||||
)
|
||||
print('[migration] shouts_handle %d: %s @%s' % (
|
||||
(counter + 1), shout_dict["slug"], author["slug"]
|
||||
))
|
||||
|
||||
b = bs4.BeautifulSoup(shout_dict["body"], "html.parser")
|
||||
texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")]
|
||||
@@ -132,13 +138,13 @@ async def shouts_handle(storage, args):
|
||||
print("[migration] " + str(anonymous_author) + " authored by @anonymous")
|
||||
|
||||
|
||||
# async def remarks_handle(storage):
|
||||
# print("[migration] comments")
|
||||
# c = 0
|
||||
# for entry_remark in storage["remarks"]["data"]:
|
||||
# remark = await migrateRemark(entry_remark, storage)
|
||||
# c += 1
|
||||
# print("[migration] " + str(c) + " remarks migrated")
|
||||
async def remarks_handle(storage):
|
||||
print("[migration] comments")
|
||||
c = 0
|
||||
for entry_remark in storage["remarks"]["data"]:
|
||||
remark = await migrateRemark(entry_remark, storage)
|
||||
c += 1
|
||||
print("[migration] " + str(c) + " remarks migrated")
|
||||
|
||||
|
||||
async def comments_handle(storage):
|
||||
@@ -149,9 +155,9 @@ async def comments_handle(storage):
|
||||
for oldcomment in storage["reactions"]["data"]:
|
||||
if not oldcomment.get("deleted"):
|
||||
reaction = await migrateComment(oldcomment, storage)
|
||||
if isinstance(reaction, str):
|
||||
if type(reaction) == str:
|
||||
missed_shouts[reaction] = oldcomment
|
||||
elif isinstance(reaction, Reaction):
|
||||
elif type(reaction) == Reaction:
|
||||
reaction = reaction.dict()
|
||||
rid = reaction["id"]
|
||||
oid = reaction["oid"]
|
||||
@@ -208,7 +214,9 @@ def data_load():
|
||||
tags_data = json.loads(open("migration/data/tags.json").read())
|
||||
storage["topics"]["tags"] = tags_data
|
||||
print("[migration.load] " + str(len(tags_data)) + " tags ")
|
||||
cats_data = json.loads(open("migration/data/content_item_categories.json").read())
|
||||
cats_data = json.loads(
|
||||
open("migration/data/content_item_categories.json").read()
|
||||
)
|
||||
storage["topics"]["cats"] = cats_data
|
||||
print("[migration.load] " + str(len(cats_data)) + " cats ")
|
||||
comments_data = json.loads(open("migration/data/comments.json").read())
|
||||
@@ -227,7 +235,11 @@ def data_load():
|
||||
storage["users"]["by_oid"][x["_id"]] = x
|
||||
# storage['users']['by_slug'][x['slug']] = x
|
||||
# no user.slug yet
|
||||
print("[migration.load] " + str(len(storage["users"]["by_oid"].keys())) + " users by oid")
|
||||
print(
|
||||
"[migration.load] "
|
||||
+ str(len(storage["users"]["by_oid"].keys()))
|
||||
+ " users by oid"
|
||||
)
|
||||
for x in tags_data:
|
||||
storage["topics"]["by_oid"][x["_id"]] = x
|
||||
storage["topics"]["by_slug"][x["slug"]] = x
|
||||
@@ -235,7 +247,9 @@ def data_load():
|
||||
storage["topics"]["by_oid"][x["_id"]] = x
|
||||
storage["topics"]["by_slug"][x["slug"]] = x
|
||||
print(
|
||||
"[migration.load] " + str(len(storage["topics"]["by_slug"].keys())) + " topics by slug"
|
||||
"[migration.load] "
|
||||
+ str(len(storage["topics"]["by_slug"].keys()))
|
||||
+ " topics by slug"
|
||||
)
|
||||
for item in content_data:
|
||||
slug = get_shout_slug(item)
|
||||
|
Reference in New Issue
Block a user