From 5fedd007c7ac2cbd1e64a5340503d9580233893b Mon Sep 17 00:00:00 2001 From: Tony Rewin Date: Thu, 5 Oct 2023 23:18:06 +0300 Subject: [PATCH] git-dep3 --- migration/__init__.py | 292 -------- migration/bson2json.py | 32 - migration/data/.gitkeep | 0 migration/export.py | 159 ----- migration/extract.py | 434 ------------ migration/html2text/__init__.py | 1054 ---------------------------- migration/html2text/__main__.py | 3 - migration/html2text/cli.py | 323 --------- migration/html2text/config.py | 164 ----- migration/html2text/elements.py | 18 - migration/html2text/py.typed | 0 migration/html2text/typing.py | 3 - migration/html2text/utils.py | 287 -------- migration/tables/__init__.py | 1 - migration/tables/comments.py | 209 ------ migration/tables/content_items.py | 420 ----------- migration/tables/remarks.py | 34 - migration/tables/replacements.json | 828 ---------------------- migration/tables/topics.py | 32 - migration/tables/users.py | 167 ----- migration/utils.py | 10 - requirements.txt | 5 +- services/viewed.py | 21 +- 23 files changed, 8 insertions(+), 4488 deletions(-) delete mode 100644 migration/__init__.py delete mode 100644 migration/bson2json.py delete mode 100644 migration/data/.gitkeep delete mode 100644 migration/export.py delete mode 100644 migration/extract.py delete mode 100644 migration/html2text/__init__.py delete mode 100644 migration/html2text/__main__.py delete mode 100644 migration/html2text/cli.py delete mode 100644 migration/html2text/config.py delete mode 100644 migration/html2text/elements.py delete mode 100644 migration/html2text/py.typed delete mode 100644 migration/html2text/typing.py delete mode 100644 migration/html2text/utils.py delete mode 100644 migration/tables/__init__.py delete mode 100644 migration/tables/comments.py delete mode 100644 migration/tables/content_items.py delete mode 100644 migration/tables/remarks.py delete mode 100644 migration/tables/replacements.json delete mode 100644 migration/tables/topics.py delete mode 100644 migration/tables/users.py delete mode 100644 migration/utils.py diff --git a/migration/__init__.py b/migration/__init__.py deleted file mode 100644 index 468fa886..00000000 --- a/migration/__init__.py +++ /dev/null @@ -1,292 +0,0 @@ -""" cmd managed migration """ -import asyncio -import gc -import json -import sys -from datetime import datetime, timezone - -import bs4 - -from migration.export import export_mdx -from migration.tables.comments import migrate as migrateComment -from migration.tables.comments import migrate_2stage as migrateComment_2stage -from migration.tables.content_items import get_shout_slug -from migration.tables.content_items import migrate as migrateShout -from migration.tables.remarks import migrate as migrateRemark -from migration.tables.topics import migrate as migrateTopic -from migration.tables.users import migrate as migrateUser, post_migrate as users_post_migrate -from migration.tables.users import migrate_2stage as migrateUser_2stage -from orm import init_tables -from orm.reaction import Reaction - -TODAY = datetime.strftime(datetime.now(tz=timezone.utc), "%Y%m%d") -OLD_DATE = "2016-03-05 22:22:00.350000" - - -async def users_handle(storage): - """migrating users first""" - counter = 0 - id_map = {} - print("[migration] migrating %d users" % (len(storage["users"]["data"]))) - for entry in storage["users"]["data"]: - oid = entry["_id"] - user = migrateUser(entry) - storage["users"]["by_oid"][oid] = user # full - del user["password"] - del user["emailConfirmed"] - del user["username"] - del user["email"] - storage["users"]["by_slug"][user["slug"]] = user # public - id_map[user["oid"]] = user["slug"] - counter += 1 - ce = 0 - for entry in storage["users"]["data"]: - ce += migrateUser_2stage(entry, id_map) - users_post_migrate() - - -async def topics_handle(storage): - """topics from categories and tags""" - counter = 0 - for t in storage["topics"]["tags"] + storage["topics"]["cats"]: - if t["slug"] in storage["replacements"]: - t["slug"] = storage["replacements"][t["slug"]] - topic = migrateTopic(t) - storage["topics"]["by_oid"][t["_id"]] = topic - storage["topics"]["by_slug"][t["slug"]] = topic - counter += 1 - else: - print("[migration] topic " + t["slug"] + " ignored") - for oldslug, newslug in storage["replacements"].items(): - if oldslug != newslug and oldslug in storage["topics"]["by_slug"]: - oid = storage["topics"]["by_slug"][oldslug]["_id"] - del storage["topics"]["by_slug"][oldslug] - storage["topics"]["by_oid"][oid] = storage["topics"]["by_slug"][newslug] - print("[migration] " + str(counter) + " topics migrated") - print( - "[migration] " - + str(len(storage["topics"]["by_oid"].values())) - + " topics by oid" - ) - print( - "[migration] " - + str(len(storage["topics"]["by_slug"].values())) - + " topics by slug" - ) - - -async def shouts_handle(storage, args): - """migrating content items one by one""" - counter = 0 - discours_author = 0 - anonymous_author = 0 - pub_counter = 0 - ignored = 0 - topics_dataset_bodies = [] - topics_dataset_tlist = [] - for entry in storage["shouts"]["data"]: - gc.collect() - # slug - slug = get_shout_slug(entry) - - # single slug mode - if "-" in args and slug not in args: - continue - - # migrate - shout_dict = await migrateShout(entry, storage) - if shout_dict: - storage["shouts"]["by_oid"][entry["_id"]] = shout_dict - storage["shouts"]["by_slug"][shout_dict["slug"]] = shout_dict - # shouts.topics - if not shout_dict["topics"]: - print("[migration] no topics!") - - # with author - author = shout_dict["authors"][0] - if author["slug"] == "discours": - discours_author += 1 - if author["slug"] == "anonymous": - anonymous_author += 1 - # print('[migration] ' + shout['slug'] + ' with author ' + author) - - if entry.get("published"): - if "mdx" in args: - export_mdx(shout_dict) - pub_counter += 1 - - # print main counter - counter += 1 - print('[migration] shouts_handle %d: %s @%s' % ( - (counter + 1), shout_dict["slug"], author["slug"] - )) - - b = bs4.BeautifulSoup(shout_dict["body"], "html.parser") - texts = [shout_dict["title"].lower().replace(r"[^а-яА-Яa-zA-Z]", "")] - texts = texts + b.findAll(text=True) - topics_dataset_bodies.append(" ".join([x.strip().lower() for x in texts])) - topics_dataset_tlist.append(shout_dict["topics"]) - else: - ignored += 1 - - # np.savetxt('topics_dataset.csv', (topics_dataset_bodies, topics_dataset_tlist), delimiter=', - # ', fmt='%s') - - print("[migration] " + str(counter) + " content items were migrated") - print("[migration] " + str(pub_counter) + " have been published") - print("[migration] " + str(discours_author) + " authored by @discours") - print("[migration] " + str(anonymous_author) + " authored by @anonymous") - - -async def remarks_handle(storage): - print("[migration] comments") - c = 0 - for entry_remark in storage["remarks"]["data"]: - remark = await migrateRemark(entry_remark, storage) - c += 1 - print("[migration] " + str(c) + " remarks migrated") - - -async def comments_handle(storage): - print("[migration] comments") - id_map = {} - ignored_counter = 0 - missed_shouts = {} - for oldcomment in storage["reactions"]["data"]: - if not oldcomment.get("deleted"): - reaction = await migrateComment(oldcomment, storage) - if type(reaction) == str: - missed_shouts[reaction] = oldcomment - elif type(reaction) == Reaction: - reaction = reaction.dict() - rid = reaction["id"] - oid = reaction["oid"] - id_map[oid] = rid - else: - ignored_counter += 1 - - for reaction in storage["reactions"]["data"]: - migrateComment_2stage(reaction, id_map) - print("[migration] " + str(len(id_map)) + " comments migrated") - print("[migration] " + str(ignored_counter) + " comments ignored") - print("[migration] " + str(len(missed_shouts.keys())) + " commented shouts missed") - missed_counter = 0 - for missed in missed_shouts.values(): - missed_counter += len(missed) - print("[migration] " + str(missed_counter) + " comments dropped") - - -async def all_handle(storage, args): - print("[migration] handle everything") - await users_handle(storage) - await topics_handle(storage) - print("[migration] users and topics are migrated") - await shouts_handle(storage, args) - # print("[migration] remarks...") - # await remarks_handle(storage) - print("[migration] migrating comments") - await comments_handle(storage) - # export_email_subscriptions() - print("[migration] done!") - - -def data_load(): - storage = { - "content_items": { - "by_oid": {}, - "by_slug": {}, - }, - "shouts": {"by_oid": {}, "by_slug": {}, "data": []}, - "reactions": {"by_oid": {}, "by_slug": {}, "by_content": {}, "data": []}, - "topics": { - "by_oid": {}, - "by_slug": {}, - "cats": [], - "tags": [], - }, - "remarks": {"data": []}, - "users": {"by_oid": {}, "by_slug": {}, "data": []}, - "replacements": json.loads(open("migration/tables/replacements.json").read()), - } - try: - users_data = json.loads(open("migration/data/users.json").read()) - print("[migration.load] " + str(len(users_data)) + " users ") - tags_data = json.loads(open("migration/data/tags.json").read()) - storage["topics"]["tags"] = tags_data - print("[migration.load] " + str(len(tags_data)) + " tags ") - cats_data = json.loads( - open("migration/data/content_item_categories.json").read() - ) - storage["topics"]["cats"] = cats_data - print("[migration.load] " + str(len(cats_data)) + " cats ") - comments_data = json.loads(open("migration/data/comments.json").read()) - storage["reactions"]["data"] = comments_data - print("[migration.load] " + str(len(comments_data)) + " comments ") - content_data = json.loads(open("migration/data/content_items.json").read()) - storage["shouts"]["data"] = content_data - print("[migration.load] " + str(len(content_data)) + " content items ") - - remarks_data = json.loads(open("migration/data/remarks.json").read()) - storage["remarks"]["data"] = remarks_data - print("[migration.load] " + str(len(remarks_data)) + " remarks data ") - - # fill out storage - for x in users_data: - storage["users"]["by_oid"][x["_id"]] = x - # storage['users']['by_slug'][x['slug']] = x - # no user.slug yet - print( - "[migration.load] " - + str(len(storage["users"]["by_oid"].keys())) - + " users by oid" - ) - for x in tags_data: - storage["topics"]["by_oid"][x["_id"]] = x - storage["topics"]["by_slug"][x["slug"]] = x - for x in cats_data: - storage["topics"]["by_oid"][x["_id"]] = x - storage["topics"]["by_slug"][x["slug"]] = x - print( - "[migration.load] " - + str(len(storage["topics"]["by_slug"].keys())) - + " topics by slug" - ) - for item in content_data: - slug = get_shout_slug(item) - storage["content_items"]["by_slug"][slug] = item - storage["content_items"]["by_oid"][item["_id"]] = item - print("[migration.load] " + str(len(content_data)) + " content items") - for x in comments_data: - storage["reactions"]["by_oid"][x["_id"]] = x - cid = x["contentItem"] - storage["reactions"]["by_content"][cid] = x - ci = storage["content_items"]["by_oid"].get(cid, {}) - if "slug" in ci: - storage["reactions"]["by_slug"][ci["slug"]] = x - print( - "[migration.load] " - + str(len(storage["reactions"]["by_content"].keys())) - + " with comments" - ) - storage["users"]["data"] = users_data - storage["topics"]["tags"] = tags_data - storage["topics"]["cats"] = cats_data - storage["shouts"]["data"] = content_data - storage["reactions"]["data"] = comments_data - except Exception as e: - raise e - return storage - - -async def handling_migration(): - init_tables() - await all_handle(data_load(), sys.argv) - - -def process(): - loop = asyncio.get_event_loop() - loop.run_until_complete(handling_migration()) - - -if __name__ == "__main__": - process() diff --git a/migration/bson2json.py b/migration/bson2json.py deleted file mode 100644 index 03effe19..00000000 --- a/migration/bson2json.py +++ /dev/null @@ -1,32 +0,0 @@ -import json -import os - -import bson -import gc -from .utils import DateTimeEncoder - - -def json_tables(): - print("[migration] unpack dump/discours/*.bson to migration/data/*.json") - data = { - "content_items": [], - "content_item_categories": [], - "tags": [], - "email_subscriptions": [], - "users": [], - "comments": [], - "remarks": [] - } - for table in data.keys(): - print('[migration] bson2json for ' + table) - gc.collect() - lc = [] - bs = open("dump/discours/" + table + ".bson", "rb").read() - base = 0 - while base < len(bs): - base, d = bson.decode_document(bs, base) - lc.append(d) - data[table] = lc - open(os.getcwd() + "/migration/data/" + table + ".json", "w").write( - json.dumps(lc, cls=DateTimeEncoder) - ) diff --git a/migration/data/.gitkeep b/migration/data/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/migration/export.py b/migration/export.py deleted file mode 100644 index 102cfb14..00000000 --- a/migration/export.py +++ /dev/null @@ -1,159 +0,0 @@ -import json -import os -from datetime import datetime, timezone - -import frontmatter - -from .extract import extract_html, extract_media -from .utils import DateTimeEncoder - -OLD_DATE = "2016-03-05 22:22:00.350000" -EXPORT_DEST = "../discoursio-web/data/" -parentDir = "/".join(os.getcwd().split("/")[:-1]) -contentDir = parentDir + "/discoursio-web/content/" -ts = datetime.now(tz=timezone.utc) - - -def get_metadata(r): - authors = [] - for a in r["authors"]: - authors.append( - { # a short version for public listings - "slug": a.slug or "discours", - "name": a.name or "Дискурс", - "userpic": a.userpic or "https://discours.io/static/img/discours.png", - } - ) - metadata = {} - metadata["title"] = r.get("title", "").replace("{", "(").replace("}", ")") - metadata["authors"] = authors - metadata["createdAt"] = r.get("createdAt", ts) - metadata["layout"] = r["layout"] - metadata["topics"] = [topic for topic in r["topics"]] - metadata["topics"].sort() - if r.get("cover", False): - metadata["cover"] = r.get("cover") - return metadata - - -def export_mdx(r): - # print('[export] mdx %s' % r['slug']) - content = "" - metadata = get_metadata(r) - content = frontmatter.dumps(frontmatter.Post(r["body"], **metadata)) - ext = "mdx" - filepath = contentDir + r["slug"] - bc = bytes(content, "utf-8").decode("utf-8", "ignore") - open(filepath + "." + ext, "w").write(bc) - - -def export_body(shout, storage): - entry = storage["content_items"]["by_oid"][shout["oid"]] - if entry: - body = extract_html(entry) - media = extract_media(entry) - shout["body"] = body # prepare_html_body(entry) # prepare_md_body(entry) - shout["media"] = media - export_mdx(shout) - print("[export] html for %s" % shout["slug"]) - open(contentDir + shout["slug"] + ".html", "w").write(body) - else: - raise Exception("no content_items entry found") - - -def export_slug(slug, storage): - shout = storage["shouts"]["by_slug"][slug] - shout = storage["shouts"]["by_slug"].get(slug) - assert shout, "[export] no shout found by slug: %s " % slug - author = shout["authors"][0] - assert author, "[export] no author error" - export_body(shout, storage) - - -def export_email_subscriptions(): - email_subscriptions_data = json.loads( - open("migration/data/email_subscriptions.json").read() - ) - for data in email_subscriptions_data: - # TODO: migrate to mailgun list manually - # migrate_email_subscription(data) - pass - print( - "[migration] " - + str(len(email_subscriptions_data)) - + " email subscriptions exported" - ) - - -def export_shouts(storage): - # update what was just migrated or load json again - if len(storage["users"]["by_slugs"].keys()) == 0: - storage["users"]["by_slugs"] = json.loads( - open(EXPORT_DEST + "authors.json").read() - ) - print( - "[migration] " - + str(len(storage["users"]["by_slugs"].keys())) - + " exported authors " - ) - if len(storage["shouts"]["by_slugs"].keys()) == 0: - storage["shouts"]["by_slugs"] = json.loads( - open(EXPORT_DEST + "articles.json").read() - ) - print( - "[migration] " - + str(len(storage["shouts"]["by_slugs"].keys())) - + " exported articles " - ) - for slug in storage["shouts"]["by_slugs"].keys(): - export_slug(slug, storage) - - -def export_json( - export_articles={}, export_authors={}, export_topics={}, export_comments={} -): - open(EXPORT_DEST + "authors.json", "w").write( - json.dumps( - export_authors, - cls=DateTimeEncoder, - indent=4, - sort_keys=True, - ensure_ascii=False, - ) - ) - print("[migration] " + str(len(export_authors.items())) + " authors exported") - open(EXPORT_DEST + "topics.json", "w").write( - json.dumps( - export_topics, - cls=DateTimeEncoder, - indent=4, - sort_keys=True, - ensure_ascii=False, - ) - ) - print("[migration] " + str(len(export_topics.keys())) + " topics exported") - - open(EXPORT_DEST + "articles.json", "w").write( - json.dumps( - export_articles, - cls=DateTimeEncoder, - indent=4, - sort_keys=True, - ensure_ascii=False, - ) - ) - print("[migration] " + str(len(export_articles.items())) + " articles exported") - open(EXPORT_DEST + "comments.json", "w").write( - json.dumps( - export_comments, - cls=DateTimeEncoder, - indent=4, - sort_keys=True, - ensure_ascii=False, - ) - ) - print( - "[migration] " - + str(len(export_comments.items())) - + " exported articles with comments" - ) diff --git a/migration/extract.py b/migration/extract.py deleted file mode 100644 index 9ea84067..00000000 --- a/migration/extract.py +++ /dev/null @@ -1,434 +0,0 @@ -import base64 -import os -import re -import uuid - -from bs4 import BeautifulSoup - - -TOOLTIP_REGEX = r"(\/\/\/(.+)\/\/\/)" -contentDir = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "..", "..", "discoursio-web", "content" -) -s3 = "https://discours-io.s3.amazonaws.com/" -cdn = "https://assets.discours.io" - - -def replace_tooltips(body): - # change if you prefer regexp - newbody = body - matches = list(re.finditer(TOOLTIP_REGEX, body, re.IGNORECASE | re.MULTILINE))[1:] - for match in matches: - newbody = body.replace( - match.group(1), '' - ) # NOTE: doesn't work - if len(matches) > 0: - print("[extract] found %d tooltips" % len(matches)) - return newbody - - - -def extract_footnotes(body, shout_dict): - parts = body.split("&&&") - lll = len(parts) - newparts = list(parts) - placed = False - if lll & 1: - if lll > 1: - i = 1 - print("[extract] found %d footnotes in body" % (lll - 1)) - for part in parts[1:]: - if i & 1: - placed = True - if 'a class="footnote-url" href=' in part: - print("[extract] footnote: " + part) - fn = 'a class="footnote-url" href="' - exxtracted_link = part.split(fn, 1)[1].split('"', 1)[0] - extracted_body = part.split(fn, 1)[1].split('>', 1)[1].split('', 1)[0] - print("[extract] footnote link: " + extracted_link) - with local_session() as session: - Reaction.create({ - "shout": shout_dict['id'], - "kind": ReactionKind.FOOTNOTE, - "body": extracted_body, - "range": str(body.index(fn + link) - len('<')) + ':' + str(body.index(extracted_body) + len('')) - }) - newparts[i] = "ℹ️" - else: - newparts[i] = part - i += 1 - return ("".join(newparts), placed) - - -def place_tooltips(body): - parts = body.split("&&&") - lll = len(parts) - newparts = list(parts) - placed = False - if lll & 1: - if lll > 1: - i = 1 - print("[extract] found %d tooltips" % (lll - 1)) - for part in parts[1:]: - if i & 1: - placed = True - if 'a class="footnote-url" href=' in part: - print("[extract] footnote: " + part) - fn = 'a class="footnote-url" href="' - link = part.split(fn, 1)[1].split('"', 1)[0] - extracted_part = ( - part.split(fn, 1)[0] + " " + part.split("/", 1)[-1] - ) - newparts[i] = ( - "" - + extracted_part - + "" - ) - else: - newparts[i] = "%s" % part - # print('[extract] ' + newparts[i]) - else: - # print('[extract] ' + part[:10] + '..') - newparts[i] = part - i += 1 - return ("".join(newparts), placed) - - -IMG_REGEX = r"\!\[(.*?)\]\((data\:image\/(png|jpeg|jpg);base64\,((?:[A-Za-z\d+\/]{4})*(?:[A-Za-z\d+\/]{3}=" -IMG_REGEX += r"|[A-Za-z\d+\/]{2}==)))\)" - -parentDir = "/".join(os.getcwd().split("/")[:-1]) -public = parentDir + "/discoursio-web/public" -cache = {} - - -def reextract_images(body, oid): - # change if you prefer regexp - matches = list(re.finditer(IMG_REGEX, body, re.IGNORECASE | re.MULTILINE))[1:] - i = 0 - for match in matches: - print("[extract] image " + match.group(1)) - ext = match.group(3) - name = oid + str(i) - link = public + "/upload/image-" + name + "." + ext - img = match.group(4) - title = match.group(1) # NOTE: this is not the title - if img not in cache: - content = base64.b64decode(img + "==") - print(str(len(img)) + " image bytes been written") - open("../" + link, "wb").write(content) - cache[img] = name - i += 1 - else: - print("[extract] image cached " + cache[img]) - body.replace( - str(match), "![" + title + "](" + cdn + link + ")" - ) # WARNING: this does not work - return body - - -IMAGES = { - "data:image/png": "png", - "data:image/jpg": "jpg", - "data:image/jpeg": "jpg", -} - -b64 = ";base64," - - -def extract_imageparts(bodyparts, prefix): - # recursive loop - newparts = list(bodyparts) - for current in bodyparts: - i = bodyparts.index(current) - for mime in IMAGES.keys(): - if mime == current[-len(mime) :] and (i + 1 < len(bodyparts)): - print("[extract] " + mime) - next = bodyparts[i + 1] - ext = IMAGES[mime] - b64end = next.index(")") - b64encoded = next[:b64end] - name = prefix + "-" + str(len(cache)) - link = "/upload/image-" + name + "." + ext - print("[extract] name: " + name) - print("[extract] link: " + link) - print("[extract] %d bytes" % len(b64encoded)) - if b64encoded not in cache: - try: - content = base64.b64decode(b64encoded + "==") - open(public + link, "wb").write(content) - print( - "[extract] " - + str(len(content)) - + " image bytes been written" - ) - cache[b64encoded] = name - except Exception: - raise Exception - # raise Exception('[extract] error decoding image %r' %b64encoded) - else: - print("[extract] cached link " + cache[b64encoded]) - name = cache[b64encoded] - link = cdn + "/upload/image-" + name + "." + ext - newparts[i] = ( - current[: -len(mime)] - + current[-len(mime) :] - + link - + next[-b64end:] - ) - newparts[i + 1] = next[:-b64end] - break - return ( - extract_imageparts( - newparts[i] + newparts[i + 1] + b64.join(bodyparts[(i + 2) :]), prefix - ) - if len(bodyparts) > (i + 1) - else "".join(newparts) - ) - - -def extract_dataimages(parts, prefix): - newparts = list(parts) - for part in parts: - i = parts.index(part) - if part.endswith("]("): - [ext, rest] = parts[i + 1].split(b64) - name = prefix + "-" + str(len(cache)) - if ext == "/jpeg": - ext = "jpg" - else: - ext = ext.replace("/", "") - link = "/upload/image-" + name + "." + ext - print("[extract] filename: " + link) - b64end = rest.find(")") - if b64end != -1: - b64encoded = rest[:b64end] - print("[extract] %d text bytes" % len(b64encoded)) - # write if not cached - if b64encoded not in cache: - try: - content = base64.b64decode(b64encoded + "==") - open(public + link, "wb").write(content) - print("[extract] " + str(len(content)) + " image bytes") - cache[b64encoded] = name - except Exception: - raise Exception - # raise Exception('[extract] error decoding image %r' %b64encoded) - else: - print("[extract] 0 image bytes, cached for " + cache[b64encoded]) - name = cache[b64encoded] - - # update link with CDN - link = cdn + "/upload/image-" + name + "." + ext - - # patch newparts - newparts[i + 1] = link + rest[b64end:] - else: - raise Exception("cannot find the end of base64 encoded string") - else: - print("[extract] dataimage skipping part " + str(i)) - continue - return "".join(newparts) - - -di = "data:image" - - -def extract_md_images(body, prefix): - newbody = "" - body = ( - body.replace("\n! [](" + di, "\n ![](" + di) - .replace("\n[](" + di, "\n![](" + di) - .replace(" [](" + di, " ![](" + di) - ) - parts = body.split(di) - if len(parts) > 1: - newbody = extract_dataimages(parts, prefix) - else: - newbody = body - return newbody - - -def cleanup_md(body): - newbody = ( - body.replace("<", "") - .replace(">", "") - .replace("{", "(") - .replace("}", ")") - .replace("…", "...") - .replace(" __ ", " ") - .replace("_ _", " ") - .replace("****", "") - .replace("\u00a0", " ") - .replace("\u02c6", "^") - .replace("\u00a0", " ") - .replace("\ufeff", "") - .replace("\u200b", "") - .replace("\u200c", "") - ) # .replace('\u2212', '-') - return newbody - - -def extract_md(body, shout_dict = None): - newbody = body - if newbody: - newbody = cleanup_md(newbody) - if not newbody: - raise Exception("cleanup error") - - if shout_dict: - - uid = shout_dict['id'] or uuid.uuid4() - newbody = extract_md_images(newbody, uid) - if not newbody: - raise Exception("extract_images error") - - newbody, placed = extract_footnotes(body, shout_dict) - if not newbody: - raise Exception("extract_footnotes error") - - return newbody - - -def extract_media(entry): - ''' normalized media extraction method ''' - # media [ { title pic url body } ]} - kind = entry.get("type") - if not kind: - print(entry) - raise Exception("shout no layout") - media = [] - for m in entry.get("media") or []: - # title - title = m.get("title", "").replace("\n", " ").replace(" ", " ") - artist = m.get("performer") or m.get("artist") - if artist: - title = artist + " - " + title - - # pic - url = m.get("fileUrl") or m.get("url", "") - pic = "" - if m.get("thumborId"): - pic = cdn + "/unsafe/1600x/" + m["thumborId"] - - # url - if not url: - if kind == "Image": - url = pic - elif "youtubeId" in m: - url = "https://youtube.com/?watch=" + m["youtubeId"] - elif "vimeoId" in m: - url = "https://vimeo.com/" + m["vimeoId"] - # body - body = m.get("body") or m.get("literatureBody") or "" - media.append({ - "url": url, - "pic": pic, - "title": title, - "body": body - }) - return media - - -def prepare_html_body(entry): - # body modifications - body = "" - kind = entry.get("type") - addon = "" - if kind == "Video": - addon = "" - for m in entry.get("media") or []: - if "youtubeId" in m: - addon += '