discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

body preparation fixes, migration refactorings

Browse Source
This commit is contained in:
tonyrewin
2022-07-03 13:02:34 +03:00
parent 1ae64e8732
commit 34a8c161c1
4 changed files with 222 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

345
migration/tables/content_items.py
View File

@@ -1,15 +1,12 @@
from dateutil.parser import parse as date_parse
import frontmatter
import json
import sqlalchemy
from orm import Shout, ShoutTopic, ShoutRating, ShoutViewByDay, User, shout
# from bs4 import BeautifulSoup
from migration.html2text import html2text
from transliterate import translit
from datetime import datetime
from orm.base import local_session
from orm.community import Community
from migration.extract import extract
from migration.extract import prepare_body
import os
DISCOURS_USER = {
@@ -19,9 +16,9 @@ DISCOURS_USER = {
'userpic': 'https://discours.io/images/logo-mini.svg',
'createdAt': '2016-03-05 22:22:00.350000'
}
OLD_DATE = '2016-03-05 22:22:00.350000'
retopics = json.loads(open('migration/tables/replacements.json').read())
ts = datetime.now()
type2layout = {
'Article': 'article',
'Literature': 'prose',
@@ -42,44 +39,17 @@ def get_metadata(r):
metadata['cover'] = r.get('cover')
return metadata
retopics = json.loads(open('migration/tables/replacements.json').read())
def migrate(entry, users_by_oid, topics_by_oid):
'''
type Shout {
slug: String!
author: Int!
body: String!
createdAt: DateTime!
updatedAt: DateTime!
deletedAt: DateTime
deletedBy: Int
rating: Int
ratings: [Rating]
published: Bool!
publishedAt: DateTime # if there is no published field - it is not published
replyTo: String # another shout
tags: [String] # actual values
topics: [String] # topic-slugs, order has matter
title: String
versionOf: String
visibleForRoles: [String] # role ids are strings
visibleForUsers: [Int]
views: Int
}
'''
# print(entry)
content = ''
# init, set title and layout
r = {
'layout': type2layout[entry['type']],
'title': entry['title'],
'community': Community.default_community.id,
'authors': [],
'topics': [],
'rating': entry.get('rating', 0),
'rating': 0,
'ratings': [],
'createdAt': entry.get('createdAt', '2016-03-05 22:22:00.350000')
'createdAt': []
}
# slug
@@ -92,126 +62,33 @@ def migrate(entry, users_by_oid, topics_by_oid):
except: raise Exception
if s: r['slug'] = s
else: raise Exception
# topics
category = entry['category']
mainTopic = topics_by_oid.get(category)
if mainTopic:
r['mainTopic'] = mainTopic["slug"]
topic_oids = [category, ]
topic_errors = []
topic_oids.extend(entry.get('tags', []))
for oid in topic_oids:
if oid in topics_by_oid:
r['topics'].append(topics_by_oid[oid])
else:
# print('ERROR: unknown old topic id: ' + oid)
topic_errors.append(oid)
# cover
if entry.get('image') is not None:
r['cover'] = entry['image']['url']
if entry.get('thumborId') is not None:
r['cover'] = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
if entry.get('updatedAt') is not None:
r['updatedAt'] = date_parse(entry['updatedAt'])
# body
body = ''
body_orig = entry.get('body')
if not body_orig: body_orig = ''
# body modifications
if entry.get('type') == 'Literature':
for m in entry.get('media', []):
t = m.get('title', '')
if t: body_orig += '### ' + t + '\n'
body_orig += (m.get('body', '') or '')
body_orig += '\n' + m.get('literatureBody', '') + '\n'
elif entry.get('type') == 'Video':
providers = set([])
video_url = ''
require = False
for m in entry.get('media', []):
yt = m.get('youtubeId', '')
vm = m.get('vimeoId', '')
if yt:
require = True
providers.add('YouTube')
video_url = 'https://www.youtube.com/watch?v=' + yt
body += '<YouTube youtubeId=\'' + yt + '\' />\n'
if vm:
require = True
providers.add('Vimeo')
video_url = 'https://vimeo.com/' + vm
body += '<Vimeo vimeoId=\'' + vm + '\' />\n'
body += extract(html2text(m.get('body', '')), entry['_id'])
if video_url == '#': print(entry.get('media', 'UNKNOWN MEDIA PROVIDER!'))
if require: body = 'import { ' + ','.join(list(providers)) + ' } from \'solid-social\'\n\n' + body + '\n'
body += extract(html2text(body_orig), entry['_id'])
elif entry.get('type') == 'Music':
require = False
for m in entry.get('media', []):
if 'fileUrl' in m:
require = True
artist = m.get('performer')
trackname = ''
if artist: trackname += artist + ' - '
trackname += m.get('title','')
body += '<MusicPlayer src=\"' + m['fileUrl'] + '\" title=\"' + trackname + '\" />\n'
body += extract(html2text(m.get('body', '')), entry['_id'])
else:
print(m)
if require: body = 'import MusicPlayer from \'$/components/Article/MusicPlayer\'\n\n' + body + '\n'
body += extract(html2text(body_orig), entry['_id'])
elif entry.get('type') == 'Image':
cover = r.get('cover')
images = {}
for m in entry.get('media', []):
t = m.get('title', '')
if t: body += '#### ' + t + '\n'
u = m.get('image', {}).get('url', '')
if 'cloudinary' in u:
u = m.get('thumborId')
if not u: u = cover
if u not in images.keys():
if u.startswith('production'): u = 'https://discours-io.s3.amazonaws.com/' + u
body += '![' + m.get('title','').replace('\n', ' ') + '](' + u + ')\n' # TODO: gallery here
images[u] = u
body += extract(html2text(m.get('body', '')), entry['_id']) + '\n'
body += extract(html2text(body_orig), entry['_id'])
# simple post or no body stored
if body == '':
if not body_orig:
print('[migration] using body history...')
try: body_orig += entry.get('bodyHistory', [{'body': ''}])[0].get('body', '')
except: pass
# need to extract
# body_html = str(BeautifulSoup(body_orig, features="html.parser"))
body += extract(html2text(body_orig), entry['_id'])
c = ''
if entry.get('thumborId'):
c = 'https://assets.discours.io/unsafe/1600x/' + entry['thumborId']
else:
# EVERYTHING IS FINE HERE
pass
# replace some topics
for oldtopicslug, newtopicslug in retopics.items():
body.replace(oldtopicslug, newtopicslug)
c = entry.get('image', {}).get('url')
if not c or 'cloudinary' in c:
c = ''
r['cover'] = c
# authors
# timestamps
# get author data
userdata = {}
try: userdata = users_by_oid[entry['createdBy']]
except KeyError:
r['createdAt'] = date_parse(entry.get('createdAt', OLD_DATE))
r['updatedAt'] = date_parse(entry['updatedAt']) if 'updatedAt' in entry else ts
if entry.get('published'):
r['publishedAt'] = date_parse(entry.get('publishedAt', OLD_DATE))
if r['publishedAt'] == OLD_DATE: r['publishedAt'] = ts
if 'deletedAt' in entry: r['deletedAt'] = date_parse(entry['deletedAt'])
# connected users' data
# r['deletedBy'] = entry.get('deletedBy', '0') # TypeError: 'deletedBy' is an invalid keyword argument for Shout
oid = entry.get('createdBy', '')
userdata = users_by_oid.get(oid, {})
if not userdata.get('slug'):
app = entry.get('application')
if app:
userslug = translit(app['name'], 'ru', reversed=True).replace(' ', '-').replace('\'', '').replace('.', '-').lower()
@@ -232,20 +109,22 @@ def migrate(entry, users_by_oid, topics_by_oid):
'userpic': 'https://discours.io/image/logo-mini.svg'
}
# set author data
r['body'] = body
shout_dict = r.copy()
author = { # a short version for public listings
'slug': userdata.get('slug', 'discours'),
'name': userdata.get('name', 'Дискурс'),
'userpic': userdata.get('userpic', '')
}
shout_dict['authors'] = [ author, ]
r['authors'] = [ author, ]
# body
body = prepare_body(entry)
# save mdx for prerender if published
if entry['published']:
metadata = get_metadata(shout_dict)
r['body'] = body
if entry.get('published'):
content = ''
metadata = get_metadata(r)
content = frontmatter.dumps(frontmatter.Post(r['body'], **metadata))
ext = 'mdx'
parentDir = '/'.join(os.getcwd().split('/')[:-1])
@@ -255,90 +134,116 @@ def migrate(entry, users_by_oid, topics_by_oid):
open(filepath + '.' + ext, 'w').write(bc)
# open(filepath + '.html', 'w').write(body_orig)
# topics
category = entry['category']
mainTopic = topics_by_oid.get(category)
if mainTopic:
r['mainTopic'] = mainTopic["slug"]
topic_oids = [category, ]
topic_errors = []
topic_oids.extend(entry.get('tags', []))
for oid in topic_oids:
if oid in topics_by_oid:
r['topics'].append(topics_by_oid[oid])
else:
# print('ERROR: unknown old topic id: ' + oid)
topic_errors.append(oid)
# set prepared shout data
shout_dict = r.copy()
del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout
del shout_dict['ratings']
# get author
user = None
email = userdata.get('email')
authorslug = userdata.get('slug')
with local_session() as session:
try:
if email: user = session.query(User).filter(User.email == email).first()
if not user and authorslug: user = session.query(User).filter(User.slug == authorslug).first()
if not user and userdata: user = User.create(**userdata)
except:
print('[migration] shout author error: \n%r' % entry)
raise Exception
assert user, 'could not get a user'
shout_dict['authors'] = [ user, ]
# save shout to db
s = object()
try: s = Shout.create(**shout_dict)
except: print('[migration] shout create error: \n%r' % shout_dict)
# shout ratings
try:
shout_dict['createdAt'] = date_parse(r.get('createdAt')) if entry.get('createdAt') else ts
shout_dict['publishedAt'] = date_parse(entry.get('publishedAt')) if entry.get('published') else None
if entry.get('deletedAt') is not None:
shout_dict['deletedAt'] = date_parse(entry.get('deletedAt'))
shout_dict['deletedBy'] = entry.get('deletedBy', '0')
del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout
del shout_dict['ratings']
# get user
user = None
email = userdata.get('email')
slug = userdata.get('slug')
with local_session() as session:
try:
if email: user = session.query(User).filter(User.email == email).first()
if not user and slug: user = session.query(User).filter(User.slug == slug).first()
if not user and userdata: user = User.create(**userdata)
except:
print('[migration] content_items error: \n%r' % entry)
assert user, 'could not get a user'
shout_dict['authors'] = [ user, ]
# create shout
s = object()
try: s = Shout.create(**shout_dict)
except: print('[migration] content_items error: \n%r' % entry)
# shout ratings
shout_dict['ratings'] = []
for shout_rating_old in entry.get('ratings',[]):
with local_session() as session:
rater = session.query(User).\
filter(User.old_id == shout_rating_old['createdBy']).first()
if rater:
shout_rating_dict = {
'value': shout_rating_old['value'],
'rater': rater.slug,
'shout': s.slug
}
cts = shout_rating_old.get('createdAt')
if cts: shout_rating_dict['ts'] = date_parse(cts)
try:
rater = session.query(User).filter(User.old_id == shout_rating_old['createdBy']).first()
if rater:
shout_rating_dict = {
'value': shout_rating_old['value'],
'rater': rater.slug,
'shout': s.slug
}
cts = shout_rating_old.get('createdAt')
if cts: shout_rating_dict['ts'] = date_parse(cts)
shout_rating = session.query(ShoutRating).\
filter(ShoutRating.shout == s.slug).\
filter(ShoutRating.rater == rater.slug).first()
if shout_rating:
shout_rating_dict['value'] += int(shout_rating.value or 0)
shout_rating_dict['value'] = int(shout_rating_dict['value'] or 0) + int(shout_rating.value or 0)
shout_rating.update(shout_rating_dict)
else: ShoutRating.create(**shout_rating_dict)
shout_dict['ratings'].append(shout_rating_dict)
except sqlalchemy.exc.IntegrityError:
print('[migration] shout_rating error: \n%r' % shout_rating_dict)
pass
# shout topics
except:
print('[migration] shout rating error: \n%r' % shout_rating_old)
# raise Exception
# shout topics
try:
shout_dict['topics'] = []
for topic in r['topics']:
try:
tpc = topics_by_oid[topic['oid']]
slug = retopics.get(tpc['slug'], tpc['slug'])
ShoutTopic.create(**{ 'shout': s.slug, 'topic': slug })
shout_dict['topics'].append(slug)
except sqlalchemy.exc.IntegrityError:
pass
# shout views
tpc = topics_by_oid[topic['oid']]
oldslug = tpc['slug']
newslug = retopics.get(oldslug, oldslug)
need_create_topic = False
if newslug:
with local_session() as session:
shout_topic_new = session.query(ShoutTopic)\
.filter(ShoutTopic.shout == s.slug)\
.filter(ShoutTopic.topic == newslug).first()
shout_topic_old = session.query(ShoutTopic)\
.filter(ShoutTopic.shout == s.slug)\
.filter(ShoutTopic.topic == oldslug).first()
if not shout_topic_new:
if shout_topic_old:
shout_topic_old.update({ 'slug': newslug })
else:
need_create_topic = True
if need_create_topic:
ShoutTopic.create(**{ 'shout': s.slug, 'topic': newslug })
shout_dict['topics'].append(newslug)
except:
print('[migration] shout topic error: \n%r' % entry)
raise Exception
# shout views
try:
views = entry.get('views', 1)
ShoutViewByDay.create(
shout = s.slug,
value = views
)
except Exception as e:
raise e
except:
print('[migration] shout view error: \n%r' % entry)
# raise Exception
shout_dict['old_id'] = entry.get('_id')
return shout_dict, topic_errors