migrating comments and adding authors
This commit is contained in:
parent
1714a60e99
commit
38e702923c
20
migrate.py
20
migrate.py
|
|
@ -146,14 +146,24 @@ def shouts():
|
|||
def export_shouts(limit):
|
||||
print('reading json...')
|
||||
newdata = json.loads(open('migration/data/shouts.dict.json', 'r').read())
|
||||
print(str(len(newdata.keys())) + ' loaded')
|
||||
print(str(len(newdata.keys())) + ' shouts loaded')
|
||||
|
||||
users_old = json.loads(open('migration/data/users.dict.json').read())
|
||||
export_authors = json.loads(open('../src/data/authors.json').read())
|
||||
print(str(len(export_authors.items())) + ' pre-exported authors loaded')
|
||||
users_slug = { u['slug']: u for old_id, u in users_old.items()}
|
||||
print(str(len(users_slug.items())) + ' users loaded')
|
||||
|
||||
export_list = [i for i in newdata.items() if i[1]['layout'] == 'article' and i[1]['published']]
|
||||
export_list = sorted(export_list, key=lambda item: item[1]['createdAt'] or OLD_DATE, reverse=True)
|
||||
print(str(len(export_list)) + ' filtered')
|
||||
|
||||
export_list = export_list[:limit or len(export_list)]
|
||||
export_clean = {}
|
||||
for (slug, article) in export_list:
|
||||
if article['layout'] == 'article':
|
||||
for author in article['authors']:
|
||||
export_authors[author['slug']] = users_slug[author['slug']]
|
||||
export_clean[article['slug']] = extract_images(article)
|
||||
metadata = get_metadata(article)
|
||||
content = frontmatter.dumps(frontmatter.Post(article['body'], **metadata))
|
||||
|
|
@ -165,7 +175,13 @@ def export_shouts(limit):
|
|||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False))
|
||||
print(str(len(export_clean.items())) + ' exported')
|
||||
print(str(len(export_clean.items())) + ' articles exported')
|
||||
open('../src/data/authors.json', 'w').write(json.dumps(export_authors,
|
||||
cls=DateTimeEncoder,
|
||||
indent=4,
|
||||
sort_keys=True,
|
||||
ensure_ascii=False))
|
||||
print(str(len(export_authors.items())) + ' total authors exported')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -1,10 +1,33 @@
|
|||
from html2text import html2text
|
||||
import datetime
|
||||
import json
|
||||
from os.path import abspath
|
||||
from orm import Shout
|
||||
from orm.base import local_session
|
||||
from migration.html2text import html2text
|
||||
|
||||
# markdown = Converter()
|
||||
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
|
||||
topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
|
||||
|
||||
def migrate(entry):
|
||||
'''
|
||||
{
|
||||
"_id": "hdtwS8fSyFLxXCgSC",
|
||||
"body": "<p>",
|
||||
"contentItem": "mnK8KsJHPRi8DrybQ",
|
||||
"createdBy": "bMFPuyNg6qAD2mhXe",
|
||||
"thread": "01/",
|
||||
"createdAt": "2016-04-19 04:33:53+00:00",
|
||||
"ratings": [
|
||||
{ "createdBy": "AqmRukvRiExNpAe8C", "value": 1 },
|
||||
{ "createdBy": "YdE76Wth3yqymKEu5", "value": 1 }
|
||||
],
|
||||
"rating": 2,
|
||||
"updatedAt": "2020-05-27 19:22:57.091000+00:00",
|
||||
"updatedBy": "0"
|
||||
}
|
||||
|
||||
->
|
||||
|
||||
type Comment {
|
||||
id: Int!
|
||||
author: Int!
|
||||
|
|
@ -21,10 +44,16 @@ def migrate(entry):
|
|||
old_id: String
|
||||
}
|
||||
'''
|
||||
# TODO: implement comments migration
|
||||
return {
|
||||
'slug': entry['slug'],
|
||||
'createdAt': entry['createdAt'],
|
||||
'body': html2text(entry['body']),
|
||||
'replyTo': entry['']
|
||||
}
|
||||
with local_session() as session:
|
||||
shout_id = session.query(Shout).filter(Shout.old_id == entry['_id']).first()
|
||||
return {
|
||||
'old_id': entry['_id'],
|
||||
'old_thread': entry['thread'],
|
||||
'createdBy': users_dict[entry['createdBy']],
|
||||
'createdAt': entry['createdAt'],
|
||||
'body': html2text(entry['body']),
|
||||
'shout': shout_id,
|
||||
'rating': entry['rating'],
|
||||
'ratings': [] # TODO: ratings in comments
|
||||
}
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ from datetime import datetime
|
|||
from sqlalchemy.exc import IntegrityError
|
||||
from orm.base import local_session
|
||||
|
||||
comments_data = json.loads(open(abspath('migration/data/comments.json')).read())
|
||||
comments_dict = { x['_id']: x for x in comments_data }
|
||||
users_dict = json.loads(open(abspath('migration/data/users.dict.json')).read())
|
||||
topics_dict = json.loads(open(abspath('migration/data/topics.dict.json')).read()) # old_id keyed
|
||||
users_dict['0'] = {
|
||||
|
|
@ -77,6 +79,7 @@ def migrate(entry):
|
|||
'views': entry.get('views', 0),
|
||||
'rating': entry.get('rating', 0),
|
||||
'ratings': [],
|
||||
'comments': entry.get('comments', []),
|
||||
'createdAt': entry.get('createdAt', '2016-03-05 22:22:00.350000')
|
||||
}
|
||||
r['slug'] = entry.get('slug', '')
|
||||
|
|
@ -184,37 +187,60 @@ def migrate(entry):
|
|||
content = frontmatter.dumps(frontmatter.Post(body, **metadata))
|
||||
|
||||
if entry['published']:
|
||||
# if r.get('old_id', None):
|
||||
# ext = 'html'
|
||||
# content = str(body).replace('<p></p>', '').replace('<p> </p>', '')
|
||||
# else:
|
||||
ext = 'md'
|
||||
open('migration/content/' +
|
||||
r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content)
|
||||
try:
|
||||
shout_dict = r.copy()
|
||||
shout_dict['authors'] = [user, ]
|
||||
if entry.get('createdAt') is not None:
|
||||
shout_dict['createdAt'] = parse(r.get('createdAt'))
|
||||
else:
|
||||
shout_dict['createdAt'] = ts
|
||||
if entry.get('published'):
|
||||
if entry.get('publishedAt') is not None:
|
||||
shout_dict['publishedAt'] = parse(entry.get('publishedAt'))
|
||||
else:
|
||||
shout_dict['publishedAt'] = ts
|
||||
del shout_dict['published']
|
||||
# del shout_dict['views']
|
||||
# del shout_dict['rating']
|
||||
del shout_dict['ratings'] # FIXME
|
||||
try:
|
||||
s = Shout.create(**shout_dict)
|
||||
r['id'] = s.id
|
||||
except:
|
||||
pass
|
||||
except:
|
||||
r['body'] = 'body moved'
|
||||
print(r)
|
||||
# print(s)
|
||||
raise Exception
|
||||
shout_dict = r.copy()
|
||||
shout_dict['authors'] = [user, ]
|
||||
if entry.get('createdAt') is not None:
|
||||
shout_dict['createdAt'] = parse(r.get('createdAt'))
|
||||
else:
|
||||
shout_dict['createdAt'] = ts
|
||||
if entry.get('published'):
|
||||
if entry.get('publishedAt') is not None:
|
||||
shout_dict['publishedAt'] = parse(entry.get('publishedAt'))
|
||||
else:
|
||||
shout_dict['publishedAt'] = ts
|
||||
del shout_dict['published']
|
||||
|
||||
shout_dict['comments'] = []
|
||||
for cid in r['comments']:
|
||||
comment = comments_dict[cid]
|
||||
comment_ratings = []
|
||||
for cr in comment['ratings']:
|
||||
comment_ratings.append({
|
||||
'value': cr['value'],
|
||||
'createdBy': users_dict[cr['createdBy']],
|
||||
'createdAt': cr['createdAt'] or ts})
|
||||
shout_dict['comments'].append({
|
||||
'old_id': comment['_id'],
|
||||
'old_thread': comment['thread'], # TODO: old_thread to replyTo logix
|
||||
'createdBy': users_dict[comment['createdBy']],
|
||||
'createdAt': comment['createdAt'] or ts,
|
||||
'body': html2text(comment['body']),
|
||||
'shout': shout_dict['old_id'],
|
||||
'rating': comment['rating'],
|
||||
'ratings': comment_ratings
|
||||
})
|
||||
|
||||
shout_dict['ratings'] = []
|
||||
for rating in r['ratings']:
|
||||
shout_dict['ratings'].append({
|
||||
'value': rating['value'],
|
||||
'createdBy': users_dict[rating['createdBy']],
|
||||
'createdAt': r['createdAt'] or ts})
|
||||
|
||||
try:
|
||||
del shout_dict['views'] # FIXME
|
||||
del shout_dict['rating'] # FIXME
|
||||
del shout_dict['ratings'] # FIXME
|
||||
# del shout_dict['comments']
|
||||
s = Shout.create(**shout_dict) # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state'
|
||||
r['id'] = s.id
|
||||
except Exception as e:
|
||||
pass # raise e
|
||||
except Exception as e:
|
||||
if not r['body']: r['body'] = 'body moved'
|
||||
raise e
|
||||
return r
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user