diff --git a/migrate.py b/migrate.py index d96712a0..2ad0ee7f 100644 --- a/migrate.py +++ b/migrate.py @@ -9,7 +9,7 @@ from migration.tables.content_item_categories import migrate as migrateCategory from migration.tables.tags import migrate as migrateTag from migration.tables.comments import migrate as migrateComment from migration.utils import DateTimeEncoder -from orm import Community +from orm import Community, Topic from dateutil.parser import parse as date_parse from orm.base import local_session @@ -165,6 +165,13 @@ if __name__ == '__main__': # limiting try: limit = int(sys.argv[2]) if len(sys.argv) > 2 else len(content_data) except ValueError: limit = len(content_data) + + if not topics_by_cat: + with local_session() as session: + topics = session.query(Topic).all() + print("loaded %s topics" % len(topics)) + for topic in topics: + topics_by_cat[topic.cat_id] = topic for entry in content_data[:limit]: try: @@ -306,14 +313,17 @@ if __name__ == '__main__': elif cmd == "topics": topics(export_topics, topics_by_slug, topics_by_cat, topics_by_tag, cats_data, tags_data) elif cmd == "shouts": - Community.create(**{ - 'id' : 0, - 'slug': 'discours.io', - 'name': 'Дискурс', - 'pic': 'https://discours.io/images/logo-min.svg', - 'createdBy': '0', - 'createdAt': date_parse(OLD_DATE) - }) + with local_session() as session: + community = session.query(Community).filter(Community.id == 0).first() + if not community: + Community.create(**{ + 'id' : 0, + 'slug': 'discours.io', + 'name': 'Дискурс', + 'pic': 'https://discours.io/images/logo-min.svg', + 'createdBy': '0', + 'createdAt': date_parse(OLD_DATE) + }) shouts(content_data, shouts_by_slug, shouts_by_oid) # NOTE: listens limit elif cmd == "export_shouts": export_shouts(shouts_by_slug, export_articles, export_authors, content_dict) diff --git a/migration/tables/content_items.py b/migration/tables/content_items.py index 39715a34..78db4a3b 100644 --- a/migration/tables/content_items.py +++ b/migration/tables/content_items.py @@ -3,7 +3,7 @@ import frontmatter import json import sqlite3 import sqlalchemy -from orm import Shout, Comment, Topic, ShoutRating, User #, TODO: CommentRating +from orm import Shout, Comment, Topic, ShoutTopic, ShoutRating, User from bs4 import BeautifulSoup from migration.html2text import html2text from migration.tables.comments import migrate as migrateComment @@ -37,7 +37,7 @@ def get_metadata(r): metadata['authors'] = r.get('authors') metadata['createdAt'] = r.get('createdAt', ts) metadata['layout'] = r['layout'] - metadata['topics'] = r['topics'] + metadata['topics'] = [topic.slug for topic in r['topics']] if r.get('cover', False): metadata['cover'] = r.get('cover') return metadata @@ -66,6 +66,7 @@ def migrate(entry, users_by_oid, topics_by_oid): views: Int } ''' + # print(entry) content = '' r = { 'layout': type2layout[entry['type']], @@ -73,7 +74,6 @@ def migrate(entry, users_by_oid, topics_by_oid): 'community': 0, 'authors': [], 'topics': [], - 'published': entry.get('published', False), 'views': entry.get('views', 0), 'rating': entry.get('rating', 0), 'ratings': [], @@ -90,9 +90,9 @@ def migrate(entry, users_by_oid, topics_by_oid): # print(entry) raise Exception try: - r['topics'].append(topics_by_oid[entry['category']]['slug']) + r['topics'].append(topics_by_oid[entry['category']]) except Exception: - print(entry['category']) + print("invalid category %s" % (entry['category'])) if entry.get('image') is not None: r['cover'] = entry['image']['url'] if entry.get('thumborId') is not None: @@ -174,13 +174,12 @@ def migrate(entry, users_by_oid, topics_by_oid): open('migration/content/' + r['layout'] + '/' + r['slug'] + '.' + ext, 'w').write(content) try: shout_dict['createdAt'] = date_parse(r.get('createdAt')) if entry.get('createdAt') else ts - shout_dict['publishedAt'] = date_parse(entry.get('publishedAt')) if entry.get('published') else ts + shout_dict['publishedAt'] = date_parse(entry.get('publishedAt')) if entry.get('published') else None if entry.get('deletedAt') is not None: shout_dict['deletedAt'] = date_parse(entry.get('deletedAt')) shout_dict['deletedBy'] = entry.get('deletedBy', '0') - del shout_dict['published'] # invalid keyword argument for Shout del shout_dict['topics'] # FIXME: AttributeError: 'str' object has no attribute '_sa_instance_state' del shout_dict['views'] # FIXME: TypeError: 'views' is an invalid keyword argument for Shout del shout_dict['rating'] # FIXME: TypeError: 'rating' is an invalid keyword argument for Shout @@ -223,11 +222,8 @@ def migrate(entry, users_by_oid, topics_by_oid): shout_dict['ratings'].append(shout_rating_dict) # shout topics shout_dict['topics'] = [] - for topic_slug in r['topics']: - topic = session.query(Topic).filter(Topic.slug == topic_slug).first() - if not topic: - try: topic = Topic.create(**{ 'slug': topic_slug, 'title': topic_slug }) - except Exception as e: raise e + for topic in r['topics']: + ShoutTopic.create(**{ 'shout': s.id, 'topic': topic.id }) shout_dict['topics'].append(topic.slug) except Exception as e: if not shout_dict['body']: r['body'] = 'body moved'