psql-migration-fixes

migrate.py

@@ -7,6 +7,9 @@ from migration.tables.content_items import get_shout_slug, migrate as migrateSho
 from migration.tables.topics import migrate as migrateTopic
 from migration.tables.comments import migrate as migrateComment
 from migration.tables.comments import migrate_2stage as migrateComment_2stage
+from orm.base import local_session
+from orm.community import Community
+from orm.user import User
 
 OLD_DATE = '2016-03-05 22:22:00.350000'
 

migration/html2text/__init__.py

@@ -1037,5 +1037,5 @@ def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = config.BO
 	if h: 
 		h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
 		h = h.handle(html.strip())
-		print('[html2text] %d bytes' % len(html))
+		# print('[html2text] %d bytes' % len(html))
 	return h

migration/tables/comments.py

@@ -57,7 +57,7 @@ def migrate(entry, storage):
 			author = session.query(User).filter(User.oid == entry['createdBy']).first()
 			shout_dict = storage['shouts']['by_oid'][shout_oid]
 			if shout_dict:
-				comment_dict['shout'] = shout_dict['oid']
+				comment_dict['shout'] = shout_dict['slug']
 				comment_dict['createdBy'] = author.slug if author else 'discours'
 				# FIXME if entry.get('deleted'): comment_dict['deletedAt'] = date_parse(entry['updatedAt']) or ts
 				# comment_dict['deletedBy'] = session.query(User).filter(User.oid == (entry.get('updatedBy') or dd['oid'])).first()

migration/tables/content_items.py

@@ -7,14 +7,6 @@ from orm.base import local_session
 from migration.extract import prepare_body
 from orm.community import Community
 
-DISCOURS_USER = {
-	'id': 9999999,
-	'slug': 'discours',
-	'name': 'Дискурс',
-	'email': 'welcome@discours.io',
-	'userpic': 'https://discours.io/images/logo-mini.svg',
-	'createdAt': '2016-03-05 22:22:00.350000'
-}
 OLD_DATE = '2016-03-05 22:22:00.350000'
 ts = datetime.now()
 type2layout = {
@@ -38,7 +30,7 @@ def migrate(entry, storage):
 	r = {
 		'layout': type2layout[entry['type']],
 		'title': entry['title'],
-		'community': 0,
+		'community': Community.default_community.id,
 		'authors': [],
 		'topics': set([]),
 		'rating': 0,
@@ -70,12 +62,7 @@ def migrate(entry, storage):
 				'wasOnlineAt': ts
 			}
 		else: 
-			userdata = {
+			userdata = User.default_user.dict()
-				'name': 'Дискурс',
-				'slug': 'discours',
-				'email': 'welcome@discours.io',
-				'userpic': 'https://discours.io/image/logo-mini.svg'
-			} 
 	assert userdata, 'no user found for %s from ' % [oid, len(users_by_oid.keys())]
 	r['authors'] = [userdata, ]
 
@@ -151,7 +138,7 @@ def migrate(entry, storage):
 
 	try: 
 		s = Shout.create(**shout_dict)
-	except sqlalchemy.exc.IntegrityError:
+	except sqlalchemy.exc.IntegrityError as e:
 		with local_session() as session:
 			s = session.query(Shout).filter(Shout.slug == shout_dict['slug']).first()
 			bump = False
@@ -168,6 +155,7 @@ def migrate(entry, storage):
 					s.update(shout_dict)
 			else:
 				print('[migration] something went wrong with shout: \n%r' % shout_dict)
+				raise e
 			session.commit()
 	except:
 		print(s)
@@ -182,15 +170,17 @@ def migrate(entry, storage):
 		if newslug:
 			with local_session() as session:
 				shout_topic_old = session.query(ShoutTopic)\
-					.filter(ShoutTopic.shout == s.slug)\
+					.filter(ShoutTopic.shout == shout_dict['slug'])\
 					.filter(ShoutTopic.topic == oldslug).first()
 				if shout_topic_old: 
 					shout_topic_old.update({ 'slug': newslug })
 				else: 
 					shout_topic_new = session.query(ShoutTopic)\
-						.filter(ShoutTopic.shout == s.slug)\
+						.filter(ShoutTopic.shout == shout_dict['slug'])\
 						.filter(ShoutTopic.topic == newslug).first()
-					if not shout_topic_new: ShoutTopic.create(**{ 'shout': s.slug, 'topic': newslug })
+					if not shout_topic_new: 
+						try: ShoutTopic.create(**{ 'shout': shout_dict['slug'], 'topic': newslug })
+						except: print('[migration] shout topic error: ' + newslug)
 				session.commit()
 			if newslug not in shout_dict['topics']:
 				shout_dict['topics'].append(newslug)
@@ -208,12 +198,12 @@ def migrate(entry, storage):
 					shout_rating_dict = {
 						'value': shout_rating_old['value'],
 						'rater': rater.slug,
-						'shout': s.slug
+						'shout': shout_dict['slug']
 					}
 					cts = shout_rating_old.get('createdAt')
 					if cts: shout_rating_dict['ts'] = date_parse(cts)
 					shout_rating = session.query(ShoutRating).\
-						filter(ShoutRating.shout == s.slug).\
+						filter(ShoutRating.shout == shout_dict['slug']).\
 						filter(ShoutRating.rater == rater.slug).first()
 					if shout_rating:
 						shout_rating_dict['value'] = int(shout_rating_dict['value'] or 0) + int(shout_rating.value or 0)
@@ -225,7 +215,7 @@ def migrate(entry, storage):
 		# raise Exception
 
 	# shout views
-	ShoutViewByDay.create( shout = s.slug, value = entry.get('views', 1) )
+	ShoutViewByDay.create( shout = shout_dict['slug'], value = entry.get('views', 1) )
 	del shout_dict['ratings']
 	shout_dict['oid'] = entry.get('_id')
 	storage['shouts']['by_oid'][entry['_id']] = shout_dict

orm/community.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from enum import unique
 from sqlalchemy import Column, Integer, String, ForeignKey, DateTime
 from sqlalchemy.orm import relationship, backref
 from orm.base import Base, local_session
@@ -16,7 +17,7 @@ class Community(Base):
 	__tablename__ = 'community'
 
 	name: str = Column(String, nullable=False, comment="Name")
-	slug: str = Column(String, nullable = False)
+	slug: str = Column(String, nullable = False, unique=True, comment="Slug")
 	desc: str = Column(String, nullable=False, default='')
 	pic: str = Column(String, nullable=False, default='')
 	createdAt: str = Column(DateTime, nullable=False, default = datetime.now, comment="Created at")
@@ -30,7 +31,7 @@ class Community(Base):
 			default = Community.create(
 				name = "Дискурс",
 				slug = "discours",
-				createdBy = 0 #TODO: use default user
+				createdBy = "discours"
 			)
 
 		Community.default_community = default

orm/user.py

@@ -73,13 +73,14 @@ class User(Base):
 	@staticmethod
 	def init_table():
 		with local_session() as session:
-			default = session.query(User).filter(User.slug == "default").first()
+			default = session.query(User).filter(User.slug == "discours").first()
 		if not default:
 			default = User.create(
 				id = 0,
-				email = "discours@discours.io",
+				email = "welcome@discours.io",
-				username = "discours",
+				username = "welcome@discours.io",
-				slug = "default"
+				slug = "discours",
+				userpic = 'https://discours.io/images/logo-mini.svg',
 			)
 
 		User.default_user = default

settings.py

@@ -10,7 +10,7 @@ RESET_PWD_URL = environ.get("RESET_PWD_URL") or "https://localhost:8080/reset_pw
 CONFIRM_EMAIL_URL = environ.get("CONFIRM_EMAIL_URL") or "https://new.discours.io"
 ERROR_URL_ON_FRONTEND = environ.get("ERROR_URL_ON_FRONTEND") or "https://new.discours.io"
 
-DB_URL = environ.get("DATABASE_URL") or environ.get("DB_URL") or "postgres://localhost:5432" or "sqlite:///db.sqlite3"
+DB_URL = environ.get("DATABASE_URL") or environ.get("DB_URL") or "postgresql://localhost:5432/discoursio" or "sqlite:///db.sqlite3"
 JWT_ALGORITHM = "HS256"
 JWT_SECRET_KEY = "8f1bd7696ffb482d8486dfbc6e7d16dd-secret-key"
 JWT_LIFE_SPAN = 24 * 60 * 60  # seconds