From 725d29ae8ec8424f645604afab7620f1ff0e0a5e Mon Sep 17 00:00:00 2001 From: tonyrewin Date: Wed, 14 Dec 2022 09:49:49 +0300 Subject: [PATCH] maybe-fix-outofmemory --- migration/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/migration/__init__.py b/migration/__init__.py index 7995cec6..4e4e6f36 100644 --- a/migration/__init__.py +++ b/migration/__init__.py @@ -5,7 +5,7 @@ import os import subprocess import sys from datetime import datetime, timezone - +import gc import bs4 from migration.tables.comments import migrate as migrateComment from migration.tables.comments import migrate_2stage as migrateComment_2stage @@ -88,6 +88,7 @@ async def shouts_handle(storage, args): topics_dataset_bodies = [] topics_dataset_tlist = [] for entry in storage["shouts"]["data"]: + gc.collect() # slug slug = get_shout_slug(entry) @@ -319,7 +320,9 @@ async def handle_auto(): mongo_download(url) bson_handle() await all_handle(data_load(), sys.argv) - create_pgdump() + if "dump" in sys.argv: + print("[migration] creating pgdump") + create_pgdump() async def handle_comments():