discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

fixed-storages

Browse Source
This commit is contained in:
tonyrewin
2022-11-26 10:46:06 +03:00
parent d898466ccd
commit bc09f414e0
4 changed files with 57 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
services/stat/reacted.py
View File

@@ -163,32 +163,29 @@ class ReactedStorage:
@staticmethod
async def recount_changed(session):
start = time.time()
self = ReactedStorage
async with self.lock:
sss = list(self.modified_shouts)
c = 0
for slug in sss:
siblings = session.query(Reaction).where(Reaction.shout == slug).all()
c += len(siblings)
await self.recount(siblings)
sss = list(self.modified_shouts)
c = 0
for slug in sss:
siblings = session.query(Reaction).where(Reaction.shout == slug).all()
c += len(siblings)
await self.recount(siblings)
print("[stat.reacted] %d reactions recounted" % c)
print("[stat.reacted] %d shouts modified" % len(self.modified_shouts))
print("[stat.reacted] %d topics" % len(self.reacted["topics"].values()))
print("[stat.reacted] %d authors" % len(self.reacted["authors"].values()))
print("[stat.reacted] %d replies" % len(self.reacted["reactions"]))
self.modified_shouts = set([])
end = time.time()
print("[stat.reacted] recount_changed took %fs " % (end - start))
print("[stat.reacted] %d reactions recounted" % c)
print("[stat.reacted] %d shouts modified" % len(self.modified_shouts))
print("[stat.reacted] %d topics" % len(self.reacted["topics"].values()))
print("[stat.reacted] %d authors" % len(self.reacted["authors"].values()))
print("[stat.reacted] %d replies" % len(self.reacted["reactions"]))
self.modified_shouts = set([])
@staticmethod
async def worker():
while True:
try:
with local_session() as session:
ts = time.time()
await ReactedStorage.recount_changed(session)
print("[stat.reacted] recount_changed took %fs " % (time.time() - ts))
except Exception as err:
print("[stat.reacted] recount error %s" % (err))
await asyncio.sleep(ReactedStorage.period)

6
services/stat/topicstat.py
View File

@@ -17,7 +17,6 @@ class TopicStat:
@staticmethod
async def load_stat(session):
start = time.time()
self = TopicStat
shout_topics = session.query(ShoutTopic, Shout).join(Shout).all()
print("[stat.topics] %d links for shouts" % len(shout_topics))
@@ -43,9 +42,6 @@ class TopicStat:
self.followers_by_topic[topic] = self.followers_by_topic.get(topic, dict())
self.followers_by_topic[topic][userslug] = userslug
end = time.time()
print("[stat.topics] load_stat took %fs " % (end - start))
@staticmethod
async def get_shouts(topic):
self = TopicStat
@@ -59,8 +55,10 @@ class TopicStat:
while True:
try:
with local_session() as session:
ts = time.time()
async with self.lock:
await self.load_stat(session)
print("[stat.topicstat] load_stat took %fs " % (time.time() - ts))
except Exception as err:
raise Exception(err)
if first_run:

35
services/stat/viewed.py
View File

@@ -86,24 +86,23 @@ class ViewedStorage:
""" query all the pages from ackee sorted by views count """
start = time.time()
self = ViewedStorage
async with self.lock:
try:
self.pages = await self.client.execute_async(load_pages)
self.pages = self.pages["domains"][0]["statistics"]["pages"]
print("[stat.viewed] ackee pages updated")
shouts = {}
try:
self.pages = await self.client.execute_async(load_pages)
self.pages = self.pages["domains"][0]["statistics"]["pages"]
print("[stat.viewed] ackee pages updated")
shouts = {}
try:
for page in self.pages:
p = page["value"].split("?")[0]
slug = p.split('discours.io/')[-1]
shouts[slug] = page["count"]
for slug, v in shouts:
await ViewedStorage.increment(slug, v)
except Exception:
pass
print("[stat.viewed] %d pages collected " % len(shouts.keys()))
except Exception as e:
raise e
for page in self.pages:
p = page["value"].split("?")[0]
slug = p.split('discours.io/')[-1]
shouts[slug] = page["count"]
for slug, v in shouts:
await ViewedStorage.increment(slug, v)
except Exception:
pass
print("[stat.viewed] %d pages collected " % len(shouts.keys()))
except Exception as e:
raise e
end = time.time()
print("[stat.viewed] update_pages took %fs " % (end - start))
@@ -180,8 +179,10 @@ class ViewedStorage:
async with self.lock:
while True:
try:
ts = time.time()
await self.update_pages()
failed = 0
print("[stat.viewed] update_pages took %fs " % (time.time() - ts))
except Exception:
failed += 1
print("[stat.viewed] update failed #%d, wait 10 seconds" % failed)