discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

restructured,inbox-removed

Browse Source
This commit is contained in:
Tony Rewin
2023-10-05 21:46:18 +03:00
parent 6dfec6714a
commit deac939ed8
49 changed files with 886 additions and 1549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
services/db.py Normal file
View File

@@ -0,0 +1,56 @@
from typing import TypeVar, Any, Dict, Generic, Callable
from sqlalchemy import create_engine, Column, Integer
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from sqlalchemy.sql.schema import Table
from settings import DB_URL
engine = create_engine(
DB_URL, echo=False, pool_size=10, max_overflow=20
)
T = TypeVar("T")
REGISTRY: Dict[str, type] = {}
def local_session():
return Session(bind=engine, expire_on_commit=False)
class Base(declarative_base()):
__table__: Table
__tablename__: str
__new__: Callable
__init__: Callable
__allow_unmapped__ = True
__abstract__ = True
__table_args__ = {"extend_existing": True}
id = Column(Integer, primary_key=True)
def __init_subclass__(cls, **kwargs):
REGISTRY[cls.__name__] = cls
@classmethod
def create(cls: Generic[T], **kwargs) -> Generic[T]:
instance = cls(**kwargs)
return instance.save()
def save(self) -> Generic[T]:
with local_session() as session:
session.add(self)
session.commit()
return self
def update(self, input):
column_names = self.__table__.columns.keys()
for (name, value) in input.items():
if name in column_names:
setattr(self, name, value)
def dict(self) -> Dict[str, Any]:
column_names = self.__table__.columns.keys()
return {c: getattr(self, c) for c in column_names}

39
services/exceptions.py Normal file
View File

@@ -0,0 +1,39 @@
from starlette.exceptions import HTTPException
# TODO: remove traceback from logs for defined exceptions
class BaseHttpException(HTTPException):
states_code = 500
detail = "500 Server error"
class ExpiredToken(BaseHttpException):
states_code = 401
detail = "401 Expired Token"
class InvalidToken(BaseHttpException):
states_code = 401
detail = "401 Invalid Token"
class Unauthorized(BaseHttpException):
states_code = 401
detail = "401 Unauthorized"
class ObjectNotExist(BaseHttpException):
code = 404
detail = "404 Object Does Not Exist"
class OperationNotAllowed(BaseHttpException):
states_code = 403
detail = "403 Operation Is Not Allowed"
class InvalidPassword(BaseHttpException):
states_code = 403
message = "403 Invalid Password"

46
services/inbox/presence.py
View File

@@ -1,46 +0,0 @@
# from base.exceptions import Unauthorized
from auth.tokenstorage import SessionToken
from base.redis import redis
async def set_online_status(user_id, status):
if user_id:
if status:
await redis.execute("SADD", "users-online", user_id)
else:
await redis.execute("SREM", "users-online", user_id)
async def on_connect(req, params):
if not isinstance(params, dict):
req.scope["connection_params"] = {}
return
token = params.get('token')
if not token:
# raise Unauthorized("Please login")
return {
"error": "Please login first"
}
else:
payload = await SessionToken.verify(token)
if payload and payload.user_id:
req.scope["user_id"] = payload.user_id
await set_online_status(payload.user_id, True)
async def on_disconnect(req):
user_id = req.scope.get("user_id")
await set_online_status(user_id, False)
# FIXME: not used yet
def context_value(request):
context = {}
print(f"[inbox.presense] request debug: {request}")
if request.scope["type"] == "websocket":
# request is an instance of WebSocket
context.update(request.scope["connection_params"])
else:
context["token"] = request.META.get("authorization")
return context

22
services/inbox/sse.py
View File

@@ -1,22 +0,0 @@
from sse_starlette.sse import EventSourceResponse
from starlette.requests import Request
from graphql.type import GraphQLResolveInfo
from resolvers.inbox.messages import message_generator
# from base.exceptions import Unauthorized
# https://github.com/enisdenjo/graphql-sse/blob/master/PROTOCOL.md
async def sse_messages(request: Request):
print(f'[SSE] request\n{request}\n')
info = GraphQLResolveInfo()
info.context['request'] = request.scope
user_id = request.scope['user'].user_id
if user_id:
event_generator = await message_generator(None, info)
return EventSourceResponse(event_generator)
else:
# raise Unauthorized("Please login")
return {
"error": "Please login first"
}

12
services/main.py
View File

@@ -1,13 +1,13 @@
from services.search import SearchService
from services.stat.viewed import ViewedStorage
from base.orm import local_session
from stat.viewed import ViewedStorage
from services.db import local_session
async def storages_init():
with local_session() as session:
print('[main] initialize SearchService')
print("[main] initialize SearchService")
await SearchService.init(session)
print('[main] SearchService initialized')
print('[main] initialize storages')
print("[main] SearchService initialized")
print("[main] initialize storages")
await ViewedStorage.init()
print('[main] storages initialized')
print("[main] storages initialized")

35
services/presence.py Normal file
View File

@@ -0,0 +1,35 @@
import json
from orm.reaction import Reaction
from orm.shout import Shout
from services.redis import redis
async def notify_reaction(reaction: Reaction):
channel_name = f"new_reaction"
data = {**reaction, "kind": f"new_reaction{reaction.kind}"}
try:
await redis.publish(channel_name, json.dumps(data))
except Exception as e:
print(f"Failed to publish to channel {channel_name}: {e}")
async def notify_shout(shout: Shout):
channel_name = f"new_shout"
data = {**shout, "kind": "new_shout"}
try:
await redis.publish(channel_name, json.dumps(data))
except Exception as e:
print(f"Failed to publish to channel {channel_name}: {e}")
async def notify_follower(follower_id: int, author_id: int):
channel_name = f"new_follower"
data = {
"follower_id": follower_id,
"author_id": author_id,
"kind": "new_follower",
}
try:
await redis.publish(channel_name, json.dumps(data))
except Exception as e:
print(f"Failed to publish to channel {channel_name}: {e}")

58
services/redis.py Normal file
View File

@@ -0,0 +1,58 @@
import asyncio
import aredis
from settings import REDIS_URL
class RedisCache:
def __init__(self, uri=REDIS_URL):
self._uri: str = uri
self.pubsub_channels = []
self._instance = None
async def connect(self):
self._instance = aredis.StrictRedis.from_url(self._uri, decode_responses=True)
async def disconnect(self):
self._instance.connection_pool.disconnect()
self._instance = None
async def execute(self, command, *args, **kwargs):
while not self._instance:
await asyncio.sleep(1)
try:
print("[redis] " + command + " " + " ".join(args))
return await self._instance.execute_command(command, *args, **kwargs)
except Exception:
pass
async def subscribe(self, *channels):
if not self._instance:
await self.connect()
for channel in channels:
await self._instance.subscribe(channel)
self.pubsub_channels.append(channel)
async def unsubscribe(self, *channels):
if not self._instance:
return
for channel in channels:
await self._instance.unsubscribe(channel)
self.pubsub_channels.remove(channel)
async def publish(self, channel, data):
if not self._instance:
return
await self._instance.publish(channel, data)
async def lrange(self, key, start, stop):
print(f"[redis] LRANGE {key} {start} {stop}")
return await self._instance.lrange(key, start, stop)
async def mget(self, key, *keys):
print(f"[redis] MGET {key} {keys}")
return await self._instance.mget(key, *keys)
redis = RedisCache()
__all__ = ["redis"]

15
services/schema.py Normal file
View File

@@ -0,0 +1,15 @@
from ariadne import MutationType, QueryType, SubscriptionType, ScalarType
datetime_scalar = ScalarType("DateTime")
@datetime_scalar.serializer
def serialize_datetime(value):
return value.isoformat()
query = QueryType()
mutation = MutationType()
subscription = SubscriptionType()
resolvers = [query, mutation, subscription, datetime_scalar]

10
services/search.py
View File

@@ -1,8 +1,8 @@
import asyncio
import json
from base.redis import redis
from orm.shout import Shout
from resolvers.zine.load import load_shouts_by
from services.redis import redis
from db.shout import Shout
from schema.zine.load import load_shouts_by
class SearchService:
@@ -12,7 +12,7 @@ class SearchService:
@staticmethod
async def init(session):
async with SearchService.lock:
print('[search.service] did nothing')
print("[search.service] did nothing")
SearchService.cache = {}
@staticmethod
@@ -24,7 +24,7 @@ class SearchService:
"title": text,
"body": text,
"limit": limit,
"offset": offset
"offset": offset,
}
payload = await load_shouts_by(None, None, options)
await redis.execute("SET", text, json.dumps(payload))

890
services/stat/ackee.graphql
View File

@@ -1,890 +0,0 @@
# Integers that will have a value of 0 or more.
scalar UnsignedInt
# A date-time string at UTC, such as 2007-12-03T10:15:30Z, compliant with the `date-time` format outlined in section 5.6 of the RFC 3339 profile of the ISO 8601 standard for representation of dates and times using the Gregorian calendar.
scalar DateTime
# Floats that will have a value greater than 0.
scalar PositiveFloat
type Token {
# Token identifier. Use this value for authentication.
id: ID!
# Identifies the date and time when the object was created.
created: DateTime!
# Identifies the date and time when the object was updated.
updated: DateTime!
}
input CreateTokenInput {
# Username used to protect the Ackee instance.
username: String!
# Password used to protect the Ackee instance.
password: String!
# Title of the token.
title: String
}
type CreateTokenPayload {
# Indicates that the token creation was successful. Might be 'null' otherwise.
success: Boolean
# The newly created token.
payload: Token
}
type DeleteTokenPayload {
# Indicates that the token deletion was successful. Might be 'null' otherwise.
success: Boolean
}
type Mutation {
# Create a new token. The token is required in order to access protected data.
createToken(input: CreateTokenInput!): CreateTokenPayload!
# Delete an existing token. The token than can't be used anymore for authentication.
deleteToken(id: ID!): DeleteTokenPayload!
# Create a new permanent token. The token is required in order to access protected data.
createPermanentToken(
input: CreatePermanentTokenInput!
): CreatePermanentTokenPayload!
# Update an existing permanent token.
updatePermanentToken(
id: ID!
input: UpdatePermanentTokenInput!
): UpdatePermanentTokenPayload!
# Delete an existing permanent token. The token than can't be used anymore for authentication.
deletePermanentToken(id: ID!): DeletePermanentTokenPayload!
# Create a new record to track a page visit.
createRecord(domainId: ID!, input: CreateRecordInput!): CreateRecordPayload!
# Update an existing record to track the duration of a visit.
updateRecord(id: ID!): UpdateRecordPayload!
# Create a new domain.
createDomain(input: CreateDomainInput!): CreateDomainPayload!
# Update an existing domain.
updateDomain(id: ID!, input: UpdateDomainInput!): UpdateDomainPayload!
# Delete an existing domain.
deleteDomain(id: ID!): DeleteDomainPayload!
# Create a new event.
createEvent(input: CreateEventInput!): CreateEventPayload!
# Update an existing event.
updateEvent(id: ID!, input: UpdateEventInput!): UpdateEventPayload!
# Delete an existing event.
deleteEvent(id: ID!): DeleteEventPayload!
# Create a new action to track an event.
createAction(eventId: ID!, input: CreateActionInput!): CreateActionPayload!
# Update an existing action.
updateAction(id: ID!, input: UpdateActionInput!): UpdateActionPayload!
}
type PermanentToken {
# Permanent token identifier. Use this value for authentication.
id: ID!
# Title of the permanent token.
title: String!
# Identifies the date and time when the object was created.
created: DateTime!
# Identifies the date and time when the object was updated.
updated: DateTime!
}
input CreatePermanentTokenInput {
# Title of the permanent token.
title: String!
}
type CreatePermanentTokenPayload {
# Indicates that the permanent token creation was successful. Might be 'null' otherwise.
success: Boolean
# The newly created permanent token.
payload: PermanentToken
}
input UpdatePermanentTokenInput {
# Title of the permanent token.
title: String!
}
type UpdatePermanentTokenPayload {
# Indicates that the permanent token update was successful. Might be 'null' otherwise.
success: Boolean
# The updated permanent token.
payload: PermanentToken
}
type DeletePermanentTokenPayload {
# Indicates that the permanent token deletion was successful. Might be 'null' otherwise.
success: Boolean
}
type Query {
# Data of a specific permanent token.
permanentToken(id: ID!): PermanentToken
# Data of all existing permanent tokens.
permanentTokens: [PermanentToken!]
# Data of a specific domain.
domain(id: ID!): Domain
# Data of all existing domains.
domains: [Domain!]
# Data of a specific event.
event(id: ID!): Event
# Data of all existing events.
events: [Event!]
# Facts of all domains combined. Usually simple data that can be represented in one value.
facts: Facts!
# Statistics of all domains combined. Usually data that needs to be represented in a list or chart.
statistics: DomainStatistics!
}
# Page views will be stored in records. They contain data about the visit and user. Ackee tries its best to keep tracked data anonymized. Several steps are used to avoid that users are identifiable, while still providing helpful analytics.
type Record {
# Record identifier.
id: ID!
# URL of the page.
siteLocation: URL!
# Where the user came from. Either unknown, a specific page or just the domain. This depends on the browser of the user.
siteReferrer: URL
# Where the user came from. Specified using the source query parameter.
source: String
# Preferred language of the user. ISO 639-1 formatted.
siteLanguage: String
# Width of the screen used by the user to visit the site.
screenWidth: UnsignedInt
# Height of the screen used by the user to visit the site.
screenHeight: UnsignedInt
# Color depth of the screen used by the user to visit the site.
screenColorDepth: UnsignedInt
# Device used by the user to visit the site.
deviceName: String
# Manufacturer of the device used by the user to visit the site.
deviceManufacturer: String
# Operating system used by the user to visit the site.
osName: String
# Operating system version used by the user to visit the site.
osVersion: String
# Browser used by the user to visit the site.
browserName: String
# Version of the browser used by the user to visit the site.
browserVersion: String
# Width of the browser used by the user to visit the site.
browserWidth: UnsignedInt
# Height of the browser used by the user to visit the site.
browserHeight: UnsignedInt
# Identifies the date and time when the object was created.
created: DateTime!
# Identifies the date and time when the object was updated.
updated: DateTime!
}
input CreateRecordInput {
# URL of the page.
siteLocation: URL!
# Where the user came from. Either unknown, a specific page or just the domain. This depends on the browser of the user.
siteReferrer: URL
# Where the user came from. Specified using the source query parameter.
source: String
# Preferred language of the user. ISO 639-1 formatted.
siteLanguage: String
# Width of the screen used by the user to visit the site.
screenWidth: UnsignedInt
# Height of the screen used by the user to visit the site.
screenHeight: UnsignedInt
# Color depth of the screen used by the user to visit the site.
screenColorDepth: UnsignedInt
# Device used by the user to visit the site.
deviceName: String
# Manufacturer of the device used by the user to visit the site.
deviceManufacturer: String
# Operating system used by the user to visit the site.
osName: String
# Operating system version used by the user to visit the site.
osVersion: String
# Browser used by the user to visit the site.
browserName: String
# Version of the browser used by the user to visit the site.
browserVersion: String
# Width of the browser used by the user to visit the site.
browserWidth: UnsignedInt
# Height of the browser used by the user to visit the site.
browserHeight: UnsignedInt
}
type CreateRecordPayload {
# Indicates that the record creation was successful. Might be 'null' otherwise.
success: Boolean
# The newly created record.
payload: Record
}
type UpdateRecordPayload {
# Indicates that the record update was successful. Might be 'null' otherwise.
success: Boolean
}
# Domains are required to track views. You can create as many domains as you want, but it's recommended to create on domain per project/site. This allows you to view facts and statistics separately.
type Domain {
# Domain identifier.
id: ID!
# Title of the domain.
title: String!
# Facts about a domain. Usually simple data that can be represented in one value.
facts: Facts!
# Statistics of a domain. Usually data that needs to be represented in a list or chart.
statistics: DomainStatistics!
# Identifies the date and time when the object was created.
created: DateTime!
# Identifies the date and time when the object was updated.
updated: DateTime!
}
input CreateDomainInput {
# Title of the domain.
title: String!
}
type CreateDomainPayload {
# Indicates that the domain creation was successful. Might be 'null' otherwise.
success: Boolean
# The newly created domain.
payload: Domain
}
input UpdateDomainInput {
# Title of the domain.
title: String!
}
type UpdateDomainPayload {
# Indicates that the domain update was successful. Might be 'null' otherwise.
success: Boolean
# The updated domain.
payload: Domain
}
type DeleteDomainPayload {
# Indicates that the domain deletion was successful. Might be 'null' otherwise.
success: Boolean
}
enum EventType {
# The UI will display the data of this event as a bar chart with totalized values.
TOTAL_CHART
# The UI will display the data of this event as a bar chart with average values.
AVERAGE_CHART
# The UI will display the data of this event as a list of entries with totalized values.
TOTAL_LIST
# The UI will display the data of this event as a list of entries with average values.
AVERAGE_LIST
}
# Events are required to track actions. You can create as many events as you want. This allows you to analyse specific actions happening on your sites. Like a button click or a successful sale.
type Event {
# Event identifier.
id: ID!
# Title of the event.
title: String!
# Type of the event. Allows you to decide how Ackee should display the data of this event in the UI.
type: EventType!
# Statistics of an event. The data is available in different types, depending on whether they are to be shown in a chart or list.
statistics: EventStatistics!
# Identifies the date and time when the object was created.
created: DateTime!
# Identifies the date and time when the object was updated.
updated: DateTime!
}
input CreateEventInput {
# Title of the event.
title: String!
# Type of the event.
type: EventType!
}
type CreateEventPayload {
# Indicates that the event creation was successful. Might be 'null' otherwise.
success: Boolean
# The newly created event.
payload: Event
}
input UpdateEventInput {
# Title of the event.
title: String!
# Type of the event.
type: EventType!
}
type UpdateEventPayload {
# Indicates that the event update was successful. Might be 'null' otherwise.
success: Boolean
# The updated event.
payload: Event
}
type DeleteEventPayload {
# Indicates that the event deletion was successful. Might be 'null' otherwise.
success: Boolean
}
# Event entries will be stored as actions.
type Action {
# Action identifier.
id: ID!
# Optional key that will be used to group similar actions in the UI.
key: String!
# Numerical value that is added to all other numerical values of the key, grouped by day, month or year.
# Use '1' to count how many times an event occurred or a price (e.g. '1.99') to see the sum of successful checkouts in a shop.
value: PositiveFloat!
# Details allow you to store more data along with the associated action.
details: String
# Identifies the date and time when the object was created.
created: DateTime!
# Identifies the date and time when the object was updated.
updated: DateTime!
}
input CreateActionInput {
# Key that will be used to group similar actions in the UI.
key: String!
# Numerical value that is added to all other numerical values of the key, grouped by day, month or year.
# Use '1' to count how many times an event occurred or a price (e.g. '1.99') to see the sum of successful checkouts in a shop.
value: PositiveFloat
# Details allow you to store more data along with the associated action.
details: String
}
type CreateActionPayload {
# Indicates that the action creation was successful. Might be 'null' otherwise.
success: Boolean
# The newly created action.
payload: Action
}
input UpdateActionInput {
# Key that will be used to group similar actions in the UI.
key: String!
# Numerical value that is added to all other numerical values of the key, grouped by day, month or year.
# Use '1' to count how many times an event occurred or a price (e.g. '1.99') to see the sum of successful checkouts in a shop.
# Reset an existing value using 'null'.
value: PositiveFloat
# Details allow you to store more data along with the associated action.
details: String
}
type UpdateActionPayload {
# Indicates that the action update was successful. Might be 'null' otherwise.
success: Boolean
}
type AverageViews {
# Average number of views per day during the last 14 days, excluding the current day.
count: UnsignedInt!
# Percentage change of the average views when comparing the last 7 days with the previous 7 days.
# Might be undefined when there's not enough data to compare.
change: Float
}
type AverageDuration {
# Average visit duration in milliseconds for the last 14 days, excluding the current day.
count: UnsignedInt!
# Percentage change of the average visit duration when comparing the last 7 days with the previous 7 days.
# Might be undefined when there's not enough data to compare.
change: Float
}
# Facts about a domain. Usually simple data that can be represented in one value.
type Facts {
# Facts identifier.
id: ID!
# Number of visitors currently on your site.
activeVisitors: UnsignedInt!
# Details about the average number of views.
averageViews: AverageViews!
# Details about the average visit duration.
averageDuration: AverageDuration!
# Number of unique views today.
viewsToday: UnsignedInt!
# Number of unique views this month.
viewsMonth: UnsignedInt!
# Number of unique views this year.
viewsYear: UnsignedInt!
}
scalar URL
enum Interval {
# Group by day.
DAILY
# Group by month.
MONTHLY
# Group by year.
YEARLY
}
enum Sorting {
# Entries with the most occurrences will be shown at the top.
TOP
# Entries sorted by time. The newest entries will be shown at the top.
RECENT
# Entries that appeared for the first time will be shown at the top.
NEW
}
enum Range {
# Data of the last 24 hours.
LAST_24_HOURS
# Data of the last 7 days.
LAST_7_DAYS
# Data of the last 30 days.
LAST_30_DAYS
# Data of the last 6 months.
LAST_6_MONTHS
}
enum ViewType {
# Unique site views.
UNIQUE
# Total page views.
TOTAL
}
type View {
# View identifier.
id: ID!
# Date of visits.
# Either YYYY, YYYY-MM or YYYY-MM-DD depending on the current interval.
value: String!
# Amount of occurrences.
count: UnsignedInt!
}
type Page {
# Page identifier.
id: ID!
# URL of the page.
value: URL!
# Amount of occurrences.
count: UnsignedInt
# Identifies the date and time when the object was created.
created: DateTime
}
enum ReferrerType {
# Use source parameter instead of referrer when available.
WITH_SOURCE
# Omit source parameters and show referrers only.
NO_SOURCE
# Omit referrers and show source parameters only.
ONLY_SOURCE
}
type Referrer {
# Referrer identifier.
id: ID!
# Either the URL of the referrer or the source parameter of the page to indicate where the visit comes from.
value: String!
# Amount of occurrences.
count: UnsignedInt
# Identifies the date and time when the object was created.
created: DateTime
}
type Duration {
# Duration identifier.
id: ID!
# Date of average duration.
# Either YYYY, YYYY-MM or YYYY-MM-DD depending on the current interval.
value: String!
# Average duration in milliseconds.
count: UnsignedInt!
}
enum SystemType {
# Include system version.
WITH_VERSION
# Omit system version.
NO_VERSION
}
type System {
# System identifier.
id: ID!
# Name of the system. With or without the version.
value: String!
# Amount of occurrences.
count: UnsignedInt
# Identifies the date and time when the object was created.
created: DateTime
}
enum DeviceType {
# Include model name.
WITH_MODEL
# Omit model name.
NO_MODEL
}
type Device {
# Device identifier.
id: ID!
# Name of the device. With or without the model.
value: String!
# Amount of occurrences.
count: UnsignedInt
# Identifies the date and time when the object was created.
created: DateTime
}
enum BrowserType {
# Include browser version.
WITH_VERSION
# Omit browser version.
NO_VERSION
}
type Browser {
# Browser identifier.
id: ID!
# Name of the browser. With or without the version.
value: String!
# Amount of occurrences.
count: UnsignedInt
# Identifies the date and time when the object was created.
created: DateTime
}
enum SizeType {
# Browser height in pixels.
BROWSER_WIDTH
# Browser width in pixels.
BROWSER_HEIGHT
# Browser width and height in pixels.
BROWSER_RESOLUTION
# Browser height in pixels.
SCREEN_WIDTH
# Browser width in pixels.
SCREEN_HEIGHT
# Browser width and height in pixels.
SCREEN_RESOLUTION
}
type Size {
# Size identifier.
id: ID!
# Screen or browser width, height or resolution.
value: String!
# Amount of occurrences.
count: UnsignedInt
# Identifies the date and time when the object was created.
created: DateTime
}
type Language {
# Language identifier.
id: ID!
# Name of the language or language code when unknown.
value: String!
# Amount of occurrences.
count: UnsignedInt
# Identifies the date and time when the object was created.
created: DateTime
}
# Statistics of a domain. Usually data that needs to be represented in a list or chart.
type DomainStatistics {
# Statistic identifier.
id: ID!
# Amount of views grouped by day, month or year.
views(
interval: Interval!
type: ViewType!
# Number of entries to return. Starts with the current day, month or year depending on the chosen interval.
limit: Int = 14
): [View!]
# Pages viewed by your visitors.
pages(
sorting: Sorting!
range: Range = LAST_7_DAYS
# Number of entries to return.
limit: Int = 30
): [Page!]
# Where your visitors are coming from.
referrers(
sorting: Sorting!
type: ReferrerType!
range: Range = LAST_7_DAYS
# Number of entries to return.
limit: Int = 30
): [Referrer!]
# Average visit duration by day, month or year.
durations(
interval: Interval!
# Number of entries to return. Starts with the current day, month or year depending on the chosen interval.
limit: Int = 14
): [Duration!]
# Systems used by your visitors.
systems(
sorting: Sorting!
type: SystemType!
range: Range = LAST_7_DAYS
# Number of entries to return.
limit: Int = 30
): [System!]
# Devices used by your visitors.
devices(
sorting: Sorting!
type: DeviceType!
range: Range = LAST_7_DAYS
# Number of entries to return.
limit: Int = 30
): [Device!]
# Browsers used by your visitors.
browsers(
sorting: Sorting!
type: BrowserType!
range: Range = LAST_7_DAYS
# Number of entries to return.
limit: Int = 30
): [Browser!]
# Screen or browser sizes used by your visitors.
sizes(
sorting: Sorting!
type: SizeType!
range: Range = LAST_7_DAYS
# Number of entries to return.
limit: Int = 30
): [Size!]
# Browser languages used by your visitors.
languages(
sorting: Sorting!
range: Range = LAST_7_DAYS
# Number of entries to return.
limit: Int = 30
): [Language!]
}
enum EventChartType {
# Total sum of values.
TOTAL
# Average sum of values.
AVERAGE
}
enum EventListType {
# Total sum of values.
TOTAL
# Average sum of values.
AVERAGE
}
type EventChartEntry {
# Event entry identifier.
id: ID!
# Date of the event entry.
# Either YYYY, YYYY-MM or YYYY-MM-DD depending on the current interval.
value: String!
# Sum of values on that date.
count: Float!
}
type EventListEntry {
# Event entry identifier.
id: ID!
# Key of the event entry.
value: String!
# Sum of values of the current event key.
count: Float
# Identifies the date and time when the object was created.
created: DateTime
}
# Statistics of an event. The data is available in different types, depending on whether they are to be shown in a chart or list.
type EventStatistics {
# Statistic identifier.
id: ID!
# The chart type should be used when showing events in a chart. It groups events by an interval and shows the total or average sum of values on each entry.
chart(
interval: Interval!
type: EventChartType!
# Number of entries to return. Starts with the current day, month or year depending on the chosen interval.
limit: Int = 14
): [EventChartEntry!]
# The list type should be used when showing events in a list. It groups events by their key and shows the total or average sum of values on each entry.
list(
sorting: Sorting!
type: EventListType!
range: Range = LAST_7_DAYS
# Number of entries to return.
limit: Int = 30
): [EventListEntry!]
}

79
services/stat/viewed.py → services/viewed.py
View File

@@ -6,13 +6,13 @@ from ssl import create_default_context
from gql import Client, gql
from gql.transport.aiohttp import AIOHTTPTransport
from sqlalchemy import func
from base.orm import local_session
from orm import User, Topic
from services.db import local_session
from orm import Topic
from orm.shout import ShoutTopic, Shout
load_facts = gql("""
load_facts = gql(
"""
query getDomains {
domains {
id
@@ -25,9 +25,11 @@ query getDomains {
}
}
}
""")
"""
)
load_pages = gql("""
load_pages = gql(
"""
query getDomains {
domains {
title
@@ -41,8 +43,9 @@ query getDomains {
}
}
}
""")
schema_str = open(path.dirname(__file__) + '/ackee.graphql').read()
"""
)
schema_str = open(path.dirname(__file__) + "/ackee.graphql").read()
token = environ.get("ACKEE_TOKEN", "")
@@ -52,8 +55,8 @@ def create_client(headers=None, schema=None):
transport=AIOHTTPTransport(
url="https://ackee.discours.io/api",
ssl=create_default_context(),
headers=headers
)
headers=headers,
),
)
@@ -71,21 +74,24 @@ class ViewedStorage:
@staticmethod
async def init():
""" graphql client connection using permanent token """
"""graphql client connection using permanent token"""
self = ViewedStorage
async with self.lock:
if token:
self.client = create_client({
"Authorization": "Bearer %s" % str(token)
}, schema=schema_str)
print("[stat.viewed] * authorized permanentely by ackee.discours.io: %s" % token)
self.client = create_client(
{"Authorization": "Bearer %s" % str(token)}, schema=schema_str
)
print(
"[stat.viewed] * authorized permanentely by ackee.discours.io: %s"
% token
)
else:
print("[stat.viewed] * please set ACKEE_TOKEN")
self.disabled = True
@staticmethod
async def update_pages():
""" query all the pages from ackee sorted by views count """
"""query all the pages from ackee sorted by views count"""
print("[stat.viewed] ⎧ updating ackee pages data ---")
start = time.time()
self = ViewedStorage
@@ -96,7 +102,7 @@ class ViewedStorage:
try:
for page in self.pages:
p = page["value"].split("?")[0]
slug = p.split('discours.io/')[-1]
slug = p.split("discours.io/")[-1]
shouts[slug] = page["count"]
for slug in shouts.keys():
await ViewedStorage.increment(slug, shouts[slug])
@@ -118,7 +124,7 @@ class ViewedStorage:
# unused yet
@staticmethod
async def get_shout(shout_slug):
""" getting shout views metric by slug """
"""getting shout views metric by slug"""
self = ViewedStorage
async with self.lock:
shout_views = self.by_shouts.get(shout_slug)
@@ -126,7 +132,9 @@ class ViewedStorage:
shout_views = 0
with local_session() as session:
try:
shout = session.query(Shout).where(Shout.slug == shout_slug).one()
shout = (
session.query(Shout).where(Shout.slug == shout_slug).one()
)
self.by_shouts[shout_slug] = shout.views
self.update_topics(session, shout_slug)
except Exception as e:
@@ -136,7 +144,7 @@ class ViewedStorage:
@staticmethod
async def get_topic(topic_slug):
""" getting topic views value summed """
"""getting topic views value summed"""
self = ViewedStorage
topic_views = 0
async with self.lock:
@@ -146,24 +154,28 @@ class ViewedStorage:
@staticmethod
def update_topics(session, shout_slug):
""" updates topics counters by shout slug """
"""updates topics counters by shout slug"""
self = ViewedStorage
for [shout_topic, topic] in session.query(ShoutTopic, Topic).join(Topic).join(Shout).where(
Shout.slug == shout_slug
).all():
for [shout_topic, topic] in (
session.query(ShoutTopic, Topic)
.join(Topic)
.join(Shout)
.where(Shout.slug == shout_slug)
.all()
):
if not self.by_topics.get(topic.slug):
self.by_topics[topic.slug] = {}
self.by_topics[topic.slug][shout_slug] = self.by_shouts[shout_slug]
@staticmethod
async def increment(shout_slug, amount=1, viewer='ackee'):
""" the only way to change views counter """
async def increment(shout_slug, amount=1, viewer="ackee"):
"""the only way to change views counter"""
self = ViewedStorage
async with self.lock:
# TODO optimize, currenty we execute 1 DB transaction per shout
with local_session() as session:
shout = session.query(Shout).where(Shout.slug == shout_slug).one()
if viewer == 'old-discours':
if viewer == "old-discours":
# this is needed for old db migration
if shout.viewsOld == amount:
print(f"viewsOld amount: {amount}")
@@ -174,7 +186,9 @@ class ViewedStorage:
if shout.viewsAckee == amount:
print(f"viewsAckee amount: {amount}")
else:
print(f"viewsAckee amount changed: {shout.viewsAckee} --> {amount}")
print(
f"viewsAckee amount changed: {shout.viewsAckee} --> {amount}"
)
shout.viewsAckee = amount
session.commit()
@@ -185,7 +199,7 @@ class ViewedStorage:
@staticmethod
async def worker():
""" async task worker """
"""async task worker"""
failed = 0
self = ViewedStorage
if self.disabled:
@@ -205,9 +219,10 @@ class ViewedStorage:
if failed == 0:
when = datetime.now(timezone.utc) + timedelta(seconds=self.period)
t = format(when.astimezone().isoformat())
print("[stat.viewed] ⎩ next update: %s" % (
t.split("T")[0] + " " + t.split("T")[1].split(".")[0]
))
print(
"[stat.viewed] ⎩ next update: %s"
% (t.split("T")[0] + " " + t.split("T")[1].split(".")[0])
)
await asyncio.sleep(self.period)
else:
await asyncio.sleep(10)

70
services/zine/gittask.py
View File

@@ -1,70 +0,0 @@
import asyncio
import subprocess
from pathlib import Path
from settings import SHOUTS_REPO
class GitTask:
"""every shout update use a new task"""
queue = asyncio.Queue()
def __init__(self, input, username, user_email, comment):
self.slug = input["slug"]
self.shout_body = input["body"]
self.username = username
self.user_email = user_email
self.comment = comment
GitTask.queue.put_nowait(self)
def init_repo(self):
repo_path = "%s" % (SHOUTS_REPO)
Path(repo_path).mkdir()
cmd = (
"cd %s && git init && "
"git config user.name 'discours' && "
"git config user.email 'discours@discours.io' && "
"touch initial && git add initial && "
"git commit -m 'init repo'" % (repo_path)
)
output = subprocess.check_output(cmd, shell=True)
print(output)
def execute(self):
repo_path = "%s" % (SHOUTS_REPO)
if not Path(repo_path).exists():
self.init_repo()
# cmd = "cd %s && git checkout master" % (repo_path)
# output = subprocess.check_output(cmd, shell=True)
# print(output)
shout_filename = "%s.mdx" % (self.slug)
shout_full_filename = "%s/%s" % (repo_path, shout_filename)
with open(shout_full_filename, mode="w", encoding="utf-8") as shout_file:
shout_file.write(bytes(self.shout_body, "utf-8").decode("utf-8", "ignore"))
author = "%s <%s>" % (self.username, self.user_email)
cmd = "cd %s && git add %s && git commit -m '%s' --author='%s'" % (
repo_path,
shout_filename,
self.comment,
author,
)
output = subprocess.check_output(cmd, shell=True)
print(output)
@staticmethod
async def git_task_worker():
print("[service.git] starting task worker")
while True:
task = await GitTask.queue.get()
try:
task.execute()
except Exception as err:
print("[service.git] worker error: %s" % (err))