Compare commits
42 Commits
f964aa380e
...
dev
Author | SHA1 | Date | |
---|---|---|---|
![]() |
32e5615528 | ||
![]() |
dfa61abad4 | ||
![]() |
5c4c02aa84 | ||
![]() |
265a3e3fe6 | ||
be3d6f7f76 | |||
7353676000 | |||
9b6cd9cc37 | |||
02e7b7cfbb | |||
fc7bca08f8 | |||
acb6f291d3 | |||
7ae76562fa | |||
aa49f26689 | |||
30303969bd | |||
d554d0ef14 | |||
c005c86303 | |||
112e74284a | |||
9194e897fe | |||
0f545161d5 | |||
90e8b7272a | |||
61e9953c86 | |||
e4cd4bcc2d | |||
8133d0030f | |||
3242758817 | |||
![]() |
2964b72635 | ||
![]() |
2e6678c657 | ||
![]() |
765fa28ecc | ||
![]() |
41c52a4f08 | ||
![]() |
46759142df | ||
![]() |
ef2c902b32 | ||
![]() |
72fc9bd667 | ||
![]() |
6ef45d27f5 | ||
![]() |
f35dcf2b1e | ||
![]() |
cf37dbf103 | ||
![]() |
0934b583da | ||
![]() |
f2d8883ba5 | ||
![]() |
0d430c9f65 | ||
![]() |
ae122412f4 | ||
![]() |
06a912f1a9 | ||
![]() |
2ba6aa64d2 | ||
![]() |
cc36b46fd7 | ||
a070149438 | |||
caeb925989 |
@@ -31,6 +31,6 @@ jobs:
|
|||||||
uses: dokku/github-action@master
|
uses: dokku/github-action@master
|
||||||
with:
|
with:
|
||||||
branch: 'main'
|
branch: 'main'
|
||||||
git_remote_url: 'ssh://dokku@stagging.discours.io:22/uploader'
|
git_remote_url: 'ssh://dokku@staging.discours.io:22/uploader'
|
||||||
ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
|
ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||||
|
|
||||||
|
25
Dockerfile
25
Dockerfile
@@ -1,12 +1,21 @@
|
|||||||
FROM python:slim
|
FROM python:alpine
|
||||||
|
|
||||||
|
# Update package lists and install necessary dependencies
|
||||||
|
RUN apk update && apk add --no-cache build-base icu-data-full curl python3-dev musl-dev && curl -sSL https://install.python-poetry.org | python
|
||||||
|
|
||||||
|
# Set working directory
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy only the pyproject.toml file initially
|
||||||
|
COPY pyproject.toml /app/
|
||||||
|
|
||||||
|
# Install poetry and dependencies
|
||||||
|
RUN pip install poetry && poetry config virtualenvs.create false && poetry install --no-root --only main
|
||||||
|
|
||||||
|
# Copy the rest of the files
|
||||||
COPY . /app
|
COPY . /app
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y git gcc curl postgresql && \
|
# Expose the port
|
||||||
curl -sSL https://install.python-poetry.org | python - && \
|
EXPOSE 8080
|
||||||
echo "export PATH=$PATH:/root/.local/bin" >> ~/.bashrc && \
|
|
||||||
. ~/.bashrc && \
|
|
||||||
poetry config virtualenvs.create false && \
|
|
||||||
poetry install --no-dev
|
|
||||||
|
|
||||||
CMD ["python", "main.py"]
|
CMD ["python", "server.py"]
|
||||||
|
@@ -14,7 +14,7 @@
|
|||||||
- STORJ_END_POINT
|
- STORJ_END_POINT
|
||||||
- STORJ_BUCKET_NAME
|
- STORJ_BUCKET_NAME
|
||||||
- CDN_DOMAIN
|
- CDN_DOMAIN
|
||||||
|
- AUTH_URL
|
||||||
|
|
||||||
### Локальная разработка
|
### Локальная разработка
|
||||||
|
|
||||||
@@ -23,7 +23,7 @@ mkdir .venv
|
|||||||
python3.12 -m venv .venv
|
python3.12 -m venv .venv
|
||||||
poetry env use .venv/bin/python3.12
|
poetry env use .venv/bin/python3.12
|
||||||
poetry update
|
poetry update
|
||||||
poetry run python main.py
|
poetry run python server.py
|
||||||
```
|
```
|
||||||
### Интеграция в Core
|
### Интеграция в Core
|
||||||
|
|
||||||
|
99
auth.py
99
auth.py
@@ -1,51 +1,74 @@
|
|||||||
from functools import wraps
|
from functools import wraps
|
||||||
import aiohttp
|
|
||||||
from aiohttp import web
|
|
||||||
|
|
||||||
AUTH_URL = 'https://auth.discours.io'
|
import aiohttp
|
||||||
|
from starlette.responses import JSONResponse
|
||||||
|
|
||||||
|
from logger import root_logger as logger
|
||||||
|
from settings import AUTH_URL
|
||||||
|
|
||||||
|
|
||||||
|
async def request_data(gql, headers=None):
|
||||||
|
if headers is None:
|
||||||
|
headers = {"Content-Type": "application/json"}
|
||||||
|
try:
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
async with session.post(AUTH_URL, json=gql, headers=headers) as response:
|
||||||
|
if response.status == 200:
|
||||||
|
data = await response.json()
|
||||||
|
errors = data.get("errors")
|
||||||
|
if errors:
|
||||||
|
logger.error(f"HTTP Errors: {errors}")
|
||||||
|
else:
|
||||||
|
return data
|
||||||
|
except Exception as e:
|
||||||
|
# Handling and logging exceptions during authentication check
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
logger.error(f"request_data error: {e}")
|
||||||
|
logger.error(traceback.format_exc())
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
async def check_auth(req):
|
async def check_auth(req):
|
||||||
|
logger.info('checking auth token')
|
||||||
|
logger.debug(req.headers)
|
||||||
token = req.headers.get("Authorization")
|
token = req.headers.get("Authorization")
|
||||||
headers = {"Authorization": token, "Content-Type": "application/json"} # "Bearer " + removed
|
user_id = ""
|
||||||
print(f"[services.auth] checking auth token: {token}")
|
if token:
|
||||||
|
# Logging the authentication token
|
||||||
|
logger.debug(f"{token}")
|
||||||
|
query_name = "validate_jwt_token"
|
||||||
|
operation = "ValidateToken"
|
||||||
|
variables = {"params": {"token_type": "access_token", "token": token}}
|
||||||
|
|
||||||
query_name = "session"
|
gql = {
|
||||||
query_type = "query"
|
"query": f"query {operation}($params: ValidateJWTTokenInput!) {{"
|
||||||
operation = "GetUserId"
|
+ f"{query_name}(params: $params) {{ is_valid claims }} "
|
||||||
|
+ "}",
|
||||||
gql = {
|
"variables": variables,
|
||||||
"query": query_type + " " + operation + " { " + query_name + " { user { id } } }",
|
"operationName": operation,
|
||||||
"operationName": operation,
|
}
|
||||||
"variables": None,
|
data = await request_data(gql)
|
||||||
}
|
if data:
|
||||||
|
logger.debug(data)
|
||||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30.0)) as session:
|
user_data = data.get("data", {}).get(query_name, {}).get("claims", {})
|
||||||
async with session.post(AUTH_URL, headers=headers, json=gql) as response:
|
user_id = user_data.get("sub", "")
|
||||||
print(f"[services.auth] {AUTH_URL} response: {response.status}")
|
return user_id
|
||||||
if response.status != 200:
|
|
||||||
return False, None
|
|
||||||
r = await response.json()
|
|
||||||
if r:
|
|
||||||
user_id = r.get("data", {}).get(query_name, {}).get("user", {}).get("id", None)
|
|
||||||
is_authenticated = user_id is not None
|
|
||||||
return is_authenticated, user_id
|
|
||||||
return False, None
|
|
||||||
|
|
||||||
|
|
||||||
def login_required(f):
|
def login_required(f):
|
||||||
@wraps(f)
|
"""
|
||||||
async def decorated_function(*args, **kwargs):
|
A decorator that requires user authentication before accessing a route.
|
||||||
info = args[1]
|
"""
|
||||||
context = info.context
|
|
||||||
req = context.get("request")
|
|
||||||
is_authenticated, user_id = await check_auth(req)
|
|
||||||
if not is_authenticated:
|
|
||||||
raise web.HTTPUnauthorized(text="You are not logged in") # Return HTTP 401 Unauthorized
|
|
||||||
else:
|
|
||||||
context["user_id"] = user_id
|
|
||||||
|
|
||||||
# If the user is authenticated, execute the resolver
|
@wraps(f)
|
||||||
return await f(*args, **kwargs)
|
async def decorated_function(req, *args, **kwargs):
|
||||||
|
user_id = await check_auth(req)
|
||||||
|
if user_id:
|
||||||
|
logger.info(f" got {user_id}")
|
||||||
|
req.state.user_id = user_id.strip()
|
||||||
|
return await f(req, *args, **kwargs)
|
||||||
|
else:
|
||||||
|
return JSONResponse({"detail": "Not authorized"}, status_code=401)
|
||||||
|
|
||||||
return decorated_function
|
return decorated_function
|
||||||
|
80
logger.py
Normal file
80
logger.py
Normal file
@@ -0,0 +1,80 @@
|
|||||||
|
import logging
|
||||||
|
|
||||||
|
import colorlog
|
||||||
|
|
||||||
|
# Define the color scheme
|
||||||
|
color_scheme = {
|
||||||
|
"DEBUG": "cyan",
|
||||||
|
"INFO": "green",
|
||||||
|
"WARNING": "yellow",
|
||||||
|
"ERROR": "red",
|
||||||
|
"CRITICAL": "red,bg_white",
|
||||||
|
"DEFAULT": "white",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Define secondary log colors
|
||||||
|
secondary_colors = {
|
||||||
|
"log_name": {"DEBUG": "blue"},
|
||||||
|
"asctime": {"DEBUG": "cyan"},
|
||||||
|
"process": {"DEBUG": "purple"},
|
||||||
|
"module": {"DEBUG": "cyan,bg_blue"},
|
||||||
|
"funcName": {"DEBUG": "light_white,bg_blue"},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Define the log format string
|
||||||
|
fmt_string = "%(log_color)s%(levelname)s: %(log_color)s[%(module)s.%(funcName)s]%(reset)s %(white)s%(message)s"
|
||||||
|
|
||||||
|
# Define formatting configuration
|
||||||
|
fmt_config = {
|
||||||
|
"log_colors": color_scheme,
|
||||||
|
"secondary_log_colors": secondary_colors,
|
||||||
|
"style": "%",
|
||||||
|
"reset": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MultilineColoredFormatter(colorlog.ColoredFormatter):
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.log_colors = kwargs.pop("log_colors", {})
|
||||||
|
self.secondary_log_colors = kwargs.pop("secondary_log_colors", {})
|
||||||
|
|
||||||
|
def format(self, record):
|
||||||
|
message = record.getMessage()
|
||||||
|
if "\n" in message:
|
||||||
|
lines = message.split("\n")
|
||||||
|
first_line = lines[0]
|
||||||
|
record.message = first_line
|
||||||
|
formatted_first_line = super().format(record)
|
||||||
|
formatted_lines = [formatted_first_line]
|
||||||
|
for line in lines[1:]:
|
||||||
|
formatted_lines.append(line)
|
||||||
|
return "\n".join(formatted_lines)
|
||||||
|
else:
|
||||||
|
return super().format(record)
|
||||||
|
|
||||||
|
|
||||||
|
# Create a MultilineColoredFormatter object for colorized logging
|
||||||
|
formatter = MultilineColoredFormatter(fmt_string, **fmt_config)
|
||||||
|
|
||||||
|
# Create a stream handler for logging output
|
||||||
|
stream = logging.StreamHandler()
|
||||||
|
stream.setFormatter(formatter)
|
||||||
|
|
||||||
|
# Set up the root logger with the same formatting
|
||||||
|
root_logger = logging.getLogger()
|
||||||
|
root_logger.setLevel(logging.DEBUG)
|
||||||
|
root_logger.addHandler(stream)
|
||||||
|
|
||||||
|
ignore_logs = [
|
||||||
|
"_trace",
|
||||||
|
"httpx",
|
||||||
|
"_client",
|
||||||
|
"_trace.atrace",
|
||||||
|
"aiohttp",
|
||||||
|
"_client",
|
||||||
|
"._make_request",
|
||||||
|
]
|
||||||
|
for lgr in ignore_logs:
|
||||||
|
loggr = logging.getLogger(lgr)
|
||||||
|
loggr.setLevel(logging.INFO)
|
8
main.py
8
main.py
@@ -1,13 +1,13 @@
|
|||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
import uuid
|
import uuid
|
||||||
|
from auth import login_required
|
||||||
import boto3
|
import boto3
|
||||||
from botocore.exceptions import BotoCoreError, ClientError
|
from botocore.exceptions import BotoCoreError, ClientError
|
||||||
from starlette.applications import Starlette
|
from starlette.applications import Starlette
|
||||||
from starlette.responses import JSONResponse
|
from starlette.responses import JSONResponse
|
||||||
from starlette.routing import Route
|
from starlette.routing import Route
|
||||||
from starlette.requests import Request
|
from starlette.requests import Request
|
||||||
from auth import check_auth
|
|
||||||
|
|
||||||
|
|
||||||
STORJ_ACCESS_KEY = os.environ.get('STORJ_ACCESS_KEY')
|
STORJ_ACCESS_KEY = os.environ.get('STORJ_ACCESS_KEY')
|
||||||
@@ -17,7 +17,7 @@ STORJ_BUCKET_NAME = os.environ.get('STORJ_BUCKET_NAME')
|
|||||||
CDN_DOMAIN = os.environ.get('CDN_DOMAIN')
|
CDN_DOMAIN = os.environ.get('CDN_DOMAIN')
|
||||||
|
|
||||||
|
|
||||||
@check_auth
|
@login_required
|
||||||
async def upload_handler(request: Request):
|
async def upload_handler(request: Request):
|
||||||
form = await request.form()
|
form = await request.form()
|
||||||
file = form.get('file')
|
file = form.get('file')
|
||||||
@@ -59,12 +59,12 @@ async def upload_handler(request: Request):
|
|||||||
return JSONResponse({'error': 'Failed to upload file'}, status_code=500)
|
return JSONResponse({'error': 'Failed to upload file'}, status_code=500)
|
||||||
|
|
||||||
routes = [
|
routes = [
|
||||||
Route('/upload', upload_handler, methods=['POST']),
|
Route('/', upload_handler, methods=['POST']),
|
||||||
]
|
]
|
||||||
|
|
||||||
app = Starlette(debug=True, routes=routes)
|
app = Starlette(debug=True, routes=routes)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import uvicorn
|
import uvicorn
|
||||||
uvicorn.run(app, host='0.0.0.0', port=80)
|
uvicorn.run(app, host='0.0.0.0', port=8080)
|
||||||
|
|
@@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "discoursio-migrator"
|
name = "discoursio-migrator"
|
||||||
version = "0.2.6"
|
version = "0.3.0"
|
||||||
description = ""
|
description = ""
|
||||||
authors = ["discoursio devteam"]
|
authors = ["discoursio devteam"]
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
@@ -8,40 +8,27 @@ readme = "README.md"
|
|||||||
|
|
||||||
[tool.poetry.dependencies]
|
[tool.poetry.dependencies]
|
||||||
python = "^3.12"
|
python = "^3.12"
|
||||||
aiohttp = "^3.9.1"
|
|
||||||
uvicorn = "^0.24.0.post1"
|
|
||||||
starlette = "^0.33.0"
|
starlette = "^0.33.0"
|
||||||
boto3 = "^1.33.6"
|
aioboto3 = "^9.0.0"
|
||||||
botocore = "^1.33.6"
|
python-multipart = "^0.0.5"
|
||||||
|
colorlog = "^6.8.2"
|
||||||
[tool.poetry.dev-dependencies]
|
granian = "^1.3.1"
|
||||||
black = "^23.10.1"
|
aiohttp = "^3.9.5"
|
||||||
|
|
||||||
[tool.poetry.group.dev.dependencies]
|
[tool.poetry.group.dev.dependencies]
|
||||||
setuptools = "^69.0.2"
|
ruff = "^0.3.5"
|
||||||
|
isort = "^5.13.2"
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["poetry-core"]
|
requires = ["poetry-core>=1.0.0"]
|
||||||
build-backend = "poetry.core.masonry.api"
|
build-backend = "poetry.core.masonry.api"
|
||||||
|
|
||||||
[tool.black]
|
[tool.pyright]
|
||||||
line-length = 120
|
venvPath = "."
|
||||||
target-version = ['py312']
|
venv = ".venv"
|
||||||
include = '\.pyi?$'
|
|
||||||
exclude = '''
|
[tool.isort]
|
||||||
(
|
multi_line_output = 3
|
||||||
/(
|
include_trailing_comma = true
|
||||||
\.eggs
|
force_grid_wrap = 0
|
||||||
| \.git
|
line_length = 120
|
||||||
| \.hg
|
|
||||||
| \.mypy_cache
|
|
||||||
| \.tox
|
|
||||||
| \.venv
|
|
||||||
| _build
|
|
||||||
| buck-out
|
|
||||||
| build
|
|
||||||
| dist
|
|
||||||
)/
|
|
||||||
| foo.py
|
|
||||||
)
|
|
||||||
'''
|
|
||||||
|
26
server.py
Normal file
26
server.py
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
from granian.constants import Interfaces
|
||||||
|
from granian.server import Granian
|
||||||
|
import subprocess
|
||||||
|
from logger import root_logger as logger
|
||||||
|
from settings import PORT
|
||||||
|
|
||||||
|
|
||||||
|
def is_docker_container_running(name):
|
||||||
|
cmd = ["docker", "ps", "-f", f"name={name}"]
|
||||||
|
output = subprocess.run(cmd, capture_output=True, text=True).stdout
|
||||||
|
logger.info(output)
|
||||||
|
return name in output
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
logger.info("started")
|
||||||
|
|
||||||
|
granian_instance = Granian(
|
||||||
|
"main:app",
|
||||||
|
address="0.0.0.0", # noqa S104
|
||||||
|
port=PORT,
|
||||||
|
threads=4,
|
||||||
|
websockets=False,
|
||||||
|
interface=Interfaces.ASGI,
|
||||||
|
)
|
||||||
|
granian_instance.serve()
|
9
settings.py
Normal file
9
settings.py
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
from os import environ
|
||||||
|
|
||||||
|
PORT = 8080
|
||||||
|
AUTH_URL = environ.get("AUTH_URL") or "https://auth.discours.io/graphql"
|
||||||
|
STORJ_ACCESS_KEY = environ.get("STORJ_ACCESS_KEY")
|
||||||
|
STORJ_SECRET_KEY = environ.get("STORJ_SECRET_KEY")
|
||||||
|
STORJ_END_POINT = environ.get("STORJ_END_POINT")
|
||||||
|
STORJ_BUCKET_NAME = environ.get("STORJ_BUCKET_NAME")
|
||||||
|
CDN_DOMAIN = environ.get("CDN_DOMAIN")
|
Reference in New Issue
Block a user