discours.io/core
5
0
Fork 0
Code Pull Requests Actions Projects Activity

fmt

Browse Source
This commit is contained in:
Untone
2024-02-21 19:14:58 +03:00
parent 88cd6e1060
commit 214af0cf51
33 changed files with 750 additions and 773 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
services/search.py
View File

@@ -7,69 +7,60 @@ from opensearchpy import OpenSearch
from services.logger import root_logger as logger
from services.rediscache import redis
ELASTIC_HOST = os.environ.get("ELASTIC_HOST", "").replace("https://", "")
ELASTIC_USER = os.environ.get("ELASTIC_USER", "")
ELASTIC_PASSWORD = os.environ.get("ELASTIC_PASSWORD", "")
ELASTIC_PORT = os.environ.get("ELASTIC_PORT", 9200)
ELASTIC_AUTH = f"{ELASTIC_USER}:{ELASTIC_PASSWORD}" if ELASTIC_USER else ""
ELASTIC_HOST = os.environ.get('ELASTIC_HOST', '').replace('https://', '')
ELASTIC_USER = os.environ.get('ELASTIC_USER', '')
ELASTIC_PASSWORD = os.environ.get('ELASTIC_PASSWORD', '')
ELASTIC_PORT = os.environ.get('ELASTIC_PORT', 9200)
ELASTIC_AUTH = f'{ELASTIC_USER}:{ELASTIC_PASSWORD}' if ELASTIC_USER else ''
ELASTIC_URL = os.environ.get(
"ELASTIC_URL", f"https://{ELASTIC_AUTH}@{ELASTIC_HOST}:{ELASTIC_PORT}"
'ELASTIC_URL', f'https://{ELASTIC_AUTH}@{ELASTIC_HOST}:{ELASTIC_PORT}'
)
REDIS_TTL = 86400 # 1 day in seconds
index_settings = {
"settings": {
"index": {
"number_of_shards": 1,
"auto_expand_replicas": "0-all",
},
"analysis": {
"analyzer": {
"ru": {
"tokenizer": "standard",
"filter": ["lowercase", "ru_stop", "ru_stemmer"],
'settings': {
'index': {'number_of_shards': 1, 'auto_expand_replicas': '0-all'},
'analysis': {
'analyzer': {
'ru': {
'tokenizer': 'standard',
'filter': ['lowercase', 'ru_stop', 'ru_stemmer'],
}
},
"filter": {
"ru_stemmer": {
"type": "stemmer",
"language": "russian",
},
"ru_stop": {
"type": "stop",
"stopwords": "_russian_",
},
'filter': {
'ru_stemmer': {'type': 'stemmer', 'language': 'russian'},
'ru_stop': {'type': 'stop', 'stopwords': '_russian_'},
},
},
},
"mappings": {
"properties": {
"body": {"type": "text", "analyzer": "ru"},
"title": {"type": "text", "analyzer": "ru"},
'mappings': {
'properties': {
'body': {'type': 'text', 'analyzer': 'ru'},
'title': {'type': 'text', 'analyzer': 'ru'},
# 'author': {'type': 'text'},
}
},
}
expected_mapping = index_settings["mappings"]
expected_mapping = index_settings['mappings']
class SearchService:
def __init__(self, index_name="search_index"):
def __init__(self, index_name='search_index'):
self.index_name = index_name
self.manager = Manager()
self.client = None
# Используем менеджер для создания Lock и Value
self.lock = self.manager.Lock()
self.initialized_flag = self.manager.Value("i", 0)
self.initialized_flag = self.manager.Value('i', 0)
# Only initialize the instance if it's not already initialized
if not self.initialized_flag.value and ELASTIC_HOST:
try:
self.client = OpenSearch(
hosts=[{"host": ELASTIC_HOST, "port": ELASTIC_PORT}],
hosts=[{'host': ELASTIC_HOST, 'port': ELASTIC_PORT}],
http_compress=True,
http_auth=(ELASTIC_USER, ELASTIC_PASSWORD),
use_ssl=True,
@@ -78,34 +69,34 @@ class SearchService:
ssl_show_warn=False,
# ca_certs = ca_certs_path
)
logger.info(" Клиент OpenSearch.org подключен")
logger.info(' Клиент OpenSearch.org подключен')
if self.lock.acquire(blocking=False):
try:
self.check_index()
finally:
self.lock.release()
else:
logger.debug(" проверка пропущена")
logger.debug(' проверка пропущена')
except Exception as exc:
logger.error(f" {exc}")
logger.error(f' {exc}')
self.client = None
def info(self):
if isinstance(self.client, OpenSearch):
logger.info(" Поиск подключен") # : {self.client.info()}')
logger.info(' Поиск подключен') # : {self.client.info()}')
else:
logger.info(" * Задайте переменные среды для подключения к серверу поиска")
logger.info(' * Задайте переменные среды для подключения к серверу поиска')
def delete_index(self):
if self.client:
logger.debug(f" Удаляем индекс {self.index_name}")
logger.debug(f' Удаляем индекс {self.index_name}')
self.client.indices.delete(index=self.index_name, ignore_unavailable=True)
def create_index(self):
if self.client:
if self.lock.acquire(blocking=False):
try:
logger.debug(f" Создаём новый индекс: {self.index_name} ")
logger.debug(f' Создаём новый индекс: {self.index_name} ')
self.client.indices.create(
index=self.index_name, body=index_settings
)
@@ -114,11 +105,11 @@ class SearchService:
finally:
self.lock.release()
else:
logger.debug(" ..")
logger.debug(' ..')
def put_mapping(self):
if self.client:
logger.debug(f" Разметка индекации {self.index_name}")
logger.debug(f' Разметка индекации {self.index_name}')
self.client.indices.put_mapping(
index=self.index_name, body=expected_mapping
)
@@ -142,36 +133,28 @@ class SearchService:
finally:
self.lock.release()
else:
logger.debug(" ..")
logger.debug(' ..')
def index(self, shout):
if self.client:
id_ = str(shout.id)
logger.debug(f" Индексируем пост {id_}")
logger.debug(f' Индексируем пост {id_}')
self.client.index(index=self.index_name, id=id_, body=shout.dict())
async def search(self, text, limit, offset):
logger.debug(f" Ищем: {text}")
search_body = {
"query": {"match": {"_all": text}},
}
logger.debug(f' Ищем: {text}')
search_body = {'query': {'match': {'_all': text}}}
if self.client:
search_response = self.client.search(
index=self.index_name, body=search_body, size=limit, from_=offset
)
hits = search_response["hits"]["hits"]
hits = search_response['hits']['hits']
results = [
{
**hit["_source"],
"score": hit["_score"],
}
for hit in hits
]
results = [{**hit['_source'], 'score': hit['_score']} for hit in hits]
# Use Redis as cache with TTL
redis_key = f"search:{text}"
await redis.execute("SETEX", redis_key, REDIS_TTL, json.dumps(results))
redis_key = f'search:{text}'
await redis.execute('SETEX', redis_key, REDIS_TTL, json.dumps(results))
return []