2025-04-27 09:53:49 +00:00
|
|
|
|
"""
|
|
|
|
|
|
Модуль для обработки HTML-фрагментов
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
2025-05-16 06:23:48 +00:00
|
|
|
|
import trafilatura
|
|
|
|
|
|
|
2025-06-01 23:56:11 +00:00
|
|
|
|
from utils.logger import root_logger as logger
|
|
|
|
|
|
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
|
|
|
|
|
def extract_text(html: str) -> str:
|
|
|
|
|
|
"""
|
2025-06-01 23:56:11 +00:00
|
|
|
|
Извлекает чистый текст из HTML
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
|
|
|
|
|
Args:
|
2025-06-01 23:56:11 +00:00
|
|
|
|
html: HTML строка
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
|
|
|
|
|
Returns:
|
2025-06-01 23:56:11 +00:00
|
|
|
|
str: Извлеченный текст или пустая строка
|
2025-05-16 06:23:48 +00:00
|
|
|
|
"""
|
2025-06-01 23:56:11 +00:00
|
|
|
|
try:
|
|
|
|
|
|
result = trafilatura.extract(
|
|
|
|
|
|
html,
|
|
|
|
|
|
include_comments=False,
|
|
|
|
|
|
include_tables=True,
|
|
|
|
|
|
include_formatting=False,
|
|
|
|
|
|
favor_precision=True,
|
|
|
|
|
|
)
|
|
|
|
|
|
return result or ""
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
logger.error(f"Error extracting text: {e}")
|
|
|
|
|
|
return ""
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
|
|
|
|
|
|
2025-04-27 09:53:49 +00:00
|
|
|
|
def wrap_html_fragment(fragment: str) -> str:
|
|
|
|
|
|
"""
|
|
|
|
|
|
Оборачивает HTML-фрагмент в полную HTML-структуру для корректной обработки.
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
2025-04-27 09:53:49 +00:00
|
|
|
|
Args:
|
|
|
|
|
|
fragment: HTML-фрагмент для обработки
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
2025-04-27 09:53:49 +00:00
|
|
|
|
Returns:
|
|
|
|
|
|
str: Полный HTML-документ
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
2025-04-27 09:53:49 +00:00
|
|
|
|
Example:
|
|
|
|
|
|
>>> wrap_html_fragment("<p>Текст параграфа</p>")
|
|
|
|
|
|
'<!DOCTYPE html><html><head><meta charset="utf-8"></head><body><p>Текст параграфа</p></body></html>'
|
|
|
|
|
|
"""
|
|
|
|
|
|
if not fragment or not fragment.strip():
|
|
|
|
|
|
return fragment
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
2025-04-27 09:53:49 +00:00
|
|
|
|
# Проверяем, является ли контент полным HTML-документом
|
2025-05-16 06:23:48 +00:00
|
|
|
|
is_full_html = fragment.strip().startswith("<!DOCTYPE") or fragment.strip().startswith("<html")
|
|
|
|
|
|
|
2025-04-27 09:53:49 +00:00
|
|
|
|
# Если это фрагмент, оборачиваем его в полный HTML-документ
|
|
|
|
|
|
if not is_full_html:
|
|
|
|
|
|
return f"""<!DOCTYPE html>
|
|
|
|
|
|
<html>
|
|
|
|
|
|
<head>
|
|
|
|
|
|
<meta charset="utf-8">
|
|
|
|
|
|
<title></title>
|
|
|
|
|
|
</head>
|
|
|
|
|
|
<body>
|
|
|
|
|
|
{fragment}
|
|
|
|
|
|
</body>
|
|
|
|
|
|
</html>"""
|
2025-05-16 06:23:48 +00:00
|
|
|
|
|
|
|
|
|
|
return fragment
|
