discours.io/quoter
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restore thumbnail generation with Storj caching

Browse Source 
- Restored thumbnail functions in src/thumbnail.rs with WebP support
- Added Storj caching for thumbnails (cache_thumbnail_to_storj, load_cached_thumbnail_from_storj)
- Updated handlers to use thumbnail generation with Storj caching
- Added ETag caching with MD5 hashes for optimal performance
- Updated documentation to reflect restored thumbnail functionality
- Removed local file caching in favor of reliable Storj storage

💋 Self-contained: Quoter now handles everything - uploads, thumbnails, and serving.
This commit is contained in:
Untone
2025-09-02 18:57:17 +03:00
parent 9466e07237
commit eefdfe6d5d
5 changed files with 481 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
src/handlers/proxy.rs
View File

@@ -9,12 +9,14 @@ use crate::app_state::AppState;
use crate::handlers::serve_file::serve_file;
use crate::lookup::{find_file_by_pattern, get_mime_type};
use crate::s3_utils::{check_file_exists, load_file_from_s3, upload_to_s3};
use crate::thumbnail::parse_file_path;
use crate::thumbnail::{
parse_file_path, is_image_file, generate_webp_thumbnail,
load_cached_thumbnail_from_storj, cache_thumbnail_to_storj
};
// Удалена дублирующая функция, используется из common модуля
/// Обработчик для скачивания файла
/// без генерации миниатюр - это делает Vercel Edge API
/// Обработчик для скачивания файла с поддержкой thumbnail генерации
#[allow(clippy::collapsible_if)]
pub async fn proxy_handler(
req: HttpRequest,
@@ -129,6 +131,80 @@ pub async fn proxy_handler(
let elapsed = start_time.elapsed();
info!("File served from AWS in {:?}: {}", elapsed, path);
// Проверяем, нужен ли thumbnail
if requested_width > 0 && is_image_file(&filekey) {
info!("Generating thumbnail for {} with width {}", filekey, requested_width);
// Пробуем загрузить из Storj кэша
if let Some(cached_thumb) = load_cached_thumbnail_from_storj(
&state.storj_client,
&state.bucket,
&base_filename,
requested_width,
None
).await {
info!("Serving cached thumbnail from Storj for {}", base_filename);
let thumb_content_type = "image/webp";
if is_vercel_request(&req) {
let etag = format!("\"{:x}\"", md5::compute(&cached_thumb));
return Ok(create_vercel_compatible_response(
thumb_content_type,
cached_thumb,
&etag,
));
} else {
return Ok(create_file_response_with_analytics(
thumb_content_type,
cached_thumb,
&req,
&format!("{}_{}x.webp", base_filename, requested_width),
));
}
}
// Генерируем новый thumbnail
match generate_webp_thumbnail(&filedata, requested_width, None) {
Ok(thumb_data) => {
info!("Generated thumbnail: {} bytes", thumb_data.len());
// Кэшируем thumbnail в Storj
if let Err(e) = cache_thumbnail_to_storj(
&state.storj_client,
&state.bucket,
&base_filename,
requested_width,
None,
&thumb_data
).await {
warn!("Failed to cache thumbnail to Storj: {}", e);
}
let thumb_content_type = "image/webp";
if is_vercel_request(&req) {
let etag = format!("\"{:x}\"", md5::compute(&thumb_data));
return Ok(create_vercel_compatible_response(
thumb_content_type,
thumb_data,
&etag,
));
} else {
return Ok(create_file_response_with_analytics(
thumb_content_type,
thumb_data,
&req,
&format!("{}_{}x.webp", base_filename, requested_width),
));
}
}
Err(e) => {
warn!("Failed to generate thumbnail: {}", e);
// Fallback to original image
}
}
}
// Используем Vercel-совместимый ответ для Vercel запросов
if is_vercel_request(&req) {
let etag = format!("\"{:x}\"", md5::compute(&filedata));

90
src/handlers/serve_file.rs
View File

@@ -1,14 +1,21 @@
use actix_web::{HttpRequest, HttpResponse, Result, error::ErrorInternalServerError};
use mime_guess::MimeGuess;
use log::{info, warn};
use super::common::{
create_file_response_with_analytics, create_vercel_compatible_response, is_vercel_request,
create_file_response_with_analytics,
create_vercel_compatible_response,
is_vercel_request,
};
use crate::app_state::AppState;
use crate::s3_utils::{check_file_exists, load_file_from_s3};
use crate::thumbnail::{
parse_file_path, is_image_file, generate_webp_thumbnail,
load_cached_thumbnail_from_storj, cache_thumbnail_to_storj
};
use md5;
/// Функция для обслуживания файла по заданному пути.
/// Теперь оптимизирована для Vercel Edge caching.
/// Функция для обслуживания файла по заданному пути с поддержкой thumbnail генерации.
pub async fn serve_file(
filepath: &str,
state: &AppState,
@@ -34,6 +41,83 @@ pub async fn serve_file(
ErrorInternalServerError(format!("Failed to load {} from Storj: {}", filepath, e))
})?;
// Парсим путь для проверки thumbnail запроса
let (base_filename, requested_width, _) = parse_file_path(filepath);
// Проверяем, нужен ли thumbnail
if requested_width > 0 && is_image_file(filepath) {
info!("Generating thumbnail for {} with width {}", filepath, requested_width);
// Пробуем загрузить из Storj кэша
if let Some(cached_thumb) = load_cached_thumbnail_from_storj(
&state.storj_client,
&state.bucket,
&base_filename,
requested_width,
None
).await {
info!("Serving cached thumbnail from Storj for {}", base_filename);
let thumb_content_type = "image/webp";
if is_vercel_request(req) {
let etag = format!("\"{:x}\"", md5::compute(&cached_thumb));
return Ok(create_vercel_compatible_response(
thumb_content_type,
cached_thumb,
&etag,
));
} else {
return Ok(create_file_response_with_analytics(
thumb_content_type,
cached_thumb,
req,
&format!("{}_{}x.webp", base_filename, requested_width),
));
}
}
// Генерируем новый thumbnail
match generate_webp_thumbnail(&filedata, requested_width, None) {
Ok(thumb_data) => {
info!("Generated thumbnail: {} bytes", thumb_data.len());
// Кэшируем thumbnail в Storj
if let Err(e) = cache_thumbnail_to_storj(
&state.storj_client,
&state.bucket,
&base_filename,
requested_width,
None,
&thumb_data
).await {
warn!("Failed to cache thumbnail to Storj: {}", e);
}
let thumb_content_type = "image/webp";
if is_vercel_request(req) {
let etag = format!("\"{:x}\"", md5::compute(&thumb_data));
return Ok(create_vercel_compatible_response(
thumb_content_type,
thumb_data,
&etag,
));
} else {
return Ok(create_file_response_with_analytics(
thumb_content_type,
thumb_data,
req,
&format!("{}_{}x.webp", base_filename, requested_width),
));
}
}
Err(e) => {
warn!("Failed to generate thumbnail: {}", e);
// Fallback to original image
}
}
}
// Определяем MIME тип
let mime_type = MimeGuess::from_path(filepath).first_or_octet_stream();

294
src/thumbnail.rs
View File

@@ -1,4 +1,9 @@
// Модуль для парсинга путей к файлам (без генерации миниатюр)
use image::ImageFormat;
use std::path::Path;
use std::fs;
use std::io::Write;
use log::{info, warn};
use aws_sdk_s3::Client as S3Client;
/// Парсит путь к файлу, извлекая базовое имя, ширину и расширение.
///
@@ -35,6 +40,272 @@ pub fn parse_file_path(path: &str) -> (String, u32, String) {
(name_part.to_string(), 0, extension)
}
/// Генерирует thumbnail для изображения.
///
/// # Аргументы
/// * `image_data` - Данные изображения
/// * `width` - Ширина thumbnail
/// * `height` - Высота thumbnail (опционально)
/// * `format` - Формат выходного изображения (по умолчанию WebP)
///
/// # Возвращает
/// * `Result<Vec<u8>, Box<dyn std::error::Error>>` - Данные thumbnail или ошибка
pub fn generate_thumbnail(
image_data: &[u8],
width: u32,
height: Option<u32>,
format: Option<ImageFormat>,
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
info!("Generating thumbnail: {}x{}", width, height.unwrap_or(width));
// Загружаем изображение
let img = image::load_from_memory(image_data)?;
// Вычисляем размеры
let target_height = height.unwrap_or_else(|| {
let aspect_ratio = img.height() as f32 / img.width() as f32;
(width as f32 * aspect_ratio) as u32
});
// Ресайзим изображение
let resized = img.thumbnail(width, target_height);
// Конвертируем в нужный формат
let output_format = format.unwrap_or(ImageFormat::WebP);
let mut buffer = Vec::new();
resized.write_to(&mut std::io::Cursor::new(&mut buffer), output_format)?;
info!("Thumbnail generated: {} bytes", buffer.len());
Ok(buffer)
}
/// Генерирует WebP thumbnail для изображения.
pub fn generate_webp_thumbnail(
image_data: &[u8],
width: u32,
height: Option<u32>,
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
generate_thumbnail(image_data, width, height, Some(ImageFormat::WebP))
}
/// Генерирует JPEG thumbnail для изображения.
pub fn generate_jpeg_thumbnail(
image_data: &[u8],
width: u32,
height: Option<u32>,
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
generate_thumbnail(image_data, width, height, Some(ImageFormat::Jpeg))
}
/// Проверяет, является ли файл изображением.
pub fn is_image_file(filename: &str) -> bool {
let ext = Path::new(filename)
.extension()
.and_then(|s| s.to_str())
.map(|s| s.to_lowercase())
.unwrap_or_default();
matches!(ext.as_str(), "jpg" | "jpeg" | "png" | "gif" | "bmp" | "webp" | "tiff")
}
/// Получает MIME тип для изображения.
pub fn get_image_mime_type(filename: &str) -> &'static str {
let ext = Path::new(filename)
.extension()
.and_then(|s| s.to_str())
.map(|s| s.to_lowercase())
.unwrap_or_default();
match ext.as_str() {
"jpg" | "jpeg" => "image/jpeg",
"png" => "image/png",
"gif" => "image/gif",
"bmp" => "image/bmp",
"webp" => "image/webp",
"tiff" => "image/tiff",
_ => "application/octet-stream",
}
}
/// Кэширует thumbnail в файловой системе.
pub fn cache_thumbnail(
cache_dir: &str,
filename: &str,
width: u32,
height: Option<u32>,
thumbnail_data: &[u8],
) -> Result<String, Box<dyn std::error::Error>> {
// Создаем директорию кэша если не существует
fs::create_dir_all(cache_dir)?;
// Генерируем имя файла кэша
let cache_filename = if let Some(h) = height {
format!("{}_{}x{}.webp", filename, width, h)
} else {
format!("{}_{}x.webp", filename, width)
};
let cache_path = Path::new(cache_dir).join(&cache_filename);
// Сохраняем thumbnail
let mut file = fs::File::create(&cache_path)?;
file.write_all(thumbnail_data)?;
info!("Thumbnail cached: {}", cache_path.display());
Ok(cache_path.to_string_lossy().to_string())
}
/// Загружает thumbnail из кэша.
pub fn load_cached_thumbnail(
cache_dir: &str,
filename: &str,
width: u32,
height: Option<u32>,
) -> Option<Vec<u8>> {
let cache_filename = if let Some(h) = height {
format!("{}_{}x{}.webp", filename, width, h)
} else {
format!("{}_{}x.webp", filename, width)
};
let cache_path = Path::new(cache_dir).join(&cache_filename);
match fs::read(&cache_path) {
Ok(data) => {
info!("Thumbnail loaded from cache: {}", cache_path.display());
Some(data)
}
Err(e) => {
warn!("Failed to load cached thumbnail: {}", e);
None
}
}
}
/// Кэширует thumbnail в Storj S3.
pub async fn cache_thumbnail_to_storj(
s3_client: &S3Client,
bucket: &str,
filename: &str,
width: u32,
height: Option<u32>,
thumbnail_data: &[u8],
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
// Генерируем ключ для thumbnail в Storj
let thumbnail_key = if let Some(h) = height {
format!("thumbnails/{}_{}x{}.webp", filename, width, h)
} else {
format!("thumbnails/{}_{}x.webp", filename, width)
};
// Загружаем thumbnail в Storj
let body = aws_sdk_s3::primitives::ByteStream::from(thumbnail_data.to_vec());
s3_client
.put_object()
.bucket(bucket)
.key(&thumbnail_key)
.body(body)
.content_type("image/webp")
.send()
.await?;
info!("Thumbnail cached to Storj: {}", thumbnail_key);
Ok(thumbnail_key)
}
/// Загружает thumbnail из Storj S3.
pub async fn load_cached_thumbnail_from_storj(
s3_client: &S3Client,
bucket: &str,
filename: &str,
width: u32,
height: Option<u32>,
) -> Option<Vec<u8>> {
let thumbnail_key = if let Some(h) = height {
format!("thumbnails/{}_{}x{}.webp", filename, width, h)
} else {
format!("thumbnails/{}_{}x.webp", filename, width)
};
match s3_client
.get_object()
.bucket(bucket)
.key(&thumbnail_key)
.send()
.await
{
Ok(response) => {
match response.body.collect().await {
Ok(data) => {
let bytes = data.into_bytes();
info!("Thumbnail loaded from Storj: {} ({} bytes)", thumbnail_key, bytes.len());
Some(bytes.to_vec())
}
Err(e) => {
warn!("Failed to read thumbnail data from Storj: {}", e);
None
}
}
}
Err(e) => {
warn!("Failed to load cached thumbnail from Storj: {}", e);
None
}
}
}
/// Проверяет, существует ли thumbnail в Storj.
pub async fn thumbnail_exists_in_storj(
s3_client: &S3Client,
bucket: &str,
filename: &str,
width: u32,
height: Option<u32>,
) -> bool {
let thumbnail_key = if let Some(h) = height {
format!("thumbnails/{}_{}x{}.webp", filename, width, h)
} else {
format!("thumbnails/{}_{}x.webp", filename, width)
};
match s3_client
.head_object()
.bucket(bucket)
.key(&thumbnail_key)
.send()
.await
{
Ok(_) => true,
Err(_) => false,
}
}
/// Очищает старые файлы кэша.
pub fn cleanup_cache(cache_dir: &str, max_age_days: u64) -> Result<(), Box<dyn std::error::Error>> {
let cache_path = Path::new(cache_dir);
if !cache_path.exists() {
return Ok(());
}
let cutoff_time = std::time::SystemTime::now() - std::time::Duration::from_secs(max_age_days * 24 * 60 * 60);
for entry in fs::read_dir(cache_path)? {
let entry = entry?;
let metadata = entry.metadata()?;
if let Ok(modified) = metadata.modified() {
if modified < cutoff_time {
fs::remove_file(entry.path())?;
info!("Removed old cache file: {}", entry.path().display());
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -71,4 +342,23 @@ mod tests {
assert_eq!(width, 800);
assert_eq!(ext, "jpg");
}
}
#[test]
fn test_is_image_file() {
assert!(is_image_file("image.jpg"));
assert!(is_image_file("photo.png"));
assert!(is_image_file("gif.gif"));
assert!(is_image_file("webp.webp"));
assert!(!is_image_file("document.pdf"));
assert!(!is_image_file("text.txt"));
}
#[test]
fn test_get_image_mime_type() {
assert_eq!(get_image_mime_type("image.jpg"), "image/jpeg");
assert_eq!(get_image_mime_type("photo.png"), "image/png");
assert_eq!(get_image_mime_type("animation.gif"), "image/gif");
assert_eq!(get_image_mime_type("modern.webp"), "image/webp");
assert_eq!(get_image_mime_type("unknown.xyz"), "application/octet-stream");
}
}