proxy-reworked

src/app_state.rs

@@ -3,7 +3,6 @@ use aws_config::BehaviorVersion;
 use aws_sdk_s3::{config::Credentials, Client as S3Client};
 use redis::{aio::MultiplexedConnection, AsyncCommands, Client as RedisClient};
 use std::env;
-use std::collections::HashMap;
 use log::info;
 use crate::s3_utils::get_s3_filelist;
 
@@ -12,7 +11,8 @@ use crate::s3_utils::get_s3_filelist;
 pub struct AppState {
     pub redis: MultiplexedConnection,
     pub storj_client: S3Client,
-    pub storj_bucket: String
+    pub aws_client: S3Client,
+    pub bucket: String
 }
 
 const PATH_MAPPING_KEY: &str = "filepath_mapping"; // Ключ для хранения маппинга путей
@@ -34,14 +34,13 @@ impl AppState {
         let s3_secret_key = env::var("STORJ_SECRET_KEY").expect("STORJ_SECRET_KEY must be set");
         let s3_endpoint = env::var("STORJ_END_POINT")
             .unwrap_or_else(|_| "https://gateway.storjshare.io".to_string());
-        let storj_bucket = env::var("STORJ_BUCKET_NAME").unwrap_or_else(|_| "discours-io".to_string());
+        let bucket = env::var("STORJ_BUCKET_NAME").unwrap_or_else(|_| "discours-io".to_string());
 
         // Получаем конфигурацию для AWS S3
-        // let aws_access_key = env::var("AWS_ACCESS_KEY").expect("AWS_ACCESS_KEY must be set");
+        let aws_access_key = env::var("AWS_ACCESS_KEY").expect("AWS_ACCESS_KEY must be set");
-        // let aws_secret_key = env::var("AWS_SECRET_KEY").expect("AWS_SECRET_KEY must be set");
+        let aws_secret_key = env::var("AWS_SECRET_KEY").expect("AWS_SECRET_KEY must be set");
-        // let aws_endpoint =
+        let aws_endpoint =
-        //    env::var("AWS_END_POINT").unwrap_or_else(|_| "https://s3.amazonaws.com".to_string());
+           env::var("AWS_END_POINT").unwrap_or_else(|_| "https://s3.amazonaws.com".to_string());
-        // let aws_bucket = env::var("AWS_BUCKET_NAME").unwrap_or_else(|_| "discours-io".to_string());
 
         // Конфигурируем клиент S3 для Storj
         let storj_config = aws_config::defaults(BehaviorVersion::latest())
@@ -60,7 +59,7 @@ impl AppState {
         let storj_client = S3Client::new(&storj_config);
 
         // Конфигурируем клиент S3 для AWS
-        /* let aws_config = aws_config::defaults(BehaviorVersion::latest())
+        let aws_config = aws_config::defaults(BehaviorVersion::latest())
             .region("eu-west-1")
             .endpoint_url(aws_endpoint)
             .credentials_provider(Credentials::new(
@@ -72,54 +71,42 @@ impl AppState {
             ))
             .load()
             .await;
-        */
+        
-        // let aws_client = S3Client::new(&aws_config);
+        let aws_client = S3Client::new(&aws_config);
 
         let app_state = AppState {
             redis: redis_connection,
             storj_client,
-            storj_bucket
+            aws_client,
+            bucket
         };
 
-        // Кэшируем список файлов из Storj S3 при старте приложения
+        // Кэшируем список файлов из AWS при старте приложения
-        app_state.cache_storj_filelist().await;
+        app_state.cache_filelist().await;
 
         app_state
     }
 
     /// Кэширует список файлов из Storj S3 в Redis.
-    pub async fn cache_storj_filelist(&self) {
+    pub async fn cache_filelist(&self) {
         info!("caching storj filelist...");
         let mut redis = self.redis.clone();
         
         // Запрашиваем список файлов из Storj S3
-        let filelist = get_s3_filelist(&self.storj_client, &self.storj_bucket).await;
+        let filelist = get_s3_filelist(&self.aws_client, &self.bucket).await;
 
         for [filename, filepath] in filelist.clone() {
-            if !filepath.starts_with("development") {
+            // Сохраняем список файлов в Redis, используя HSET для каждого файла
-                // Сохраняем список файлов в Redis, используя HSET для каждого файла
+            let _: () = redis
-                let _: () = redis
+                .hset(PATH_MAPPING_KEY, filename.clone(), filepath)
-                    .hset(PATH_MAPPING_KEY, filename.clone(), filepath)
+                .await
-                    .await
+                .expect(&format!("Failed to cache file {} in Redis", filename));
-                    .expect(&format!("Failed to cache file {} in Redis", filename));
-            }
         }
 
         info!("cached {} files", filelist.len());
     }
-
-    /// Получает кэшированный список файлов из Redis.
-    pub async fn get_cached_file_list(&self) -> Vec<String> {
-        let mut redis = self.redis.clone();
-
-        // Пытаемся получить кэшированный список из Redis
-        let cached_list: HashMap<String, String> = redis.hgetall(PATH_MAPPING_KEY).await.unwrap_or_default();
-
-        // Преобразуем HashMap в Vec<String>, используя значения (пути файлов)
-        cached_list.into_values().collect()
-    }
     
-    /// Получает путь в Storj из ключа (имени файла) в Redis.
+    /// Получает путь из ключа (имени файла) в Redis.
     pub async fn get_path(&self, filename: &str) -> Result<Option<String>, actix_web::Error> {
         let mut redis = self.redis.clone();
         let new_path: Option<String> = redis

src/handlers/proxy.rs

@@ -1,9 +1,8 @@
 use actix_web::{error::ErrorInternalServerError, web, HttpRequest, HttpResponse, Result};
-use log::{info, warn, error};
 
 use crate::app_state::AppState;
-use crate::thumbnail::{parse_thumbnail_request, find_closest_width, generate_thumbnails};
+use crate::thumbnail::{find_closest_width, generate_thumbnails, parse_image_request};
-use crate::s3_utils::{load_file_from_s3, upload_to_s3};
+use crate::s3_utils::{check_file_exists, load_file_from_s3, upload_to_s3};
 use crate::handlers::serve_file::serve_file;
 
 /// Обработчик для скачивания файла и генерации миниатюры, если она недоступна.
@@ -12,82 +11,105 @@ pub async fn proxy_handler(
     requested_res: web::Path<String>,
     state: web::Data<AppState>,
 ) -> Result<HttpResponse, actix_web::Error> {
-    info!("requested_path: {}", requested_res);
+    let normalized_path = match requested_res.ends_with("/webp") {
-    let requested_path = requested_res.replace("/webp", "");
+        true => requested_res.replace("/webp", ""),
-    let parts = requested_path.split('/').collect::<Vec<&str>>(); // Explicit type annotation
+        false => requested_res.to_string(),
-    let filename = parts[parts.len()-1];
-
-    let stored_path = match state.get_path(&filename).await {
-        Ok(Some(path)) => path,
-        Ok(None) => {
-            warn!("wrong filename: {}", filename);
-            return Ok(HttpResponse::NotFound().finish());
-        }
-        Err(e) => {
-            warn!("error: {}", e);
-            return Ok(HttpResponse::InternalServerError().finish());
-        }
     };
-    info!("stored path: {}", stored_path);
 
-    // Проверяем, запрошена ли миниатюра
+    // парсим GET запрос
-    if let Some((base_filename, requested_width, extension)) =
+    if let Some((base_filename, requested_width, extension)) = parse_image_request(&normalized_path) {
-        parse_thumbnail_request(&requested_res)
+        let filekey = format!("{}.{}", base_filename, extension);
-    {
+        let content_type = match extension.as_str() {
-        info!("thumbnail requested: {}  width: {} ext: {}", base_filename, requested_width, extension);
+            "jpg" | "jpeg" => "image/jpeg",
+            "png" => "image/png",
+            "webp" => "image/webp",
+            "gif" => "image/gif",
+            "mp3" => "audio/mpeg",
+            "wav" => "audio/x-wav",
+            "ogg" => "audio/ogg",
+            "aac" => "audio/aac",
+            "m4a" => "audio/m4a",
+            "flac" => "audio/flac",
+            _ => return Err(ErrorInternalServerError("unsupported file format"))
+        };
 
-        // Находим ближайший подходящий размер
+        return match state.get_path(&filekey).await {
-        let closest_width = find_closest_width(requested_width);
+            Ok(Some(stored_path)) => {
-        let thumb_filename = format!("{}_{}.jpg", base_filename, closest_width);
-        info!("closest width: {}, thumb_filename: {}", closest_width, thumb_filename);
 
-        // Проверяем наличие миниатюры в кэше
+                // we have stored file path in storj
-        let cached_files = state.get_cached_file_list().await;
+                if check_file_exists(&state.storj_client, &state.bucket, &stored_path).await? {
-        if !cached_files.contains(&thumb_filename) {
+                    if content_type.starts_with("image") {
-            info!("no thumb found");
+                        return match requested_width == 0 {
-            if cached_files.contains(&base_filename) {
+                            true => serve_file(&stored_path, &state).await,
-                info!("no original file found");
+                            false => {
-                // Загружаем оригинальный файл из S3
+                                // find closest thumb width
-                let original_data: Vec<u8> =
+                                let closest: u32 = find_closest_width(requested_width as u32);
-                    load_file_from_s3(&state.storj_client, &state.storj_bucket, &base_filename).await?;
+                                let thumb_filename = &format!("{}_{}.{}", base_filename, closest, extension);
+    
+                                return match check_file_exists(&state.storj_client, &state.bucket, thumb_filename).await {
+                                    Ok(true) => serve_file(thumb_filename, &state).await,
+                                    Ok(false) => {
+                                        if let Ok(filedata) = load_file_from_s3(
+                                            &state.storj_client, &state.bucket, &stored_path).await {
+                                                thumbdata_save(&filedata, &state, &base_filename, content_type).await;
+                                                serve_file(thumb_filename, &state).await
+                                            } else {
+                                                Err(ErrorInternalServerError("cannot generate thumbnail"))
+                                            }
+                                    }
+                                    Err(_) => Err(ErrorInternalServerError("failed to load thumbnail"))
+                                }
+                            }
+                        }
+                    }
+                    // not image passing thumb generation
+                }
 
-                // Генерируем миниатюру для ближайшего подходящего размера
+                // we need to download what stored_path keeping in aws
-                let image = image::load_from_memory(&original_data).map_err(|_| {
+                return match load_file_from_s3(
-                    ErrorInternalServerError("Failed to load image for thumbnail generation")
+                    &state.aws_client, 
-                })?;
+                    &state.bucket, 
-                let thumbnails_bytes =
+                    &stored_path).await {
-                    generate_thumbnails(&image).await?;
+                        Ok(filedata) => {
-                let thumbnail_bytes = thumbnails_bytes[&closest_width].clone();
+                            let _ = upload_to_s3(&state.storj_client, &state.bucket, &filekey, filedata.clone(), content_type).await;
-
+                            thumbdata_save(&filedata, &state, &base_filename, content_type).await;
-                // Загружаем миниатюру в S3
+                            
-                upload_to_s3(
+                            Ok(HttpResponse::Ok()
-                    &state.storj_client,
+                                .content_type(content_type)
-                    &state.storj_bucket,
+                                .body(filedata)
-                    &thumb_filename,
+                            )
-                    thumbnail_bytes.clone(),
+                        },
-                    "image/jpeg",
+                        Err(err) => Err(ErrorInternalServerError(err)),
-                )
+                    }
-                .await?;
+            },
-                info!("thumb was saved in storj");
+            Ok(None) => Err(ErrorInternalServerError("requested file path was not found")),
-                return Ok(HttpResponse::Ok()
+            Err(e) => Err(ErrorInternalServerError(e))
-                    .content_type("image/jpeg")
-                    .body(thumbnail_bytes));
-            } else {
-                warn!("original was not found");
-            }
-        } else {
-            info!("thumb was found");
-            return serve_file(&thumb_filename, &state).await;
         }
     }
 
-    // Если запрошен целый файл
+    Err(ErrorInternalServerError("invalid file key"))
-    info!("serving full file: {}", requested_path);
+    
-    match serve_file(&requested_path, &state).await {
-        Ok(response) => Ok(response),
-        Err(e) => {
-            error!("error: {}", e);
-            Err(e)
-        }
-    }
 }
+
+async fn thumbdata_save(original_data: &[u8], state: &AppState, original_filename: &str, content_type: &str) {
+    if content_type.starts_with("image") {
+        let ext = original_filename.split('.').last().unwrap();
+        let img = image::load_from_memory(&original_data).unwrap();
+        
+        if let Ok(thumbnails_bytes) = generate_thumbnails(&img).await {
+            for (thumb_width, thumbnail) in thumbnails_bytes {
+                let thumb_filename = &format!("{}_{}.{}", original_filename, thumb_width, ext);
+                let thumbnail_bytes = thumbnail.clone();
+                // Загружаем миниатюру в S3
+                let _ = upload_to_s3(
+                    &state.storj_client,
+                    &state.bucket,
+                    thumb_filename,
+                    thumbnail_bytes,
+                    content_type,
+                )
+                .await;
+            }
+        }
+    }
+}

src/handlers/serve_file.rs

@@ -11,7 +11,7 @@ pub async fn serve_file(filepath: &str, state: &AppState) -> Result<HttpResponse
     }
 
     // Проверяем наличие файла в Storj S3
-    let exists = check_file_exists(&state.storj_client, &state.storj_bucket, &filepath).await?;
+    let exists = check_file_exists(&state.storj_client, &state.bucket, &filepath).await?;
     if !exists {
         return Err(ErrorInternalServerError(format!("File {} not found in Storj", filepath)));
     }
@@ -20,7 +20,7 @@ pub async fn serve_file(filepath: &str, state: &AppState) -> Result<HttpResponse
     let get_object_output = state
         .storj_client
         .get_object()
-        .bucket(&state.storj_bucket)
+        .bucket(&state.bucket)
         .key(filepath)
         .send()
         .await

src/handlers/upload.rs

@@ -62,7 +62,7 @@ pub async fn upload_handler(
             // Загружаем файл в S3
             upload_to_s3(
                 &state.storj_client,
-                &state.storj_bucket,
+                &state.bucket,
                 &body,
                 file_bytes,
                 &content_type,

src/main.rs

@@ -26,7 +26,7 @@ async fn main() -> std::io::Result<()> {
     spawn_blocking(move || {
         let rt = tokio::runtime::Handle::current();
         rt.block_on(async move {
-            app_state_clone.cache_storj_filelist().await;
+            app_state_clone.cache_filelist().await;
         });
     });
 

src/s3_utils.rs

@@ -42,11 +42,11 @@ pub async fn check_file_exists(
 
 /// Загружает файл из S3.
 pub async fn load_file_from_s3(
-    storj_client: &S3Client,
+    s3_client: &S3Client,
     bucket: &str,
     key: &str,
 ) -> Result<Vec<u8>, actix_web::Error> {
-    let get_object_output = storj_client
+    let get_object_output = s3_client
         .get_object()
         .bucket(bucket)
         .key(key)

src/thumbnail.rs

@@ -6,14 +6,14 @@ pub const THUMB_WIDTHS: [u32; 6] = [10, 40, 110, 300, 600, 800];
 
 /// Парсит запрос на миниатюру, извлекая оригинальное имя файла и требуемую ширину.
 /// Пример: "filename_150.ext" -> ("filename.ext", 150)
-pub fn parse_thumbnail_request(path: &str) -> Option<(String, u32, String)> {
+pub fn parse_image_request(path: &str) -> Option<(String, u32, String)> {
     if let Some((name_part, ext_part)) = path.rsplit_once('.') {
         if let Some((base_name, width_str)) = name_part.rsplit_once('_') {
             if let Ok(width) = width_str.parse::<u32>() {
                 return Some((base_name.to_string(), width, ext_part.to_string()));
             }
         }
-        return Some((name_part.to_string(), 0, ext_part.to_string()))
+        return Some((name_part.to_string(), 0, ext_part.to_string().to_lowercase()));
     }
     None
 }