heic-sys
Some checks failed
deploy / deploy (push) Failing after 6s

This commit is contained in:
Untone 2024-11-13 11:14:53 +03:00
parent fb1541f8e3
commit bc14d86018
9 changed files with 262 additions and 237 deletions

195
Cargo.lock generated
View File

@ -901,29 +901,6 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
[[package]]
name = "bindgen"
version = "0.69.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
dependencies = [
"bitflags 2.6.0",
"cexpr",
"clang-sys",
"itertools",
"lazy_static",
"lazycell",
"log",
"prettyplease",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"syn",
"which",
]
[[package]]
name = "bit_field"
version = "0.10.2"
@ -1045,12 +1022,14 @@ dependencies = [
]
[[package]]
name = "cexpr"
version = "0.6.0"
name = "cfb"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f"
dependencies = [
"nom",
"byteorder",
"fnv",
"uuid",
]
[[package]]
@ -1069,17 +1048,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clang-sys"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "color_quant"
version = "1.1.0"
@ -1346,9 +1314,12 @@ dependencies = [
"futures",
"image",
"imageproc",
"libheif-rs",
"infer",
"kamadak-exif",
"libheif-sys",
"log",
"mime_guess",
"once_cell",
"redis",
"reqwest",
"sentry",
@ -1417,17 +1388,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "enumn"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "env_filter"
version = "0.1.2"
@ -1565,12 +1525,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "four-cc"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "431a4c31778fde52b4400de34975f219eeca55cc829a9de157cd743a5b230ecb"
[[package]]
name = "futures"
version = "0.3.31"
@ -1699,12 +1653,6 @@ version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "group"
version = "0.12.1"
@ -1802,15 +1750,6 @@ dependencies = [
"digest",
]
[[package]]
name = "home"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "hostname"
version = "0.4.0"
@ -2226,6 +2165,15 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "infer"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199"
dependencies = [
"cfb",
]
[[package]]
name = "interpolate_name"
version = "0.2.4"
@ -2288,6 +2236,15 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "kamadak-exif"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef4fc70d0ab7e5b6bafa30216a6b48705ea964cdfc29c050f2412295eba58077"
dependencies = [
"mutate_once",
]
[[package]]
name = "language-tags"
version = "0.3.2"
@ -2300,12 +2257,6 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "lebe"
version = "0.5.2"
@ -2328,39 +2279,15 @@ dependencies = [
"cc",
]
[[package]]
name = "libheif-rs"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c32c0a0c970782707070f11c8612bced800b916af4ddaf6229161dc3ceb907"
dependencies = [
"enumn",
"four-cc",
"libc",
"libheif-sys",
]
[[package]]
name = "libheif-sys"
version = "2.1.1+1.17.4"
version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f60b29be1ef3ab2aba61344f09a18c3edf552bf4f9fbec9bd68b9ea6f98e71f8"
checksum = "2fec9617ceb95892391fba66dc1d559b3b15997844f5d36b17cb96ed86e0551c"
dependencies = [
"bindgen",
"libc",
"pkg-config",
"vcpkg",
"walkdir",
]
[[package]]
name = "libloading"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
dependencies = [
"cfg-if",
"windows-targets",
]
[[package]]
@ -2513,6 +2440,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "mutate_once"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16cf681a23b4d0a43fc35024c176437f9dcd818db34e0f42ab456a0ee5ad497b"
[[package]]
name = "nalgebra"
version = "0.32.6"
@ -2851,16 +2784,6 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "prettyplease"
version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033"
dependencies = [
"proc-macro2",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.89"
@ -3178,12 +3101,6 @@ version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc_version"
version = "0.4.1"
@ -3303,15 +3220,6 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.26"
@ -4157,16 +4065,6 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.1"
@ -4265,18 +4163,6 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082"
[[package]]
name = "which"
version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
dependencies = [
"either",
"home",
"once_cell",
"rustix",
]
[[package]]
name = "wide"
version = "0.7.28"
@ -4303,15 +4189,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"

View File

@ -26,7 +26,10 @@ env_logger = "0.11.5"
actix = "0.13.5"
imageproc = "0.25.0"
ab_glyph = "0.2.29"
libheif-rs = "1.0.2"
libheif-sys = "1.12"
once_cell = "1.18"
kamadak-exif = "0.5"
infer = "0.15"
[[bin]]
name = "quoter"

View File

@ -41,8 +41,10 @@ ENV RUST_LOG=warn
# Install runtime dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends libssl3 libheif1 libtiff6 && \
rm -rf /var/lib/apt/lists/*
apt-get install -y --no-install-recommends \
libssl3 \
libtiff6 \
&& rm -rf /var/lib/apt/lists/*
# Copy the compiled binary from the build stage
COPY --from=build /quoter/target/release/quoter .

View File

@ -6,6 +6,7 @@ use crate::app_state::AppState;
use crate::handlers::serve_file::serve_file;
use crate::s3_utils::{check_file_exists, load_file_from_s3, upload_to_s3};
use crate::thumbnail::{find_closest_width, parse_file_path, thumbdata_save};
use crate::lookup::{find_file_by_pattern, get_mime_type};
/// Обработчик для скачивания файла и генерации миниатюры, если она недоступна.
pub async fn proxy_handler(
@ -30,24 +31,26 @@ pub async fn proxy_handler(
warn!("normalized to lowercase: {}", ext);
let filekey = format!("{}.{}", base_filename, &ext);
warn!("filekey: {}", filekey);
let content_type = match ext.as_str() {
"jpg" | "jpeg" => "image/jpeg",
"png" => "image/png",
"webp" => "image/webp",
"gif" => "image/gif",
"jfif" => "image/jpeg",
"heic" | "heif" => "image/heic",
"tif" | "tiff" => "image/tiff",
"mp3" => "audio/mpeg",
"wav" => "audio/x-wav",
"ogg" => "audio/ogg",
"aac" => "audio/aac",
"m4a" => "audio/m4a",
"flac" => "audio/flac",
_ => {
error!("unsupported file format");
return Err(ErrorInternalServerError("unsupported file format"));
},
let content_type = match get_mime_type(&ext) {
Some(mime) => mime.to_string(),
None => {
let mut redis = state.redis.clone();
match find_file_by_pattern(&mut redis, &base_filename).await {
Ok(Some(found_file)) => {
if let Some(found_ext) = found_file.split('.').last() {
get_mime_type(found_ext)
.unwrap_or("application/octet-stream")
.to_string()
} else {
"application/octet-stream".to_string()
}
}
_ => {
error!("unsupported file format");
return Err(ErrorInternalServerError("unsupported file format"));
}
}
}
};
warn!("content_type: {}", content_type);
@ -148,7 +151,7 @@ pub async fn proxy_handler(
&state.bucket,
&filekey,
filedata.clone(),
content_type,
&content_type,
).await {
error!("Failed to upload to Storj: {} - Error: {}", filekey, e);
} else {
@ -226,7 +229,7 @@ pub async fn proxy_handler(
&state.bucket,
&filekey,
filedata.clone(),
content_type,
&content_type,
)
.await {
warn!("cannot upload to storj: {}", e);

View File

@ -4,9 +4,11 @@ use log::{error, warn};
use crate::app_state::AppState;
use crate::auth::{get_id_by_token, user_added_file};
use crate::s3_utils::{generate_key_with_extension, upload_to_s3};
use crate::s3_utils::{self, upload_to_s3, generate_key_with_extension};
use crate::lookup::store_file_info;
use futures::TryStreamExt;
use crate::handlers::MAX_WEEK_BYTES;
use crate::thumbnail::convert_heic_to_jpeg;
/// Обработчик для аплоада файлов.
pub async fn upload_handler(
@ -30,67 +32,90 @@ pub async fn upload_handler(
let mut body = "ok".to_string();
while let Ok(Some(field)) = payload.try_next().await {
let mut field = field;
let mut file_bytes = Vec::new();
let mut file_size: u64 = 0;
let content_type = field.content_type().unwrap().to_string();
let file_name = field
.content_disposition()
.unwrap()
.get_filename()
.map(|f| f.to_string());
// Читаем данные файла
while let Ok(Some(chunk)) = field.try_next().await {
file_size += chunk.len() as u64;
file_bytes.extend_from_slice(&chunk);
}
if let Some(name) = file_name {
let mut file_bytes = Vec::new();
let mut file_size: u64 = 0;
// Читаем данные файла
while let Ok(Some(chunk)) = field.try_next().await {
file_size += chunk.len() as u64;
file_bytes.extend_from_slice(&chunk);
// Определяем MIME-тип из содержимого файла
let detected_mime_type = match s3_utils::detect_mime_type(&file_bytes) {
Some(mime) => mime,
None => {
warn!("Неподдерживаемый формат файла");
return Err(actix_web::error::ErrorUnsupportedMediaType("Неподдерживаемый формат файла"));
}
};
// Проверяем, что добавление файла не превышает лимит квоты
if this_week_amount + file_size > MAX_WEEK_BYTES {
warn!("Quota would exceed limit: current={}, adding={}, limit={}",
this_week_amount, file_size, MAX_WEEK_BYTES);
return Err(actix_web::error::ErrorUnauthorized("Quota exceeded"));
}
// Загружаем файл в S3 storj с использованием ключа в нижнем регистре
let filekey = generate_key_with_extension(name.clone(), content_type.to_owned());
let orig_path = name.clone();
match upload_to_s3(
&state.storj_client,
&state.bucket,
&filekey,
file_bytes.clone(),
&content_type,
).await {
Ok(_) => {
warn!("file {} uploaded to storj, incrementing quota by {} bytes", filekey, file_size);
// Инкрементируем квоту только после успешной загрузки
if let Err(e) = state.increment_uploaded_bytes(&user_id, file_size).await {
error!("Failed to increment quota: {}", e);
// Можно добавить откат загрузки файла здесь
return Err(e);
}
// Сохраняем информацию о загруженном файле
user_added_file(&mut state.redis.clone(), &user_id, &filekey).await?;
state.set_path(&filekey, &orig_path).await;
// Логируем новое значение квоты
if let Ok(new_quota) = state.get_or_create_quota(&user_id).await {
warn!("New quota for user {}: {} bytes", user_id, new_quota);
}
body = filekey;
}
// Для HEIC файлов конвертируем в JPEG
let (file_bytes, content_type) = if detected_mime_type == "image/heic" {
match convert_heic_to_jpeg(&file_bytes) {
Ok(jpeg_data) => (jpeg_data, "image/jpeg".to_string()),
Err(e) => {
warn!("Failed to upload to storj: {}", e);
return Err(actix_web::error::ErrorInternalServerError(e));
warn!("Failed to convert HEIC to JPEG: {}", e);
(file_bytes, detected_mime_type)
}
}
} else {
(file_bytes, detected_mime_type)
};
// Получаем расширение из MIME-типа
let extension = match s3_utils::get_extension_from_mime(&content_type) {
Some(ext) => ext,
None => {
warn!("Неподдерживаемый тип содержимого: {}", content_type);
return Err(actix_web::error::ErrorUnsupportedMediaType("Неподдерживаемый тип содержимого"));
}
};
// Проверяем, что добавление файла не превышает лимит квоты
if this_week_amount + file_size > MAX_WEEK_BYTES {
warn!("Quota would exceed limit: current={}, adding={}, limit={}",
this_week_amount, file_size, MAX_WEEK_BYTES);
return Err(actix_web::error::ErrorUnauthorized("Quota exceeded"));
}
// Генерируем имя файла с правильным расширением
let filename = format!("{}.{}", uuid::Uuid::new_v4(), extension);
// Загружаем файл в S3 storj
match upload_to_s3(
&state.storj_client,
&state.bucket,
&filename,
file_bytes,
&content_type,
).await {
Ok(_) => {
warn!("file {} uploaded to storj, incrementing quota by {} bytes", filename, file_size);
if let Err(e) = state.increment_uploaded_bytes(&user_id, file_size).await {
error!("Failed to increment quota: {}", e);
return Err(e);
}
// Сохраняем информацию о файле в Redis
let mut redis = state.redis.clone();
store_file_info(&mut redis, &filename, &content_type).await?;
user_added_file(&mut redis, &user_id, &filename).await?;
// Сохраняем маппинг пути
let generated_key = generate_key_with_extension(filename.clone(), content_type.clone());
state.set_path(&filename, &generated_key).await;
if let Ok(new_quota) = state.get_or_create_quota(&user_id).await {
warn!("New quota for user {}: {} bytes", user_id, new_quota);
}
body = filename;
}
Err(e) => {
warn!("Failed to upload to storj: {}", e);
return Err(actix_web::error::ErrorInternalServerError(e));
}
}
}
Ok(HttpResponse::Ok().body(body))

65
src/lookup.rs Normal file
View File

@ -0,0 +1,65 @@
use std::collections::HashMap;
use actix_web::error::ErrorInternalServerError;
use once_cell::sync::Lazy;
use redis::aio::MultiplexedConnection;
use redis::AsyncCommands;
pub static MIME_TYPES: Lazy<HashMap<&'static str, &'static str>> = Lazy::new(|| {
let mut m = HashMap::new();
// Изображения
m.insert("jpg", "image/jpeg");
m.insert("jpeg", "image/jpeg");
m.insert("jfif", "image/jpeg");
m.insert("png", "image/png");
m.insert("webp", "image/webp");
m.insert("gif", "image/gif");
m.insert("heic", "image/heic");
m.insert("heif", "image/heic");
m.insert("tif", "image/tiff");
m.insert("tiff", "image/tiff");
// Аудио
m.insert("mp3", "audio/mpeg");
m.insert("wav", "audio/x-wav");
m.insert("ogg", "audio/ogg");
m.insert("aac", "audio/aac");
m.insert("m4a", "audio/m4a");
m.insert("flac", "audio/flac");
m
});
pub fn get_mime_type(extension: &str) -> Option<&'static str> {
MIME_TYPES.get(extension).copied()
}
/// Ищет файл в Redis по шаблону имени
pub async fn find_file_by_pattern(
redis: &mut MultiplexedConnection,
pattern: &str,
) -> Result<Option<String>, actix_web::Error> {
let pattern_key = format!("files:*{}*", pattern);
let files: Vec<String> = redis
.keys(&pattern_key)
.await
.map_err(|_| ErrorInternalServerError("Failed to search files in Redis"))?;
if files.is_empty() {
Ok(None)
} else {
Ok(Some(files[0].clone()))
}
}
/// Сохраняет файл в Redis с его MIME-типом
pub async fn store_file_info(
redis: &mut MultiplexedConnection,
filename: &str,
mime_type: &str,
) -> Result<(), actix_web::Error> {
let file_key = format!("files:{}", filename);
redis
.set::<_, _, ()>(&file_key, mime_type)
.await
.map_err(|_| ErrorInternalServerError("Failed to store file info in Redis"))?;
Ok(())
}

View File

@ -1,5 +1,6 @@
mod app_state;
mod auth;
mod lookup;
mod handlers;
mod s3_utils;
mod thumbnail;

View File

@ -2,6 +2,7 @@ use actix_web::error::ErrorInternalServerError;
use aws_sdk_s3::{error::SdkError, primitives::ByteStream, Client as S3Client};
use mime_guess::mime;
use std::str::FromStr;
use infer::get;
/// Загружает файл в S3 хранилище.
pub async fn upload_to_s3(
@ -102,3 +103,20 @@ pub async fn get_s3_filelist(client: &S3Client, bucket: &str) -> Vec<[std::strin
}
filenames
}
pub fn detect_mime_type(bytes: &[u8]) -> Option<String> {
let kind = get(bytes)?;
Some(kind.mime_type().to_string())
}
pub fn get_extension_from_mime(mime_type: &str) -> Option<&str> {
match mime_type {
"image/jpeg" => Some("jpg"),
"image/png" => Some("png"),
"image/gif" => Some("gif"),
"image/webp" => Some("webp"),
"image/heic" => Some("heic"),
"image/tiff" => Some("tiff"),
_ => None
}
}

View File

@ -195,3 +195,34 @@ pub fn find_closest_width(requested_width: u32) -> u32 {
.min_by_key(|&&width| (width as i32 - requested_width as i32).abs())
.unwrap_or(&THUMB_WIDTHS[0]) // Возвращаем самый маленький размер, если ничего не подошло
}
/// Конвертирует HEIC в JPEG
pub fn convert_heic_to_jpeg(data: &[u8]) -> Result<Vec<u8>, actix_web::Error> {
// Пробуем прочитать как обычное изображение
if let Ok(img) = image::load_from_memory(data) {
let mut buffer = Vec::new();
img.write_to(&mut Cursor::new(&mut buffer), ImageFormat::Jpeg)
.map_err(|e| actix_web::error::ErrorInternalServerError(format!("Failed to convert to JPEG: {}", e)))?;
return Ok(buffer);
}
// Если не получилось, пробуем через exif
let mut cursor = Cursor::new(data);
match exif::Reader::new().read_from_container(&mut cursor) {
Ok(_exif) => {
// Конвертируем в JPEG
let img = image::load_from_memory(data)
.map_err(|e| actix_web::error::ErrorInternalServerError(format!("Failed to load HEIC: {}", e)))?;
let mut buffer = Vec::new();
img.write_to(&mut Cursor::new(&mut buffer), ImageFormat::Jpeg)
.map_err(|e| actix_web::error::ErrorInternalServerError(format!("Failed to convert to JPEG: {}", e)))?;
Ok(buffer)
}
Err(e) => Err(actix_web::error::ErrorInternalServerError(format!(
"Failed to process HEIC: {}",
e
))),
}
}