From da1826ebba010aba14f3b1e8c0700996fd26c24e Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Thu, 14 Aug 2025 17:03:23 +0700 Subject: [PATCH] feat: Add file upload with deduplication to MinIO service --- Cargo.toml | 1 - imphnen-iam/src/v1/users/users_service.rs | 4 +- imphnen-libs/Cargo.toml | 10 +- imphnen-libs/src/minio.rs | 201 ++++++++++++++++++++-- 4 files changed, 198 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bdf3f92..7b22074 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,6 @@ rand = { version = "0.9.1", features = ["std", "alloc"] } rand_distr = "0.5.1" tower-http = { version = "0.6.4", features = ["cors", "trace"] } http-body-util = "0.1.1" - validator = { version = "0.20.0", features = ["derive"] } lazy_static = "1.4.0" regex = "1.11.1" diff --git a/imphnen-iam/src/v1/users/users_service.rs b/imphnen-iam/src/v1/users/users_service.rs index ae20524..3dd2edf 100644 --- a/imphnen-iam/src/v1/users/users_service.rs +++ b/imphnen-iam/src/v1/users/users_service.rs @@ -532,8 +532,8 @@ impl UsersServiceTrait for UsersService { let folder = format!("{}/{}", file_type.as_folder(), sanitized_user_id); - // Upload file to MinIO - match minio_service.upload_file(&file_data, &content_type, &folder, &filename).await { + // Upload file to MinIO with deduplication + match minio_service.upload_file_with_deduplication(&file_data, &content_type, &folder, &filename).await { Ok(object_path) => { // Create permanent URL (no expiration) let permanent_url = format!("https://cdn.asepharyana.tech/{}/{}", diff --git a/imphnen-libs/Cargo.toml b/imphnen-libs/Cargo.toml index 7d1dbad..03130db 100644 --- a/imphnen-libs/Cargo.toml +++ b/imphnen-libs/Cargo.toml @@ -19,8 +19,8 @@ dotenvy.workspace = true anyhow.workspace = true uuid.workspace = true base64.workspace = true -reqwest = { version = "0.11", features = ["json"] } -sha2 = "0.10" -hmac = "0.12" -hex = "0.4" -urlencoding = "2.1" +reqwest.workspace = true +sha2.workspace = true +hmac.workspace = true +hex.workspace = true +urlencoding.workspace = true diff --git a/imphnen-libs/src/minio.rs b/imphnen-libs/src/minio.rs index c979e09..f997d0f 100644 --- a/imphnen-libs/src/minio.rs +++ b/imphnen-libs/src/minio.rs @@ -6,16 +6,7 @@ use sha2::{Digest, Sha256}; use uuid::Uuid; use crate::enviroment::ENV; -// CATATAN: Pastikan untuk menambahkan dependensi ini ke `Cargo.toml` Anda: -// reqwest = { version = "0.11", features = ["json"] } -// anyhow = "1.0" -// uuid = { version = "1.3", features = ["v4"] } -// base64 = "0.21" -// log = "0.4" -// chrono = "0.4" -// sha2 = "0.10" -// hmac = "0.12" -// hex = "0.4" + // --- Struct Konfigurasi MinIO --- #[derive(Debug, Clone)] @@ -84,6 +75,121 @@ impl MinioService { Ok(service) } + /// Mengunggah file biner ke MinIO dengan deduplication berdasarkan hash. + pub async fn upload_file_with_deduplication( + &self, + file_data: &[u8], + content_type: &str, + folder: &str, + original_filename: &str, + ) -> Result { + Self::validate_file_type(content_type, file_data)?; + + // Calculate file hash + let mut hasher = Sha256::new(); + hasher.update(file_data); + let file_hash = format!("{:x}", hasher.finalize()); + let short_hash = &file_hash[..16]; // Use first 16 characters for filename + + // Check if file with same hash already exists + if let Some(existing_file) = self.check_file_exists_by_hash(folder, short_hash).await? { + log::info!("File with same content already exists: {}", existing_file); + return Ok(existing_file); + } + + let file_extension = Self::get_file_extension(original_filename); + let unique_filename = format!("{}/{}-{}.{}", folder, short_hash, Uuid::new_v4(), file_extension); + let object_name = &unique_filename; + + // Extract host from endpoint (remove protocol) + let host = self.endpoint + .trim_start_matches("https://") + .trim_start_matches("http://"); + + let url = format!("https://{}/{}/{}", host, self.bucket_name, object_name); + + // Debug logging + log::debug!("MinIO Endpoint config: {}", self.endpoint); + log::debug!("MinIO Region config: {}", self.region); + log::debug!("Upload URL: {}", url); + log::debug!("Object name: {}", object_name); + log::debug!("File hash: {}", short_hash); + + let now = Utc::now(); + let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); + let date_stamp = now.format("%Y%m%d").to_string(); + + // Use UNSIGNED-PAYLOAD for simpler signature + let payload_hash = "UNSIGNED-PAYLOAD".to_string(); + + let canonical_headers = format!( + "host:{}\nx-amz-content-sha256:{}\nx-amz-date:{}\n", + host, payload_hash, amz_date + ); + let signed_headers = "host;x-amz-content-sha256;x-amz-date"; + + // For path-style, canonical URI should be /bucket/object + let canonical_uri = format!("/{}/{}", self.bucket_name, object_name); + let canonical_request = format!( + "PUT\n{}\n\n{}\n{}\n{}", + canonical_uri, canonical_headers, signed_headers, payload_hash + ); + + log::debug!("Canonical request:\n{}", canonical_request); + + let scope = format!("{}/{}/s3/aws4_request", date_stamp, self.region); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, + scope, + hex::encode(Sha256::digest(canonical_request.as_bytes())) + ); + + log::debug!("Scope: {}", scope); + log::debug!("String to sign:\n{}", string_to_sign); + + let signing_key = self.get_signature_key(&date_stamp)?; + let mut mac = Hmac::::new_from_slice(&signing_key)?; + mac.update(string_to_sign.as_bytes()); + let signature = hex::encode(mac.finalize().into_bytes()); + + log::debug!("Generated signature: {}", signature); + + let auth_header = format!( + "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + self.access_key, scope, signed_headers, signature + ); + + // 5) Send request: Content-Type included but NOT signed + let response = self + .client + .put(&url) + .header("x-amz-date", &amz_date) + .header("x-amz-content-sha256", &payload_hash) + .header("Authorization", &auth_header) + .header("Content-Type", content_type) + // Don't set Host header manually - let reqwest handle it + // Add headers for reverse proxy support (not signed) + .header("X-Forwarded-Proto", "https") + .header("X-Forwarded-Host", host) + .body(file_data.to_vec()) + .send() + .await?; + + if !response.status().is_success() { + let status = response.status(); + let error_body = response.text().await?; + bail!( + "Gagal mengunggah file ke MinIO. Status: {}. Pesan: {}", + status, + error_body + ); + } + + log::info!("Unggahan berhasil: {} byte ke {}", file_data.len(), unique_filename); + Ok(unique_filename) + } + /// Mengunggah file biner ke MinIO. pub async fn upload_file( &self, @@ -262,6 +368,81 @@ impl MinioService { Ok(url) } + + /// Mengecek apakah file dengan hash tertentu sudah ada di bucket + pub async fn check_file_exists_by_hash(&self, folder: &str, file_hash: &str) -> Result> { + // Extract host from endpoint (remove protocol) + let host = self.endpoint + .trim_start_matches("https://") + .trim_start_matches("http://"); + + let url = format!("https://{}/{}?list-type=2&prefix={}", host, self.bucket_name, folder); + + let now = Utc::now(); + let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string(); + let date_stamp = now.format("%Y%m%d").to_string(); + let payload_hash = hex::encode(Sha256::digest(b"")); + + let canonical_query_string = format!("list-type=2&prefix={}", urlencoding::encode(folder)); + let canonical_headers = format!("host:{}\nx-amz-content-sha256:{}\nx-amz-date:{}\n", host, payload_hash, amz_date); + let signed_headers = "host;x-amz-content-sha256;x-amz-date"; + let canonical_request = format!( + "GET\n/{}\n{}\n{}\n{}\n{}", + self.bucket_name, canonical_query_string, canonical_headers, signed_headers, payload_hash + ); + + let scope = format!("{}/{}/s3/aws4_request", date_stamp, self.region); + let string_to_sign = format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + amz_date, + scope, + hex::encode(Sha256::digest(canonical_request.as_bytes())) + ); + + let signing_key = self.get_signature_key(&date_stamp)?; + let mut mac = Hmac::::new_from_slice(&signing_key)?; + mac.update(string_to_sign.as_bytes()); + let signature = hex::encode(mac.finalize().into_bytes()); + + let auth_header = format!( + "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + self.access_key, scope, signed_headers, signature + ); + + let response = self + .client + .get(&url) + .header("x-amz-date", &amz_date) + .header("x-amz-content-sha256", &payload_hash) + .header("Authorization", &auth_header) + .send() + .await?; + + if !response.status().is_success() { + return Ok(None); + } + + let body = response.text().await?; + + // Simple XML parsing to find files with matching hash + // Look for any file that contains the hash in its name + if body.contains(file_hash) { + // Extract the full file path from XML response + // This is a simplified approach - in production you might want proper XML parsing + for line in body.lines() { + if line.contains("") && line.contains(file_hash) { + if let Some(start) = line.find("") { + if let Some(end) = line.find("") { + let file_path = &line[start + 5..end]; + return Ok(Some(file_path.to_string())); + } + } + } + } + } + + Ok(None) + } /// Menghapus file dari MinIO. pub async fn delete_file(&self, object_name: &str) -> Result<()> {