MythEclipse/asepharyana-hub-scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: remove croxy proxy and image cache features

Browse Source
This commit is contained in:
asepharyana
2026-07-22 14:51:08 +07:00
parent 891064354b
commit 5c51ec895f
35 changed files with 42 additions and 2799 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+10 -200
View File
@@ -439,17 +439,6 @@ dependencies = [
"shlex",
]
[[package]]
name = "cfb"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f"
dependencies = [
"byteorder",
"fnv",
"uuid",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -476,16 +465,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "chrono-tz"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
dependencies = [
"chrono",
"phf 0.12.1",
]
[[package]]
name = "combine"
version = "4.6.7"
@@ -643,17 +622,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "croner"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576"
dependencies = [
"chrono",
"derive_builder",
"strum 0.27.2",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
@@ -713,7 +681,7 @@ dependencies = [
"cssparser-macros",
"dtoa-short",
"itoa",
"phf 0.13.1",
"phf",
"smallvec",
]
@@ -727,41 +695,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "darling"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 2.0.117",
]
[[package]]
name = "darling_macro"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core",
"quote",
"syn 2.0.117",
]
[[package]]
name = "dashmap"
version = "6.1.0"
@@ -847,37 +780,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "derive_builder"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
dependencies = [
"derive_builder_macro",
]
[[package]]
name = "derive_builder_core"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "derive_builder_macro"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
dependencies = [
"derive_builder_core",
"syn 2.0.117",
]
[[package]]
name = "derive_more"
version = "2.1.1"
@@ -1648,12 +1550,6 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
version = "1.1.0"
@@ -1697,15 +1593,6 @@ dependencies = [
"serde_core",
]
[[package]]
name = "infer"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7"
dependencies = [
"cfb",
]
[[package]]
name = "inherent"
version = "1.0.13"
@@ -2038,17 +1925,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-derive"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "num-integer"
version = "0.1.46"
@@ -2350,15 +2226,6 @@ dependencies = [
"serde",
]
[[package]]
name = "phf"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
dependencies = [
"phf_shared 0.12.1",
]
[[package]]
name = "phf"
version = "0.13.1"
@@ -2366,7 +2233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
dependencies = [
"phf_macros",
"phf_shared 0.13.1",
"phf_shared",
"serde",
]
@@ -2377,7 +2244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
dependencies = [
"phf_generator",
"phf_shared 0.13.1",
"phf_shared",
]
[[package]]
@@ -2387,7 +2254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
dependencies = [
"fastrand",
"phf_shared 0.13.1",
"phf_shared",
]
[[package]]
@@ -2397,21 +2264,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
dependencies = [
"phf_generator",
"phf_shared 0.13.1",
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "phf_shared"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981"
dependencies = [
"siphasher",
]
[[package]]
name = "phf_shared"
version = "0.13.1"
@@ -3124,7 +2982,6 @@ dependencies = [
"async-trait",
"axum 0.8.8",
"backoff",
"bytes",
"chrono",
"config",
"dashmap",
@@ -3132,9 +2989,7 @@ dependencies = [
"dotenvy",
"flate2",
"futures",
"hex",
"http",
"infer",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry_sdk",
@@ -3146,10 +3001,8 @@ dependencies = [
"sea-orm",
"serde",
"serde_json",
"sha2",
"thiserror 2.0.18",
"tokio",
"tokio-cron-scheduler",
"tower-http",
"tracing",
"tracing-subscriber",
@@ -3195,7 +3048,7 @@ dependencies = [
"serde",
"serde_json",
"sqlx",
"strum 0.26.3",
"strum",
"thiserror 2.0.18",
"time",
"tracing",
@@ -3289,7 +3142,7 @@ dependencies = [
"derive_more",
"log",
"new_debug_unreachable",
"phf 0.13.1",
"phf",
"phf_codegen",
"precomputed-hash",
"rustc-hash",
@@ -3763,7 +3616,7 @@ checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
dependencies = [
"new_debug_unreachable",
"parking_lot",
"phf_shared 0.13.1",
"phf_shared",
"precomputed-hash",
]
@@ -3774,7 +3627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
dependencies = [
"phf_generator",
"phf_shared 0.13.1",
"phf_shared",
"proc-macro2",
"quote",
]
@@ -3790,39 +3643,12 @@ dependencies = [
"unicode-properties",
]
[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "strum"
version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
[[package]]
name = "strum"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
dependencies = [
"strum_macros",
]
[[package]]
name = "strum_macros"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "subtle"
version = "2.6.1"
@@ -4053,22 +3879,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "tokio-cron-scheduler"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f50e41f200fd8ed426489bd356910ede4f053e30cebfbd59ef0f856f0d7432a"
dependencies = [
"chrono",
"chrono-tz",
"croner",
"num-derive",
"num-traits",
"tokio",
"tracing",
"uuid",
]
[[package]]
name = "tokio-macros"
version = "2.6.1"
@@ -4716,7 +4526,7 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576"
dependencies = [
"phf 0.13.1",
"phf",
"phf_codegen",
"string_cache",
"string_cache_codegen",
Cargo.toml
-5
View File
@@ -26,13 +26,11 @@ sea-orm = { version = "1.1.19", features = ["sqlx-postgres", "runtime-tokio-rust
uuid = { version = "1.10.0", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
bytes = "1.11.0"
futures = "0.3"
reqwest = { version = "0.12.28", features = ["json", "stream", "multipart"] }
http = "1.4.0"
async-trait = "0.1.89"
regex = "1.12.2"
infer = "0.19.0"
urlencoding = "2.1"
url = "2.5.8"
@@ -48,9 +46,6 @@ redis = { version = "0.32.7", features = ["tokio-rustls-comp", "safe_iterators"]
thiserror = "2.0.18"
config = { version = "0.15.19", features = ["toml"] }
tokio-cron-scheduler = "0.15.1"
hex = "0.4.3"
sha2 = "0.10.9"
utoipa = { version = "5.0", features = ["axum_extras"] }
utoipa-swagger-ui = { version = "9.0", features = ["axum"] }
src/application/anime/use_cases.rs
+3 -72
View File
@@ -3,18 +3,12 @@
//! Orchestrates repository fetching, caching, and image poster processing.
//! Returns pure domain types — no DTOs.
use std::sync::Arc;
use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::entity::anime::*;
use crate::domain::error::*;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::OtakudesuRepository;
use crate::infrastructure::services::images::cache::{
cache_image_urls_batch_lazy, get_cached_or_original,
};
const INDEX_CACHE_TTL: u64 = 10;
const GENRE_LIST_CACHE_TTL: u64 = 3600;
@@ -23,22 +17,13 @@ const DEFAULT_CACHE_TTL: u64 = 300;
pub struct AnimeUseCases {
repository: OtakudesuRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
}
impl AnimeUseCases {
pub fn new(
repository: OtakudesuRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
pub fn new(repository: OtakudesuRepository, redis_pool: Pool) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
}
@@ -49,7 +34,7 @@ impl AnimeUseCases {
pub async fn get_anime_index(&self) -> Result<AnimeData, DomainError> {
self.cache()
.get_or_set("anime:index:v2", INDEX_CACHE_TTL, || async {
let mut data = self
let data = self
.repository
.fetch_anime_index()
.await
@@ -59,33 +44,6 @@ impl AnimeUseCases {
return Err("Empty anime index — refusing to cache".to_string());
}
let mut posters: Vec<String> = data
.ongoing_anime
.iter()
.map(|item| item.poster.clone())
.collect();
posters.extend(data.complete_anime.iter().map(|item| item.poster.clone()));
let cached_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
posters,
self.semaphore.clone(),
)
.await;
let ongoing_len = data.ongoing_anime.len();
for (i, item) in data.ongoing_anime.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(i) {
item.poster = url.clone();
}
}
for (i, item) in data.complete_anime.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(ongoing_len + i) {
item.poster = url.clone();
}
}
Ok(data)
})
.await
@@ -108,39 +66,12 @@ impl AnimeUseCases {
let cache_key = format!("anime:detail:{}", slug);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
let mut data = self
let data = self
.repository
.fetch_anime_detail(&slug)
.await
.map_err(|e| e.to_string())?;
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
let rec_posters: Vec<String> = data
.recommendations
.iter()
.map(|r| r.poster.clone())
.collect();
let cached_rec_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
rec_posters,
self.semaphore.clone(),
)
.await;
for (i, rec) in data.recommendations.iter_mut().enumerate() {
if let Some(url) = cached_rec_posters.get(i) {
rec.poster = url.clone();
}
}
Ok(data)
})
.await
src/application/anime2/use_cases.rs
+10 -129
View File
@@ -8,23 +8,17 @@
//! TODO: Once parsers return domain types, replace shared types with
//! `crate::domain::entity::anime::{GenreAnimeItem, SearchAnimeItem, LatestAnimeItem}`.
use std::sync::Arc;
use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::error::*;
use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::AlqanimeRepository;
use crate::infrastructure::services::images::cache::{
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
};
use crate::infrastructure::repository::parsers::alqanime_parser as parser;
use crate::domain::entity::anime::{
CompleteAnimeItem, FilterAnimeItem, Genre, GenreAnimeItem, HasPoster, LatestAnimeItem,
CompleteAnimeItem, FilterAnimeItem, Genre, GenreAnimeItem, LatestAnimeItem,
OngoingAnimeItemWithScore, Pagination, SearchAnimeItem,
};
@@ -56,15 +50,6 @@ pub struct Anime2Item {
pub anime_url: String,
}
impl HasPoster for Anime2Item {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2Data {
pub ongoing_anime: Vec<Anime2Item>,
@@ -123,22 +108,13 @@ const COMPLETE_CACHE_TTL: u64 = 300;
pub struct Anime2UseCases {
repository: AlqanimeRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
}
impl Anime2UseCases {
pub fn new(
repository: AlqanimeRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
pub fn new(repository: AlqanimeRepository, redis_pool: Pool) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
}
@@ -168,7 +144,7 @@ impl Anime2UseCases {
})
.await
.map_err(|e| e.to_string())??;
let mut ongoing: Vec<Anime2Item> = data
let ongoing: Vec<Anime2Item> = data
.0
.into_iter()
.map(|item| Anime2Item {
@@ -182,7 +158,7 @@ impl Anime2UseCases {
})
.collect();
let mut complete: Vec<Anime2Item> = data
let complete: Vec<Anime2Item> = data
.1
.into_iter()
.map(|item| Anime2Item {
@@ -196,30 +172,6 @@ impl Anime2UseCases {
})
.collect();
let mut posters: Vec<String> =
ongoing.iter().map(|item| item.poster.clone()).collect();
posters.extend(complete.iter().map(|item| item.poster.clone()));
let cached_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
posters,
self.semaphore.clone(),
)
.await;
let ongoing_len = ongoing.len();
for (i, item) in ongoing.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(i) {
item.poster = url.clone();
}
}
for (i, item) in complete.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(ongoing_len + i) {
item.poster = url.clone();
}
}
Ok(Anime2Response {
status: "Ok".to_string(),
data: Anime2Data {
@@ -293,20 +245,12 @@ impl Anime2UseCases {
.fetch_html(&url)
.await
.map_err(|e| e.to_string())?;
let (mut data, pagination) = tokio::task::spawn_blocking(move || {
let (data, pagination) = tokio::task::spawn_blocking(move || {
parser::parse_filter_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(FilterResponse {
success: true,
data,
@@ -374,29 +318,6 @@ impl Anime2UseCases {
}
}
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
data.poster2 = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster2,
self.semaphore.clone(),
)
.await;
apply_cached_posters(
&mut data.recommendations,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(DetailResponse {
status: "Ok".to_string(),
data,
@@ -420,20 +341,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.genre_page_url(&genre_slug, page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_genre_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
@@ -454,20 +367,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.search_url(&query, page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_search_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
@@ -487,20 +392,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.latest_url(page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_latest_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
@@ -520,20 +417,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.ongoing_url(page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
@@ -553,20 +442,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.complete_url(page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_complete_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
src/application/komik/use_cases.rs
+8 -81
View File
@@ -6,10 +6,7 @@
//! `crate::infrastructure::repository::parsers::komik_parser`.
//! TODO: Move response DTOs to `crate::presentation::dto::komik`.
use std::sync::Arc;
use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::entity::anime::Pagination;
use crate::domain::entity::komik::{ChapterData, DetailData, KomikGenre, KomikItem};
@@ -17,9 +14,6 @@ use crate::domain::error::*;
use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::KomikRepository;
use crate::infrastructure::services::images::cache::{
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
};
use crate::infrastructure::repository::parsers::komik_parser as parser;
@@ -36,22 +30,13 @@ const SEARCH_CACHE_TTL: u64 = 300;
pub struct KomikUseCases {
repository: KomikRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
}
impl KomikUseCases {
pub fn new(
repository: KomikRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
pub fn new(repository: KomikRepository, redis_pool: Pool) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
}
@@ -94,20 +79,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
@@ -130,20 +107,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
@@ -162,22 +131,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let mut data =
let data =
tokio::task::spawn_blocking(move || parser::parse_komik_detail_document(&html))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
if !data.poster.is_empty() {
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
}
Ok(data)
})
.await
@@ -196,7 +155,7 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let mut data = tokio::task::spawn_blocking({
let data = tokio::task::spawn_blocking({
let chapter_url = chapter_url.clone();
move || parser::parse_komik_chapter_document(&html, &chapter_url)
})
@@ -204,14 +163,6 @@ impl KomikUseCases {
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
data.images = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
data.images,
self.semaphore.clone(),
)
.await;
Ok(data)
})
.await
@@ -278,7 +229,7 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
@@ -288,14 +239,6 @@ impl KomikUseCases {
return Err(format!("Empty komik {} page {}", list_name, page));
}
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
@@ -318,20 +261,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
@@ -354,20 +289,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
src/application/proxy/use_cases.rs
-235
View File
@@ -1,236 +1 @@
//! Proxy application use cases.
//!
//! Provides proxy fetch, image caching, and audit/repair operations.
//!
//! TODO: Move result types to `crate::presentation::dto::proxy`.
//! TODO: Add event bus integration for ImageRepaired events.
//! TODO: Replace `reqwest::Client::new()` with shared HTTP client from infrastructure.
use std::sync::Arc;
use axum::http::StatusCode;
use axum::response::Response;
use serde::Serialize;
use tracing::{error, info, warn};
use utoipa::ToSchema;
use crate::domain::error::*;
use crate::domain::repository::ImageCacheRepository;
use crate::infrastructure::repository::{ProxyRepository, SeaOrmImageCacheRepository};
use crate::infrastructure::services::images::cache::ImageCache;
// ============================================================================
// Result types — TEMPORARY: move to presentation layer
// ============================================================================
/// Result of caching an image URL.
/// TODO: Move to presentation::dto::proxy
#[derive(Debug, Serialize, ToSchema)]
pub struct ImageCacheResult {
pub success: bool,
pub original_url: String,
pub cdn_url: String,
pub from_cache: bool,
pub pending: Option<bool>,
}
/// Result of auditing/repairing a cached image URL.
/// TODO: Move to presentation::dto::proxy
#[derive(Debug, Serialize, ToSchema)]
pub struct AuditImageCacheResult {
pub success: bool,
pub original_url: String,
pub cdn_url: Option<String>,
pub was_accessible: bool,
pub re_uploaded: bool,
pub message: String,
}
// ============================================================================
// Use case struct
// ============================================================================
pub struct ProxyUseCases {
repository: ProxyRepository,
image_cache_repo: Arc<dyn ImageCacheRepository>,
}
impl ProxyUseCases {
pub fn new(
repository: ProxyRepository,
image_cache_repo: Arc<SeaOrmImageCacheRepository>,
) -> Self {
Self {
repository,
image_cache_repo,
}
}
fn build_image_cache(&self) -> ImageCache {
ImageCache::new(self.image_cache_repo.clone())
}
pub async fn fetch_with_proxy_only(&self, url: String) -> Result<Response, DomainError> {
let fetch_result = self
.repository
.fetch_with_proxy_url(&url)
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e.to_string())))?;
let mut builder = Response::builder().status(StatusCode::OK);
if let Some(content_type) = fetch_result.content_type {
builder = builder.header("Content-Type", content_type);
}
builder
.body(fetch_result.data.into())
.map_err(|e| DomainError::Repository(RepositoryError::Network(e.to_string())))
}
pub async fn image_cache(
&self,
url: String,
lazy: bool,
) -> Result<ImageCacheResult, DomainError> {
let cache = self.build_image_cache();
if let Some(cdn_url) = cache.get_cdn_url(&url).await {
return Ok(ImageCacheResult {
success: true,
original_url: url,
cdn_url,
from_cache: true,
pending: None,
});
}
if lazy {
let repo = self.image_cache_repo.clone();
let url_clone = url.clone();
tokio::spawn(async move {
let cache = ImageCache::new(repo);
match cache.get_or_cache(&url_clone).await {
Ok(cdn) => info!("[LazyCache] Cached {} -> {}", url_clone, cdn),
Err(e) => warn!("[LazyCache] Failed {}: {}", url_clone, e),
}
});
return Ok(ImageCacheResult {
success: true,
original_url: url.clone(),
cdn_url: url,
from_cache: false,
pending: Some(true),
});
}
match cache.get_or_cache(&url).await {
Ok(cdn_url) => Ok(ImageCacheResult {
success: true,
original_url: url,
cdn_url,
from_cache: false,
pending: None,
}),
Err(e) => {
error!("ImageCache error: {}", e);
Ok(ImageCacheResult {
success: false,
original_url: url.clone(),
cdn_url: url,
from_cache: false,
pending: None,
})
}
}
}
pub async fn audit_image_cache(
&self,
url: String,
) -> Result<AuditImageCacheResult, DomainError> {
let cache = self.build_image_cache();
let mut cdn_opt = cache.get_cdn_url(&url).await;
let mut original = url.clone();
if cdn_opt.is_none() {
if let Some(orig) = cache.find_original_from_cdn(&url).await {
info!("SmartAudit: {} recognized as CDN, original {}", url, orig);
original = orig;
cdn_opt = Some(url.clone());
}
}
if let Some(cdn_url) = cdn_opt {
let client = reqwest::Client::new();
let mut accessible = false;
match client.get(&cdn_url).send().await {
Ok(resp) if resp.status().is_success() => {
if let Ok(bytes) = resp.bytes().await {
if infer::get(&bytes)
.map(|k| k.mime_type().starts_with("image/"))
.unwrap_or(false)
{
accessible = true;
} else {
warn!("CDN {} returned non-image content", cdn_url);
}
}
}
Ok(resp) => warn!("CDN {} status {}", cdn_url, resp.status()),
Err(e) => warn!("CDN {} fetch error {}", cdn_url, e),
}
if accessible {
return Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(cdn_url),
was_accessible: true,
re_uploaded: false,
message: "CDN URL is accessible and the image is valid".to_string(),
});
}
info!("CDN {} inaccessible, purging and reuploading", cdn_url);
let _ = cache.invalidate(&original).await;
match cache.get_or_cache(&original).await {
Ok(new_cdn) => Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(new_cdn),
was_accessible: false,
re_uploaded: true,
message: "CDN URL was inaccessible, re-uploaded".to_string(),
}),
Err(e) => Ok(AuditImageCacheResult {
success: false,
original_url: original,
cdn_url: None,
was_accessible: false,
re_uploaded: false,
message: format!("Re-upload failed: {}", e),
}),
}
} else {
match cache.get_or_cache(&original).await {
Ok(new_cdn) => Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(new_cdn),
was_accessible: false,
re_uploaded: true,
message: "Cached newly".to_string(),
}),
Err(e) => Ok(AuditImageCacheResult {
success: false,
original_url: original,
cdn_url: None,
was_accessible: false,
re_uploaded: false,
message: format!("Cache failed: {}", e),
}),
}
}
}
}
src/bootstrap/mod.rs
-11
View File
@@ -77,26 +77,15 @@ impl Application {
// App State components
let db_arc = Arc::new(db);
let image_processing_semaphore = Arc::new(tokio::sync::Semaphore::new(
CONFIG.image_processing_concurrency,
));
let event_bus = Arc::new(crate::events::bus::EventBus::new());
let redis_pool = crate::infrastructure::cache::redis_pool::redis_pool()
.map_err(|e| anyhow::anyhow!("Failed to init Redis pool: {}", e))?;
use crate::infrastructure::repository::SeaOrmImageCacheRepository;
let image_cache_repo = Arc::new(SeaOrmImageCacheRepository::new(
db_arc.clone(),
redis_pool.clone(),
));
let app_state = Arc::new(AppState {
redis_pool,
db: db_arc.clone(),
image_processing_semaphore,
event_bus: event_bus.clone(),
image_cache_repo,
});
let app = crate::presentation::router::build_router(app_state.clone())?;
src/bootstrap/setup.rs
+3 -41
View File
@@ -1,47 +1,9 @@
//! Database schema initialization.
use sea_orm::{ConnectionTrait, DatabaseConnection, Schema, Statement};
use sea_orm::DatabaseConnection;
use tracing::info;
use crate::infrastructure::persistence::entities::image_cache;
pub async fn init(db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> {
info!("🚀 Initializing database schema...");
let backend = db.get_database_backend();
let schema = Schema::new(backend);
let tables = vec![(
"ImageCache",
schema
.create_table_from_entity(image_cache::Entity)
.if_not_exists()
.to_owned(),
)];
for (name, stmt) in tables {
match db.execute(backend.build(&stmt)).await {
Ok(_) => info!(" ✓ Table '{}' checked/created", name),
Err(e) => {
tracing::error!(" [!] Failed to create table '{}': {}", name, e);
return Err(e);
}
}
}
let index_sql =
"CREATE INDEX IF NOT EXISTS idx_image_cache_cdn_url ON \"ImageCache\" (cdn_url)";
match db.execute(Statement::from_string(backend, index_sql)).await {
Ok(_) => info!(" ✓ Index 'idx_image_cache_cdn_url' ensured"),
Err(e) => {
let err_str = e.to_string();
if err_str.contains("already exists") || err_str.contains("duplicate") {
info!(" ✓ Index 'idx_image_cache_cdn_url' already exists");
} else {
tracing::error!(" [!] Failed to create index on ImageCache: {}", e);
}
}
}
info!("✅ Database schema initialization complete.");
pub async fn init(_db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> {
info!("✅ Database schema initialization complete (no tables to create).");
Ok(())
}
src/config/mod.rs
-22
View File
@@ -47,10 +47,6 @@ pub struct AppConfig {
#[serde(default)]
pub db: DbConfig,
/// Max concurrent image processing tasks
#[serde(default = "default_image_processing_concurrency")]
pub image_processing_concurrency: usize,
/// Domain/URL values that may change between deployments
#[serde(default)]
pub urls: UrlConfig,
@@ -60,18 +56,12 @@ pub struct AppConfig {
pub struct UrlConfig {
#[serde(default = "default_site_url")]
pub site_url: String,
#[serde(default = "default_picser_api_url")]
pub picser_api_url: String,
#[serde(default = "default_fallback_upload_api_url")]
pub fallback_upload_api_url: String,
}
impl Default for UrlConfig {
fn default() -> Self {
Self {
site_url: default_site_url(),
picser_api_url: default_picser_api_url(),
fallback_upload_api_url: default_fallback_upload_api_url(),
}
}
}
@@ -181,22 +171,10 @@ fn default_log_level() -> String {
"info".to_string()
}
fn default_image_processing_concurrency() -> usize {
5
}
fn default_site_url() -> String {
"https://asepharyana.my.id".to_string()
}
fn default_picser_api_url() -> String {
"https://picser.asepharyana.my.id/api/upload".to_string()
}
fn default_fallback_upload_api_url() -> String {
"https://upload.asepharyana.my.id/api/upload".to_string()
}
fn default_db_max_connections() -> u32 {
100
}
src/domain/entity/anime.rs
-111
View File
@@ -290,115 +290,4 @@ pub struct FilterAnimeItem {
pub anime_url: String,
}
// ============================================================================
// TRAITS
// ============================================================================
/// Trait for types that have a poster image URL
pub trait HasPoster {
fn poster(&self) -> &str;
fn set_poster(&mut self, url: String);
}
// ============================================================================
// TRAIT IMPLEMENTATIONS
// ============================================================================
impl HasPoster for OngoingAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for CompleteAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for CompleteAnimeListItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for OngoingAnimeListItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for LatestAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for SearchAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for GenreAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for Recommendation {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for Anime2Item {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for OngoingAnimeItemWithScore {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for FilterAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
src/domain/entity/komik.rs
-11
View File
@@ -3,8 +3,6 @@
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use crate::domain::entity::anime::HasPoster;
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct KomikGenre {
pub name: String,
@@ -53,12 +51,3 @@ pub struct KomikItem {
pub r#type: String,
pub komik_url: String,
}
impl HasPoster for KomikItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
src/domain/repository/image_cache.rs
-19
View File
@@ -1,19 +0,0 @@
//! Repository trait for image cache (original → CDN mapping).
use async_trait::async_trait;
/// Repository for managing cached image URL mappings.
#[async_trait]
pub trait ImageCacheRepository: Send + Sync {
async fn get_from_redis(&self, key: &str) -> Option<String>;
async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String>;
async fn get_from_db(&self, original_url: &str) -> Result<Option<String>, String>;
async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String>;
async fn find_original_from_cdn(&self, cdn_url: &str) -> Result<Option<String>, String>;
async fn delete_from_db(&self, original_url: &str) -> Result<(), String>;
async fn delete_from_redis(&self, key: &str) -> Result<(), String>;
async fn get_lock(&self, key: &str) -> bool;
async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String>;
async fn release_lock(&self, key: &str) -> Result<(), String>;
async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String>;
}
src/domain/repository/mod.rs
-2
View File
@@ -1,5 +1,3 @@
pub mod image_cache;
pub mod scraping;
pub use image_cache::ImageCacheRepository;
pub use scraping::ScrapingRepository;
src/events/bus.rs
-9
View File
@@ -143,13 +143,4 @@ impl Event for OrderCreated {
const NAME: &'static str = "order.created";
}
/// Image repaired event.
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
pub struct ImageRepaired {
pub original_url: String,
pub cdn_url: String,
}
impl Event for ImageRepaired {
const NAME: &'static str = "image.repaired";
}
src/infrastructure/mod.rs
-2
View File
@@ -1,6 +1,4 @@
pub mod cache;
pub mod persistence;
pub mod repository;
pub mod scraping;
pub mod services;
pub mod utils;
src/infrastructure/persistence/entities/image_cache.rs
-71
View File
@@ -1,71 +0,0 @@
//! `SeaORM` Entity for ImageCache - stores URL mappings for CDN caching
use sea_orm::entity::prelude::*;
use serde::{Deserialize, Serialize};
#[derive(Copy, Clone, Default, Debug, DeriveEntity)]
pub struct Entity;
impl EntityName for Entity {
fn table_name(&self) -> &str {
"ImageCache"
}
}
#[derive(Clone, Debug, PartialEq, DeriveModel, DeriveActiveModel, Eq, Serialize, Deserialize)]
pub struct Model {
pub id: String,
pub original_url: String,
pub cdn_url: String,
pub created_at: DateTimeUtc,
pub expires_at: Option<DateTimeUtc>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
pub enum Column {
Id,
#[sea_orm(column_name = "original_url")]
OriginalUrl,
#[sea_orm(column_name = "cdn_url")]
CdnUrl,
#[sea_orm(column_name = "created_at")]
CreatedAt,
#[sea_orm(column_name = "expires_at")]
ExpiresAt,
}
#[derive(Copy, Clone, Debug, EnumIter, DerivePrimaryKey)]
pub enum PrimaryKey {
Id,
}
impl PrimaryKeyTrait for PrimaryKey {
type ValueType = String;
fn auto_increment() -> bool {
false
}
}
#[derive(Copy, Clone, Debug, EnumIter)]
pub enum Relation {}
impl ColumnTrait for Column {
type EntityName = Entity;
fn def(&self) -> ColumnDef {
match self {
Self::Id => ColumnType::String(StringLen::N(36u32)).def(),
Self::OriginalUrl => ColumnType::String(StringLen::N(512u32)).def().unique(),
Self::CdnUrl => ColumnType::String(StringLen::N(512u32)).def(),
Self::CreatedAt => ColumnType::TimestampWithTimeZone.def(),
Self::ExpiresAt => ColumnType::TimestampWithTimeZone.def().null(),
}
}
}
impl RelationTrait for Relation {
fn def(&self) -> RelationDef {
match *self {}
}
}
impl ActiveModelBehavior for ActiveModel {}
src/infrastructure/persistence/entities/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod image_cache;
src/infrastructure/persistence/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod entities;
src/infrastructure/repository/image_cache_seaorm.rs
-131
View File
@@ -1,131 +0,0 @@
//! SeaORM-backed implementation of ImageCacheRepository.
use async_trait::async_trait;
use chrono::Utc;
use deadpool_redis::Pool as RedisPool;
use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, Set};
use std::sync::Arc;
use crate::domain::repository::ImageCacheRepository;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::persistence::entities::image_cache;
pub struct SeaOrmImageCacheRepository {
db: Arc<DatabaseConnection>,
redis: RedisPool,
}
impl SeaOrmImageCacheRepository {
pub fn new(db: Arc<DatabaseConnection>, redis: RedisPool) -> Self {
Self { db, redis }
}
}
#[async_trait]
impl ImageCacheRepository for SeaOrmImageCacheRepository {
async fn get_from_redis(&self, key: &str) -> Option<String> {
Cache::new(&self.redis).get::<String>(key).await
}
async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String> {
Cache::new(&self.redis)
.set_with_ttl(key, &value, ttl)
.await
.map_err(|e| e.to_string())
}
async fn get_from_db(&self, original_url: &str) -> Result<Option<String>, String> {
let entry = image_cache::Entity::find()
.filter(image_cache::Column::OriginalUrl.eq(original_url))
.one(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
Ok(entry.map(|m| m.cdn_url))
}
async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String> {
let model = image_cache::ActiveModel {
id: Set(uuid::Uuid::new_v4().to_string()),
original_url: Set(original_url.to_string()),
cdn_url: Set(cdn_url.to_string()),
created_at: Set(Utc::now()),
expires_at: Set(None),
};
model
.insert(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
Ok(())
}
async fn find_original_from_cdn(&self, cdn_url: &str) -> Result<Option<String>, String> {
let entry = image_cache::Entity::find()
.filter(image_cache::Column::CdnUrl.eq(cdn_url))
.one(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
Ok(entry.map(|m| m.original_url))
}
async fn delete_from_db(&self, original_url: &str) -> Result<(), String> {
image_cache::Entity::delete_many()
.filter(image_cache::Column::OriginalUrl.eq(original_url))
.exec(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
Ok(())
}
async fn delete_from_redis(&self, key: &str) -> Result<(), String> {
Cache::new(&self.redis)
.delete(key)
.await
.map_err(|e| e.to_string())
}
async fn get_lock(&self, key: &str) -> bool {
Cache::new(&self.redis).get::<bool>(key).await.is_some()
}
async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String> {
Cache::new(&self.redis)
.set_with_ttl(key, &true, ttl)
.await
.map_err(|e| e.to_string())
}
async fn release_lock(&self, key: &str) -> Result<(), String> {
self.delete_from_redis(key).await
}
async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String> {
use deadpool_redis::redis::{cmd, AsyncCommands};
let mut conn = self.redis.get().await.map_err(|e| e.to_string())?;
for pattern in patterns {
let mut cursor: u64 = 0;
loop {
let (new_cursor, keys): (u64, Vec<String>) = cmd("SCAN")
.arg(cursor)
.arg("MATCH")
.arg(pattern)
.arg("COUNT")
.arg(100)
.query_async(&mut *conn)
.await
.map_err(|e| e.to_string())?;
if !keys.is_empty() {
let _: usize = conn.del(&keys).await.map_err(|e| e.to_string())?;
}
cursor = new_cursor;
if cursor == 0 {
break;
}
}
}
Ok(())
}
}
src/infrastructure/repository/mod.rs
-2
View File
@@ -1,12 +1,10 @@
pub mod alqanime;
pub mod image_cache_seaorm;
pub mod komik;
pub mod otakudesu;
pub mod parsers;
pub mod proxy;
pub use alqanime::AlqanimeRepository;
pub use image_cache_seaorm::SeaOrmImageCacheRepository;
pub use komik::KomikRepository;
pub use otakudesu::OtakudesuRepository;
pub use proxy::ProxyRepository;
src/infrastructure/repository/parsers/alqanime_parser.rs
+3 -12
View File
@@ -1,7 +1,7 @@
use crate::domain::entity::anime::{
CompleteAnimeItem, DetailGenre, FilterAnimeItem, Genre, GenreAnimeItem, HasPoster,
LatestAnimeItem, OngoingAnimeItem, OngoingAnimeItemWithScore, Pagination,
PaginationWithStringPages, SearchAnimeItem,
CompleteAnimeItem, DetailGenre, FilterAnimeItem, Genre, GenreAnimeItem, LatestAnimeItem,
OngoingAnimeItem, OngoingAnimeItemWithScore, Pagination, PaginationWithStringPages,
SearchAnimeItem,
};
use crate::domain::error::ScrapingError;
use crate::infrastructure::scraping::parsing_utils::parse_html;
@@ -41,15 +41,6 @@ pub struct AlqRecommendation {
pub r#type: String,
}
impl HasPoster for AlqRecommendation {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
use regex::Regex;
use scraper::Selector;
use std::sync::LazyLock;
src/infrastructure/services/images/cache.rs
-1142
View File
File diff suppressed because it is too large Load Diff
src/infrastructure/services/images/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod cache;
src/infrastructure/services/mod.rs
-1
View File
@@ -1 +0,0 @@
pub mod images;
src/lib.rs
-1
View File
@@ -27,4 +27,3 @@ pub mod bootstrap;
pub mod config;
pub mod events;
pub mod observability;
pub mod scheduler;
src/observability/openapi_modules.rs
-2
View File
@@ -41,8 +41,6 @@ use utoipa::OpenApi;
crate::presentation::handler::komik::search_slug,
crate::presentation::handler::komik::search_slug_page,
// Proxy module handlers
crate::presentation::handler::proxy::fetch_with_proxy_only,
crate::presentation::handler::proxy::image_cache,
),
components(
schemas(
src/presentation/handler/anime.rs
+1 -6
View File
@@ -78,12 +78,7 @@ pub struct GenreListResponse {
// ============================================================================
fn make_use_cases(state: &Arc<AppState>) -> AnimeUseCases {
AnimeUseCases::new(
OtakudesuRepository::new(),
state.redis_pool.clone(),
state.db.clone(),
Some(state.image_processing_semaphore.clone()),
)
AnimeUseCases::new(OtakudesuRepository::new(), state.redis_pool.clone())
}
// ============================================================================
src/presentation/handler/anime2.rs
+1 -6
View File
@@ -39,12 +39,7 @@ pub struct FilterQuery {
// ============================================================================
fn make_use_cases(state: &Arc<AppState>) -> Anime2UseCases {
Anime2UseCases::new(
AlqanimeRepository::new(),
state.redis_pool.clone(),
state.db.clone(),
Some(state.image_processing_semaphore.clone()),
)
Anime2UseCases::new(AlqanimeRepository::new(), state.redis_pool.clone())
}
// ============================================================================
src/presentation/handler/komik.rs
+1 -6
View File
@@ -25,12 +25,7 @@ use crate::presentation::state::AppState;
// ============================================================================
fn make_use_cases(state: &Arc<AppState>) -> KomikUseCases {
KomikUseCases::new(
KomikRepository::new(),
state.redis_pool.clone(),
state.db.clone(),
Some(state.image_processing_semaphore.clone()),
)
KomikUseCases::new(KomikRepository::new(), state.redis_pool.clone())
}
// ============================================================================
src/presentation/handler/proxy.rs
+1 -131
View File
@@ -1,131 +1 @@
//! Proxy and image cache API handlers.
use std::sync::Arc;
use axum::extract::{Json, Query, State};
use axum::response::Response;
use serde::Deserialize;
use tracing::info;
use utoipa::{IntoParams, ToSchema};
use crate::application::proxy::use_cases::{
AuditImageCacheResult, ImageCacheResult, ProxyUseCases,
};
use crate::infrastructure::repository::image_cache_seaorm::SeaOrmImageCacheRepository;
use crate::infrastructure::repository::ProxyRepository;
use crate::presentation::error::AppError;
use crate::presentation::state::AppState;
// ============================================================================
// Request DTOs
// ============================================================================
/// Query parameters for proxy fetch (GET).
#[derive(Debug, Deserialize, IntoParams, ToSchema)]
pub struct ProxyParams {
/// URL to fetch via proxy.
pub url: String,
}
/// Request body for image cache (POST).
#[derive(Debug, Deserialize, ToSchema)]
pub struct ImageCacheRequest {
/// Original image URL to cache.
pub url: String,
/// If true, returns original URL immediately and caches in background.
#[serde(default)]
pub lazy: bool,
}
/// Request body for auditing image cache (POST).
#[derive(Debug, Deserialize, ToSchema)]
pub struct AuditImageCacheRequest {
/// Original image URL to audit.
pub url: String,
}
// ============================================================================
// Helper
// ============================================================================
fn make_use_cases(state: &Arc<AppState>) -> ProxyUseCases {
let repo = Arc::new(SeaOrmImageCacheRepository::new(
state.db.clone(),
state.redis_pool.clone(),
));
ProxyUseCases::new(ProxyRepository::new(), repo)
}
// ============================================================================
// Handlers
// ============================================================================
/// GET /api/proxy/croxy — Fetch a URL through the proxy and return raw bytes.
#[utoipa::path(
get,
path = "/api/proxy/croxy",
tag = "proxy",
operation_id = "proxy_croxy",
params(ProxyParams),
responses(
(status = 200, description = "Proxied response", body = Vec::<u8>, content_type = "application/octet-stream"),
(status = 500, description = "Internal Server Error"),
)
)]
pub async fn fetch_with_proxy_only(
State(state): State<Arc<AppState>>,
Query(params): Query<ProxyParams>,
) -> Result<Response, AppError> {
info!("Handling proxy fetch for URL: {}", params.url);
let response = make_use_cases(&state)
.fetch_with_proxy_only(params.url)
.await?;
Ok(response)
}
/// POST /api/proxy/image-cache — Cache an image URL to CDN.
#[utoipa::path(
post,
path = "/api/proxy/image-cache",
tag = "proxy",
operation_id = "proxy_image_cache",
request_body = ImageCacheRequest,
responses(
(status = 200, description = "Image cache result", body = ImageCacheResult),
(status = 500, description = "Internal Server Error"),
)
)]
pub async fn image_cache(
State(state): State<Arc<AppState>>,
Json(req): Json<ImageCacheRequest>,
) -> Result<Json<ImageCacheResult>, AppError> {
info!(
"Handling image cache for URL: {} (lazy: {})",
req.url, req.lazy
);
let result = make_use_cases(&state)
.image_cache(req.url, req.lazy)
.await?;
Ok(Json(result))
}
/// POST /api/proxy/image-cache/audit — Audit and repair a cached image.
#[utoipa::path(
post,
path = "/api/proxy/image-cache/audit",
tag = "proxy",
operation_id = "proxy_image_cache_audit",
request_body = AuditImageCacheRequest,
responses(
(status = 200, description = "Audit result", body = AuditImageCacheResult),
(status = 500, description = "Internal Server Error"),
)
)]
pub async fn audit_image_cache(
State(state): State<Arc<AppState>>,
Json(req): Json<AuditImageCacheRequest>,
) -> Result<Json<AuditImageCacheResult>, AppError> {
info!("Handling audit cache for URL: {}", req.url);
let result = make_use_cases(&state).audit_image_cache(req.url).await?;
Ok(Json(result))
}
//! Proxy handlers.
src/presentation/router.rs
+1 -12
View File
@@ -154,18 +154,7 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
axum::routing::get(crate::presentation::handler::komik::search_slug_page),
)
// Proxy routes
.route(
"/api/proxy/croxy",
axum::routing::get(crate::presentation::handler::proxy::fetch_with_proxy_only),
)
.route(
"/api/proxy/image-cache",
axum::routing::post(crate::presentation::handler::proxy::image_cache),
)
.route(
"/api/proxy/image-cache/audit",
axum::routing::post(crate::presentation::handler::proxy::audit_image_cache),
)
// (all proxy routes removed)
// Health
.route(
"/health",
src/presentation/state.rs
-3
View File
@@ -6,7 +6,6 @@ use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::events::bus::EventBus;
use crate::infrastructure::repository::SeaOrmImageCacheRepository;
/// Shared application state injected into every handler via Axum State.
///
@@ -16,9 +15,7 @@ use crate::infrastructure::repository::SeaOrmImageCacheRepository;
pub struct AppState {
pub redis_pool: Pool,
pub db: Arc<DatabaseConnection>,
pub image_processing_semaphore: Arc<tokio::sync::Semaphore>,
pub event_bus: Arc<EventBus>,
pub image_cache_repo: Arc<SeaOrmImageCacheRepository>,
}
impl AppState {
src/scheduler/cleanup_cache.rs
-222
View File
@@ -1,222 +0,0 @@
//! Scheduled task for cleaning up old cached data.
use async_trait::async_trait;
use sea_orm::*;
use std::sync::Arc;
use tracing::{info, warn};
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::cache::redis_pool::get_redis_pool;
use crate::infrastructure::persistence::entities::image_cache;
use super::ScheduledTask;
/// Cleanup old cache data to prevent disk/memory bloat.
/// Runs daily at 2 AM to clean:
/// - Old image cache entries (>30 days)
/// - Orphaned cache keys in Redis
/// - Expired data without TTL
pub struct CleanupOldCache {
db: Arc<DatabaseConnection>,
}
impl CleanupOldCache {
pub fn new(db: Arc<DatabaseConnection>) -> Self {
Self { db }
}
}
#[async_trait]
impl ScheduledTask for CleanupOldCache {
fn name(&self) -> &'static str {
"cleanup_old_cache"
}
fn schedule(&self) -> &'static str {
// Daily at 2 AM
"0 0 2 * * *"
}
async fn run(&self) {
tracing::debug!("🧹 Starting old cache cleanup...");
let mut total_cleaned = 0;
// 1. Clean old image cache (>30 days)
match self.cleanup_old_images(30).await {
Ok(count) => {
if count > 0 {
info!("✓ Cleaned {} old image cache entries", count);
}
total_cleaned += count;
}
Err(e) => {
warn!("Failed to clean old image cache: {}", e);
}
}
// 2. Clean orphaned Redis keys
match self.cleanup_orphaned_redis_keys().await {
Ok(count) => {
if count > 0 {
info!("✓ Cleaned {} orphaned Redis keys", count);
}
total_cleaned += count;
}
Err(e) => {
warn!("Failed to clean orphaned Redis keys: {}", e);
}
}
// 3. Compact Redis memory
match self.compact_redis_memory().await {
Ok(()) => {
if total_cleaned > 0 {
info!("✓ Redis memory compacted");
}
}
Err(e) => {
warn!("Failed to compact Redis memory: {}", e);
}
}
if total_cleaned > 0 {
info!("🎉 Cache cleanup complete: {} items cleaned", total_cleaned);
}
}
}
impl CleanupOldCache {
/// Remove image cache entries older than specified days.
async fn cleanup_old_images(&self, days: i64) -> Result<usize, String> {
use chrono::{Duration, Utc};
let cutoff = Utc::now() - Duration::days(days);
// Find old entries
let old_images = image_cache::Entity::find()
.filter(image_cache::Column::CreatedAt.lt(cutoff))
.all(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
let count = old_images.len();
if count == 0 {
return Ok(0);
}
// Delete from database
let ids: Vec<String> = old_images.iter().map(|img| img.id.clone()).collect();
image_cache::Entity::delete_many()
.filter(image_cache::Column::Id.is_in(ids))
.exec(self.db.as_ref())
.await
.map_err(|e| e.to_string())?;
// Also clean from Redis
let redis_pool = get_redis_pool().map_err(|e| e.to_string())?;
let cache = Cache::new(redis_pool);
for img in old_images {
let cache_key = format!("img_cache:{}", Self::hash_url(&img.original_url));
let _ = cache.delete(&cache_key).await;
}
Ok(count)
}
/// Clean orphaned Redis keys (keys without TTL that shouldn't exist).
async fn cleanup_orphaned_redis_keys(&self) -> Result<usize, String> {
use deadpool_redis::redis::AsyncCommands;
let pool = get_redis_pool().map_err(|e| e.to_string())?;
let mut conn = pool
.get()
.await
.map_err(|e| format!("Failed to get Redis connection: {}", e))?;
let mut cleaned = 0;
// Find keys without TTL (should not exist)
let patterns = vec!["anime:*", "komik:*", "user:*:profile", "img_cache:*"];
for pattern in patterns {
let keys = {
let mut iter: deadpool_redis::redis::AsyncIter<'_, String> = conn
.scan_match(pattern)
.await
.map_err(|e| format!("Failed to scan keys: {}", e))?;
let mut keys = Vec::new();
while let Some(key_result) = iter.next_item().await {
if let Ok(key) = key_result {
keys.push(key);
}
}
keys
};
for key in keys {
let ttl: i64 = conn.ttl(key.as_str()).await.unwrap_or(-1);
// TTL = -1 means no expiration (orphaned)
// TTL = -2 means key doesn't exist
if ttl == -1 {
// Set a default TTL of 7 days for orphaned keys
let _: () = conn.expire(key.as_str(), 604800).await.unwrap_or(());
cleaned += 1;
}
}
}
Ok(cleaned)
}
/// Compact Redis memory to free up fragmented space.
async fn compact_redis_memory(&self) -> Result<(), String> {
let pool = get_redis_pool().map_err(|e| e.to_string())?;
let mut conn = pool
.get()
.await
.map_err(|e| format!("Failed to get Redis connection: {}", e))?;
// Run MEMORY PURGE command - using cmd method on connection
let _: String = deadpool_redis::redis::cmd("MEMORY")
.arg("PURGE")
.query_async(&mut *conn)
.await
.map_err(|e| format!("Failed to purge memory: {}", e))?;
Ok(())
}
/// Simple hash function for URL (same as in image_cache.rs).
/// Simple hash function for URL (same as in image_cache.rs).
fn hash_url(url: &str) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(url.as_bytes());
format!("{:x}", hasher.finalize())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_schedule() {
use sea_orm::{DatabaseBackend, MockDatabase};
let db = MockDatabase::new(DatabaseBackend::Sqlite).into_connection();
let task = CleanupOldCache { db: Arc::new(db) };
assert_eq!(task.schedule(), "0 0 2 * * *");
}
#[test]
fn test_hash_url() {
let hash = CleanupOldCache::hash_url("https://example.com/image.jpg");
assert_eq!(hash.len(), 64); // SHA256 hash length
}
}
src/scheduler/mod.rs
-5
View File
@@ -1,5 +0,0 @@
pub mod cleanup_cache;
pub mod runner;
pub use cleanup_cache::CleanupOldCache;
pub use runner::{ScheduledTask, Scheduler};
src/scheduler/runner.rs
-93
View File
@@ -1,93 +0,0 @@
//! Scheduler implementation using tokio-cron-scheduler.
use async_trait::async_trait;
use std::sync::Arc;
use tokio_cron_scheduler::{Job, JobScheduler};
use tracing::info;
/// Trait for scheduled tasks.
#[async_trait]
pub trait ScheduledTask: Send + Sync {
/// Task name for logging.
fn name(&self) -> &'static str;
/// Cron expression (e.g., "0 * * * * *" for every minute).
fn schedule(&self) -> &'static str;
/// Execute the task.
async fn run(&self);
}
/// Scheduler for running cron jobs.
pub struct Scheduler {
inner: JobScheduler,
}
impl Scheduler {
/// Create a new scheduler.
pub async fn new() -> anyhow::Result<Self> {
let scheduler = JobScheduler::new().await?;
Ok(Self { inner: scheduler })
}
/// Add a task to the scheduler.
pub async fn add<T: ScheduledTask + 'static>(&self, task: T) -> anyhow::Result<()> {
let task = Arc::new(task);
let task_name = task.name();
let schedule = task.schedule();
let job = Job::new_async(schedule, move |_uuid, _lock| {
let task = Arc::clone(&task);
Box::pin(async move {
tracing::debug!("Running scheduled task: {}", task.name());
task.run().await;
})
})?;
self.inner.add(job).await?;
info!("Scheduled task '{}' with cron: {}", task_name, schedule);
Ok(())
}
/// Add a simple job with a closure.
pub async fn add_job<F, Fut>(
&self,
name: &'static str,
schedule: &str,
f: F,
) -> anyhow::Result<()>
where
F: Fn() -> Fut + Send + Sync + 'static,
Fut: std::future::Future<Output = ()> + Send + 'static,
{
let f = Arc::new(f);
let job = Job::new_async(schedule, move |_uuid, _lock| {
let f = Arc::clone(&f);
Box::pin(async move {
info!("Running scheduled job: {}", name);
f().await;
})
})?;
self.inner.add(job).await?;
info!("Scheduled job '{}' with cron: {}", name, schedule);
Ok(())
}
/// Start the scheduler.
pub async fn start(&self) -> anyhow::Result<()> {
info!("Starting scheduler");
self.inner.start().await?;
Ok(())
}
/// Stop the scheduler.
pub async fn shutdown(&mut self) -> anyhow::Result<()> {
info!("Shutting down scheduler");
self.inner.shutdown().await?;
Ok(())
}
}
// Real scheduled tasks with actual implementations
// Real scheduled tasks with actual implementations