feat: full clean architecture refactor — domain → application → infrastructure → presentation

- Restructure from Modular MVC to Clean Architecture with 4 strict layers
- Domain: entities (anime/komik/proxy), Repository traits, typed errors
- Application: use case classes with proper error propagation
- Infrastructure: repository impls, parsers, Redis cache, HTTP/scraping, browser
- Presentation: Axum handlers, DTOs, AppState, router, AppError+IntoResponse
- Migrate all parsers (otakudesu, alqanime, komik) to native infra implementations
- Replace once_cell::sync::Lazy/OnceCell with std::sync::LazyLock/OnceLock
- Remove 150+ old files in modules/ and shared/ directories
- Remove once_cell from Cargo.toml dependencies
- Fix test/debug binaries to use new import paths
- Zero new clippy warnings

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
asepharyana
2026-07-21 12:54:23 +07:00
co-authored by Claude Opus 4.8
parent 80c96eaa42
commit 776fa828d8
205 changed files with 4241 additions and 14569 deletions
Generated
-1
View File
@@ -3347,7 +3347,6 @@ dependencies = [
"itertools", "itertools",
"log", "log",
"mime_guess", "mime_guess",
"once_cell",
"opentelemetry", "opentelemetry",
"opentelemetry-otlp", "opentelemetry-otlp",
"opentelemetry-semantic-conventions", "opentelemetry-semantic-conventions",
-1
View File
@@ -38,7 +38,6 @@ async-trait = "0.1.89"
regex = "1.12.2" regex = "1.12.2"
infer = "0.19.0" infer = "0.19.0"
once_cell = "1.21.3"
urlencoding = "2.1" urlencoding = "2.1"
url = "2.5.8" url = "2.5.8"
-51
View File
@@ -1,51 +0,0 @@
use std::sync::Arc;
use axum::Router;
use sea_orm::DatabaseConnection;
use tower_http::compression::{CompressionLayer, CompressionLevel};
use tower_http::cors::CorsLayer;
use utoipa::OpenApi;
use utoipa_swagger_ui::SwaggerUi;
use crate::shared::observability::openapi::ApiDoc;
use crate::shared::state::AppState;
pub async fn build_router(
app_state: Arc<AppState>,
db: Arc<DatabaseConnection>,
) -> anyhow::Result<Router> {
init_scheduler(db).await?;
let mut openapi = ApiDoc::openapi();
openapi.merge(crate::shared::observability::openapi_modules::ModuleApiDoc::openapi());
let app = crate::modules::routes(Router::new())
.merge(SwaggerUi::new("/docs").url("/api-docs/openapi.json", openapi))
.with_state(app_state)
.layer(axum::middleware::from_fn(
crate::shared::observability::metrics::otel_metrics_middleware,
))
.layer(CompressionLayer::new().quality(CompressionLevel::Fastest))
.layer(CorsLayer::permissive());
Ok(app)
}
async fn init_scheduler(db: Arc<DatabaseConnection>) -> anyhow::Result<()> {
let scheduler = crate::shared::scheduler::Scheduler::new()
.await
.map_err(|e| anyhow::anyhow!("Failed to create scheduler: {}", e))?;
let cache_cleanup = crate::shared::scheduler::CleanupOldCache::new(db);
scheduler
.add(cache_cleanup)
.await
.map_err(|e| anyhow::anyhow!("Failed to add cache cleanup: {}", e))?;
scheduler
.start()
.await
.map_err(|e| anyhow::anyhow!("Failed to start scheduler: {}", e))?;
tracing::info!("✓ Scheduler started");
Ok(())
}
+1
View File
@@ -0,0 +1 @@
pub mod use_cases;
+244
View File
@@ -0,0 +1,244 @@
//! Anime (Otakudesu) application use cases.
//!
//! Orchestrates repository fetching, caching, and image poster processing.
//! Returns pure domain types — no DTOs.
use std::sync::Arc;
use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::entity::anime::*;
use crate::domain::error::*;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::OtakudesuRepository;
use crate::infrastructure::services::images::cache::{
cache_image_urls_batch_lazy, get_cached_or_original,
};
const INDEX_CACHE_TTL: u64 = 10;
const GENRE_LIST_CACHE_TTL: u64 = 3600;
const DEFAULT_CACHE_TTL: u64 = 300;
pub struct AnimeUseCases {
repository: OtakudesuRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
}
impl AnimeUseCases {
pub fn new(
repository: OtakudesuRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
}
fn cache(&self) -> Cache<'_> {
Cache::new(&self.redis_pool)
}
pub async fn get_anime_index(&self) -> Result<AnimeData, DomainError> {
self.cache()
.get_or_set("anime:index:v2", INDEX_CACHE_TTL, || async {
let mut data = self
.repository
.fetch_anime_index()
.await
.map_err(|e| e.to_string())?;
if data.ongoing_anime.is_empty() && data.complete_anime.is_empty() {
return Err("Empty anime index — refusing to cache".to_string());
}
let mut posters: Vec<String> = data
.ongoing_anime
.iter()
.map(|item| item.poster.clone())
.collect();
posters.extend(data.complete_anime.iter().map(|item| item.poster.clone()));
let cached_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
posters,
self.semaphore.clone(),
)
.await;
let ongoing_len = data.ongoing_anime.len();
for (i, item) in data.ongoing_anime.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(i) {
item.poster = url.clone();
}
}
for (i, item) in data.complete_anime.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(ongoing_len + i) {
item.poster = url.clone();
}
}
Ok(data)
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
pub async fn get_genres(&self) -> Result<Vec<Genre>, DomainError> {
self.cache()
.get_or_set("anime:genres:list", GENRE_LIST_CACHE_TTL, || async {
self.repository
.fetch_genres()
.await
.map_err(|e| e.to_string())
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
pub async fn get_anime_detail(&self, slug: String) -> Result<AnimeDetailData, DomainError> {
let cache_key = format!("anime:detail:{}", slug);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
let mut data = self
.repository
.fetch_anime_detail(&slug)
.await
.map_err(|e| e.to_string())?;
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
let rec_posters: Vec<String> = data
.recommendations
.iter()
.map(|r| r.poster.clone())
.collect();
let cached_rec_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
rec_posters,
self.semaphore.clone(),
)
.await;
for (i, rec) in data.recommendations.iter_mut().enumerate() {
if let Some(url) = cached_rec_posters.get(i) {
rec.poster = url.clone();
}
}
Ok(data)
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
pub async fn get_complete_anime_page(
&self,
slug: String,
) -> Result<(Vec<CompleteAnimeListItem>, Pagination), DomainError> {
let cache_key = format!("anime:complete:{}", slug);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
self.repository
.fetch_complete_anime_page(&slug)
.await
.map_err(|e| e.to_string())
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
pub async fn get_ongoing_anime_page(
&self,
slug: String,
) -> Result<(Vec<OngoingAnimeListItem>, Pagination), DomainError> {
let cache_key = format!("anime:ongoing:{}", slug);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
self.repository
.fetch_ongoing_anime_page(&slug)
.await
.map_err(|e| e.to_string())
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
pub async fn get_latest_anime_page(
&self,
slug: String,
) -> Result<(Vec<LatestAnimeItem>, Pagination), DomainError> {
let cache_key = format!("anime:latest:{}", slug);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
self.repository
.fetch_latest_anime_page(&slug)
.await
.map_err(|e| e.to_string())
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
pub async fn get_search_anime_page(
&self,
slug: String,
page: String,
) -> Result<(Vec<SearchAnimeItem>, Pagination), DomainError> {
let cache_key = format!("anime:search:{}:{}", slug, page);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
self.repository
.fetch_search_anime_page(&slug, &page)
.await
.map_err(|e| e.to_string())
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
pub async fn get_genre_anime_page(
&self,
genre_slug: String,
page: String,
) -> Result<(Vec<GenreAnimeItem>, Pagination), DomainError> {
let cache_key = format!("anime:genre:{}:{}", genre_slug, page);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
self.repository
.fetch_genre_anime_page(&genre_slug, &page)
.await
.map_err(|e| e.to_string())
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
pub async fn get_anime_full(&self, slug: String) -> Result<AnimeFullData, DomainError> {
let cache_key = format!("anime:full:{}", slug);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
self.repository
.fetch_anime_full(&slug)
.await
.map_err(|e| e.to_string())
})
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
}
}
+1
View File
@@ -0,0 +1 @@
pub mod use_cases;
@@ -1,16 +1,110 @@
//! Anime2 (Alqanime) application use cases.
//!
//! Orchestrates repository fetching, caching, and image poster processing.
//!
//! TODO: Move parsers from `crate::modules::anime2::parser` to
//! `crate::infrastructure::repository::parsers::alqanime_parser`.
//! TODO: Move response DTOs to `crate::presentation::dto::anime2`.
//! TODO: Once parsers return domain types, replace shared types with
//! `crate::domain::entity::anime::{GenreAnimeItem, SearchAnimeItem, LatestAnimeItem}`.
use std::sync::Arc; use std::sync::Arc;
use crate::modules::anime2::parser; use deadpool_redis::Pool;
use crate::modules::anime2::repository::Anime2Repository; use sea_orm::DatabaseConnection;
use crate::modules::anime2::types::{DetailResponse, GenresResponse};
use crate::shared::database::traits::scraping_repository::ScrapingRepository; use crate::domain::error::*;
use crate::shared::errors::AppError; use crate::domain::repository::ScrapingRepository;
use crate::shared::services::images::cache::{ use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::AlqanimeRepository;
use crate::infrastructure::services::images::cache::{
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original, apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
}; };
use crate::shared::state::AppState;
use crate::shared::types::ApiResponse; use crate::infrastructure::repository::parsers::alqanime_parser as parser;
use crate::shared::utils::Cache;
use crate::domain::entity::anime::{
CompleteAnimeItem, FilterAnimeItem, Genre, GenreAnimeItem, HasPoster, LatestAnimeItem,
OngoingAnimeItemWithScore, Pagination, SearchAnimeItem,
};
use crate::presentation::dto::common::ApiResponse;
// Re-export types for handlers to use
pub use crate::domain::entity::anime::{
CompleteAnimeItem as Anime2CompleteAnimeItem, GenreAnimeItem as Anime2GenreItem,
LatestAnimeItem as Anime2LatestItem, OngoingAnimeItemWithScore as Anime2OngoingItem,
SearchAnimeItem as Anime2SearchItem,
};
pub use crate::infrastructure::repository::parsers::alqanime_parser::{
AlqDetailData, AlqDownloadItem, AlqLink, AlqRecommendation,
};
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
// Response types (will move to presentation::dto::anime2)
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2Item {
pub title: String,
pub slug: String,
pub poster: String,
pub status: String,
pub r#type: String,
pub score: String,
pub anime_url: String,
}
impl HasPoster for Anime2Item {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2Data {
pub ongoing_anime: Vec<Anime2Item>,
pub complete_anime: Vec<Anime2Item>,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2Response {
pub status: String,
pub data: Anime2Data,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct GenresResponse {
pub status: String,
pub data: Vec<Genre>,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct FiltersApplied {
pub genre: Option<String>,
pub status: Option<String>,
pub r#type: Option<String>,
pub order: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct FilterResponse {
pub success: bool,
pub data: Vec<FilterAnimeItem>,
pub pagination: Pagination,
pub filters_applied: FiltersApplied,
pub status: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct DetailResponse {
pub status: String,
pub data: AlqDetailData,
}
const INDEX_CACHE_TTL: u64 = 300; const INDEX_CACHE_TTL: u64 = 300;
const GENRE_LIST_CACHE_TTL: u64 = 3600; const GENRE_LIST_CACHE_TTL: u64 = 3600;
@@ -22,22 +116,38 @@ const LATEST_CACHE_TTL: u64 = 120;
const ONGOING_CACHE_TTL: u64 = 300; const ONGOING_CACHE_TTL: u64 = 300;
const COMPLETE_CACHE_TTL: u64 = 300; const COMPLETE_CACHE_TTL: u64 = 300;
pub struct Anime2Service { // ============================================================================
repository: Anime2Repository, // Use case struct
// ============================================================================
pub struct Anime2UseCases {
repository: AlqanimeRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
} }
impl Anime2Service { impl Anime2UseCases {
pub fn new(repository: Anime2Repository) -> Self { pub fn new(
Self { repository } repository: AlqanimeRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
} }
pub async fn index( fn cache(&self) -> Cache<'_> {
&self, Cache::new(&self.redis_pool)
app_state: Arc<AppState>, }
) -> Result<crate::modules::anime2::types::Anime2Response, AppError> {
let cache = Cache::new(&app_state.redis_pool);
cache pub async fn index(&self) -> Result<Anime2Response, DomainError> {
self.cache()
.get_or_set("anime2:index", INDEX_CACHE_TTL, || async { .get_or_set("anime2:index", INDEX_CACHE_TTL, || async {
let ongoing_html = self let ongoing_html = self
.repository .repository
@@ -50,7 +160,7 @@ impl Anime2Service {
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let mut data = tokio::task::spawn_blocking(move || { let data = tokio::task::spawn_blocking(move || {
Ok::<_, String>(( Ok::<_, String>((
parser::parse_ongoing_anime(&ongoing_html).map_err(|e| e.to_string())?, parser::parse_ongoing_anime(&ongoing_html).map_err(|e| e.to_string())?,
parser::parse_complete_anime(&complete_html).map_err(|e| e.to_string())?, parser::parse_complete_anime(&complete_html).map_err(|e| e.to_string())?,
@@ -58,47 +168,72 @@ impl Anime2Service {
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
let mut ongoing: Vec<Anime2Item> = data
.0
.into_iter()
.map(|item| Anime2Item {
title: item.title,
slug: item.slug,
poster: item.poster,
status: String::new(),
r#type: String::new(),
score: item.current_episode,
anime_url: item.anime_url,
})
.collect();
let mut complete: Vec<Anime2Item> = data
.1
.into_iter()
.map(|item| Anime2Item {
title: item.title,
slug: item.slug,
poster: item.poster,
status: String::new(),
r#type: String::new(),
score: item.episode_count,
anime_url: item.anime_url,
})
.collect();
let mut posters: Vec<String> = let mut posters: Vec<String> =
data.0.iter().map(|item| item.poster.clone()).collect(); ongoing.iter().map(|item| item.poster.clone()).collect();
posters.extend(data.1.iter().map(|item| item.poster.clone())); posters.extend(complete.iter().map(|item| item.poster.clone()));
let cached_posters = cache_image_urls_batch_lazy( let cached_posters = cache_image_urls_batch_lazy(
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
posters, posters,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
let ongoing_len = data.0.len(); let ongoing_len = ongoing.len();
for (i, item) in data.0.iter_mut().enumerate() { for (i, item) in ongoing.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(i) { if let Some(url) = cached_posters.get(i) {
item.poster = url.clone(); item.poster = url.clone();
} }
} }
for (i, item) in data.1.iter_mut().enumerate() { for (i, item) in complete.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(ongoing_len + i) { if let Some(url) = cached_posters.get(ongoing_len + i) {
item.poster = url.clone(); item.poster = url.clone();
} }
} }
Ok(crate::modules::anime2::types::Anime2Response { Ok(Anime2Response {
status: "Ok".to_string(), status: "Ok".to_string(),
data: crate::modules::anime2::types::Anime2Data { data: Anime2Data {
ongoing_anime: data.0, ongoing_anime: ongoing,
complete_anime: data.1, complete_anime: complete,
}, },
}) })
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn genre_list(&self, app_state: Arc<AppState>) -> Result<GenresResponse, AppError> { pub async fn genre_list(&self) -> Result<GenresResponse, DomainError> {
let cache = Cache::new(&app_state.redis_pool); self.cache()
cache
.get_or_set("anime2:genres:list:v3", GENRE_LIST_CACHE_TTL, || async { .get_or_set("anime2:genres:list:v3", GENRE_LIST_CACHE_TTL, || async {
let html = self let html = self
.repository .repository
@@ -118,19 +253,17 @@ impl Anime2Service {
}) })
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn filter( pub async fn filter(
&self, &self,
app_state: Arc<AppState>,
page: u32, page: u32,
genre: Option<String>, genre: Option<String>,
status: Option<String>, status: Option<String>,
anime_type: Option<String>, anime_type: Option<String>,
order: String, order: String,
) -> Result<crate::modules::anime2::types::FilterResponse, AppError> { ) -> Result<FilterResponse, DomainError> {
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!( let cache_key = format!(
"anime2:filter:{}:{:?}:{:?}:{:?}:{}", "anime2:filter:{}:{:?}:{:?}:{:?}:{}",
page, genre, status, anime_type, order page, genre, status, anime_type, order
@@ -139,7 +272,7 @@ impl Anime2Service {
let status_clone = status.clone(); let status_clone = status.clone();
let anime_type_clone = anime_type.clone(); let anime_type_clone = anime_type.clone();
cache self.cache()
.get_or_set(&cache_key, FILTER_CACHE_TTL, || async { .get_or_set(&cache_key, FILTER_CACHE_TTL, || async {
let mut url = self.repository.filter_url(page, &order); let mut url = self.repository.filter_url(page, &order);
@@ -160,26 +293,25 @@ impl Anime2Service {
.fetch_html(&url) .fetch_html(&url)
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (data, pagination) = tokio::task::spawn_blocking(move || { let (mut data, pagination) = tokio::task::spawn_blocking(move || {
parser::parse_filter_page(&html, page).map_err(|e| e.to_string()) parser::parse_filter_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
let mut final_data = data;
apply_cached_posters( apply_cached_posters(
&mut final_data, &mut data,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(crate::modules::anime2::types::FilterResponse { Ok(FilterResponse {
success: true, success: true,
data: final_data, data,
pagination, pagination,
filters_applied: crate::modules::anime2::types::FiltersApplied { filters_applied: FiltersApplied {
genre: genre_clone, genre: genre_clone,
status: status_clone, status: status_clone,
r#type: anime_type_clone, r#type: anime_type_clone,
@@ -189,18 +321,13 @@ impl Anime2Service {
}) })
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn detail( pub async fn detail(&self, slug: String) -> Result<DetailResponse, DomainError> {
&self,
app_state: Arc<AppState>,
slug: String,
) -> Result<DetailResponse, AppError> {
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("anime2:detail:{}", slug); let cache_key = format!("anime2:detail:{}", slug);
cache self.cache()
.get_or_set(&cache_key, DETAIL_CACHE_TTL, || async { .get_or_set(&cache_key, DETAIL_CACHE_TTL, || async {
let detail_url = self.repository.detail_url(&slug); let detail_url = self.repository.detail_url(&slug);
let image_url = self.repository.detail_image_url(&slug); let image_url = self.repository.detail_image_url(&slug);
@@ -231,25 +358,26 @@ impl Anime2Service {
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
data.poster = get_cached_or_original( data.poster = get_cached_or_original(
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
&data.poster, &data.poster,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
data.poster2 = get_cached_or_original( data.poster2 = get_cached_or_original(
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
&data.poster2, &data.poster2,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
// Recommendation in modules::anime2::types implements HasPoster
apply_cached_posters( apply_cached_posters(
&mut data.recommendations, &mut data.recommendations,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
@@ -259,195 +387,173 @@ impl Anime2Service {
}) })
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn genre_slug( pub async fn genre_slug(
&self, &self,
app_state: Arc<AppState>,
genre_slug: String, genre_slug: String,
page: u32, page: u32,
) -> Result<ApiResponse<Vec<crate::shared::types::entities::anime::GenreAnimeItem>>, AppError> ) -> Result<ApiResponse<Vec<GenreAnimeItem>>, DomainError> {
{
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("anime2:genre:{}:{}", genre_slug, page); let cache_key = format!("anime2:genre:{}:{}", genre_slug, page);
cache self.cache()
.get_or_set(&cache_key, GENRE_CACHE_TTL, || async { .get_or_set(&cache_key, GENRE_CACHE_TTL, || async {
let html = self let html = self
.repository .repository
.fetch_html(&self.repository.genre_page_url(&genre_slug, page)) .fetch_html(&self.repository.genre_page_url(&genre_slug, page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (data, _pagination) = tokio::task::spawn_blocking(move || { let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_genre_page(&html, page).map_err(|e| e.to_string()) parser::parse_genre_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
let mut final_data = data;
apply_cached_posters( apply_cached_posters(
&mut final_data, &mut data,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(ApiResponse::success(final_data)) Ok(ApiResponse::success(data))
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn search( pub async fn search(
&self, &self,
app_state: Arc<AppState>,
query: String, query: String,
page: u32, page: u32,
) -> Result<ApiResponse<Vec<crate::shared::types::entities::anime::SearchAnimeItem>>, AppError> ) -> Result<ApiResponse<Vec<SearchAnimeItem>>, DomainError> {
{
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("anime2:search:{}:{}", query, page); let cache_key = format!("anime2:search:{}:{}", query, page);
cache self.cache()
.get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async {
let html = self let html = self
.repository .repository
.fetch_html(&self.repository.search_url(&query, page)) .fetch_html(&self.repository.search_url(&query, page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (data, _pagination) = tokio::task::spawn_blocking(move || { let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_search_page(&html, page).map_err(|e| e.to_string()) parser::parse_search_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
let mut final_data = data;
apply_cached_posters( apply_cached_posters(
&mut final_data, &mut data,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(ApiResponse::success(final_data)) Ok(ApiResponse::success(data))
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn latest( pub async fn latest(
&self, &self,
app_state: Arc<AppState>,
page: u32, page: u32,
) -> Result<ApiResponse<Vec<crate::shared::types::entities::anime::LatestAnimeItem>>, AppError> ) -> Result<ApiResponse<Vec<LatestAnimeItem>>, DomainError> {
{
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("anime2:latest:{}", page); let cache_key = format!("anime2:latest:{}", page);
cache self.cache()
.get_or_set(&cache_key, LATEST_CACHE_TTL, || async { .get_or_set(&cache_key, LATEST_CACHE_TTL, || async {
let html = self let html = self
.repository .repository
.fetch_html(&self.repository.latest_url(page)) .fetch_html(&self.repository.latest_url(page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (data, _pagination) = tokio::task::spawn_blocking(move || { let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_latest_page(&html, page).map_err(|e| e.to_string()) parser::parse_latest_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
let mut final_data = data;
apply_cached_posters( apply_cached_posters(
&mut final_data, &mut data,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(ApiResponse::success(final_data)) Ok(ApiResponse::success(data))
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn ongoing_anime( pub async fn ongoing_anime(
&self, &self,
app_state: Arc<AppState>,
page: u32, page: u32,
) -> Result< ) -> Result<ApiResponse<Vec<OngoingAnimeItemWithScore>>, DomainError> {
ApiResponse<Vec<crate::shared::types::entities::anime::OngoingAnimeItemWithScore>>,
AppError,
> {
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("anime2:ongoing:{}", page); let cache_key = format!("anime2:ongoing:{}", page);
cache self.cache()
.get_or_set(&cache_key, ONGOING_CACHE_TTL, || async { .get_or_set(&cache_key, ONGOING_CACHE_TTL, || async {
let html = self let html = self
.repository .repository
.fetch_html(&self.repository.ongoing_url(page)) .fetch_html(&self.repository.ongoing_url(page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (data, _pagination) = tokio::task::spawn_blocking(move || { let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string()) parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
let mut final_data = data;
apply_cached_posters( apply_cached_posters(
&mut final_data, &mut data,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(ApiResponse::success(final_data)) Ok(ApiResponse::success(data))
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn complete_anime( pub async fn complete_anime(
&self, &self,
app_state: Arc<AppState>,
page: u32, page: u32,
) -> Result<ApiResponse<Vec<crate::shared::types::entities::anime::CompleteAnimeItem>>, AppError> ) -> Result<ApiResponse<Vec<CompleteAnimeItem>>, DomainError> {
{
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("anime2:complete:{}", page); let cache_key = format!("anime2:complete:{}", page);
cache self.cache()
.get_or_set(&cache_key, COMPLETE_CACHE_TTL, || async { .get_or_set(&cache_key, COMPLETE_CACHE_TTL, || async {
let html = self let html = self
.repository .repository
.fetch_html(&self.repository.complete_url(page)) .fetch_html(&self.repository.complete_url(page))
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let (data, _pagination) = tokio::task::spawn_blocking(move || { let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_complete_page(&html, page).map_err(|e| e.to_string()) parser::parse_complete_page(&html, page).map_err(|e| e.to_string())
}) })
.await .await
.map_err(|e| e.to_string())??; .map_err(|e| e.to_string())??;
let mut final_data = data;
apply_cached_posters( apply_cached_posters(
&mut final_data, &mut data,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(ApiResponse::success(final_data)) Ok(ApiResponse::success(data))
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
} }
+1
View File
@@ -0,0 +1 @@
pub mod use_cases;
@@ -1,17 +1,27 @@
//! Komik application use cases.
//!
//! Orchestrates repository fetching, caching, and image poster processing.
//!
//! TODO: Move parsers from `crate::modules::komik::parser` to
//! `crate::infrastructure::repository::parsers::komik_parser`.
//! TODO: Move response DTOs to `crate::presentation::dto::komik`.
use std::sync::Arc; use std::sync::Arc;
use crate::modules::komik::parser; use deadpool_redis::Pool;
use crate::modules::komik::repository::KomikRepository; use sea_orm::DatabaseConnection;
use crate::modules::komik::types::{
ChapterResponse, DetailResponse, GenreKomikResponse, GenresResponse, SearchKomikResponse, use crate::domain::entity::anime::Pagination;
}; use crate::domain::entity::komik::{ChapterData, DetailData, KomikGenre, KomikItem};
use crate::shared::database::traits::scraping_repository::ScrapingRepository; use crate::domain::error::*;
use crate::shared::errors::AppError; use crate::domain::repository::ScrapingRepository;
use crate::shared::services::images::cache::{ use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::KomikRepository;
use crate::infrastructure::services::images::cache::{
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original, apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
}; };
use crate::shared::state::AppState;
use crate::shared::utils::Cache; use crate::infrastructure::repository::parsers::komik_parser as parser;
const GENRE_LIST_CACHE_TTL: u64 = 3600; const GENRE_LIST_CACHE_TTL: u64 = 3600;
const GENRE_CACHE_TTL: u64 = 300; const GENRE_CACHE_TTL: u64 = 300;
@@ -19,21 +29,39 @@ const DETAIL_CACHE_TTL: u64 = 300;
const CHAPTER_CACHE_TTL: u64 = 300; const CHAPTER_CACHE_TTL: u64 = 300;
const SEARCH_CACHE_TTL: u64 = 300; const SEARCH_CACHE_TTL: u64 = 300;
pub struct KomikService { // ============================================================================
// Use case struct
// ============================================================================
pub struct KomikUseCases {
repository: KomikRepository, repository: KomikRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
} }
impl KomikService { impl KomikUseCases {
pub fn new(repository: KomikRepository) -> Self { pub fn new(
Self { repository } repository: KomikRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
} }
pub async fn genre_list(&self, app_state: Arc<AppState>) -> Result<GenresResponse, AppError> { fn cache(&self) -> Cache<'_> {
let cache = Cache::new(&app_state.redis_pool); Cache::new(&self.redis_pool)
let cache_key = "komik:genres:list:v3"; }
cache pub async fn genre_list(&self) -> Result<Vec<KomikGenre>, DomainError> {
.get_or_set(cache_key, GENRE_LIST_CACHE_TTL, || async { self.cache()
.get_or_set("komik:genres:list:v3", GENRE_LIST_CACHE_TTL, || async {
let html = self let html = self
.repository .repository
.fetch_html(&self.repository.api_url()) .fetch_html(&self.repository.api_url())
@@ -44,25 +72,20 @@ impl KomikService {
.map_err(|e| e.to_string())? .map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
Ok(GenresResponse { Ok(genres)
status: "Ok".to_string(),
data: genres,
})
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn genre_slug( pub async fn genre_slug(
&self, &self,
genre_slug: String, genre_slug: String,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<GenreKomikResponse, AppError> { let page = 1u32;
let page = 1;
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page); let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page);
cache self.cache()
.get_or_set(&cache_key, GENRE_CACHE_TTL, || async { .get_or_set(&cache_key, GENRE_CACHE_TTL, || async {
let url = self.repository.genre_url(&genre_slug, page); let url = self.repository.genre_url(&genre_slug, page);
let html = self let html = self
@@ -79,33 +102,26 @@ impl KomikService {
apply_cached_posters( apply_cached_posters(
&mut komik_list, &mut komik_list,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(GenreKomikResponse { Ok((komik_list, pagination))
status: "Ok".to_string(),
genre: genre_slug.clone(),
data: komik_list,
pagination,
})
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn genre_slug_page( pub async fn genre_slug_page(
&self, &self,
genre_slug: String, genre_slug: String,
page: u32, page: u32,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<GenreKomikResponse, AppError> {
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page); let cache_key = format!("komik:genre:{}:{}:v2", genre_slug, page);
cache self.cache()
.get_or_set(&cache_key, GENRE_CACHE_TTL, || async { .get_or_set(&cache_key, GENRE_CACHE_TTL, || async {
let url = self.repository.genre_url(&genre_slug, page); let url = self.repository.genre_url(&genre_slug, page);
let html = self let html = self
@@ -122,32 +138,22 @@ impl KomikService {
apply_cached_posters( apply_cached_posters(
&mut komik_list, &mut komik_list,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(GenreKomikResponse { Ok((komik_list, pagination))
status: "Ok".to_string(),
genre: genre_slug.clone(),
data: komik_list,
pagination,
})
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn detail_slug( pub async fn detail_slug(&self, komik_id: String) -> Result<DetailData, DomainError> {
&self,
komik_id: String,
app_state: Arc<AppState>,
) -> Result<DetailResponse, AppError> {
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("komik:detail:{}", komik_id); let cache_key = format!("komik:detail:{}", komik_id);
cache self.cache()
.get_or_set(&cache_key, DETAIL_CACHE_TTL, || async { .get_or_set(&cache_key, DETAIL_CACHE_TTL, || async {
let url = self.repository.detail_url(&komik_id); let url = self.repository.detail_url(&komik_id);
let html = self let html = self
@@ -164,29 +170,24 @@ impl KomikService {
if !data.poster.is_empty() { if !data.poster.is_empty() {
data.poster = get_cached_or_original( data.poster = get_cached_or_original(
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
&data.poster, &data.poster,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
} }
Ok(DetailResponse { status: true, data }) Ok(data)
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn chapter_slug( pub async fn chapter_slug(&self, chapter_url: String) -> Result<ChapterData, DomainError> {
&self,
chapter_url: String,
app_state: Arc<AppState>,
) -> Result<ChapterResponse, AppError> {
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("komik:chapter:{}", chapter_url); let cache_key = format!("komik:chapter:{}", chapter_url);
cache self.cache()
.get_or_set(&cache_key, CHAPTER_CACHE_TTL, || async { .get_or_set(&cache_key, CHAPTER_CACHE_TTL, || async {
let url = self.repository.chapter_url(&chapter_url); let url = self.repository.chapter_url(&chapter_url);
let html = self let html = self
@@ -204,87 +205,60 @@ impl KomikService {
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
data.images = cache_image_urls_batch_lazy( data.images = cache_image_urls_batch_lazy(
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
data.images, data.images,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(ChapterResponse { Ok(data)
message: "Ok".to_string(),
data,
})
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn manga_slug( pub async fn manga_slug(
&self, &self,
page_slug: String, page_slug: String,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<GenreKomikResponse, AppError> {
let page = page_slug let page = page_slug
.parse::<u32>() .parse::<u32>()
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; .map_err(|_| DomainError::Validation("Invalid page number".to_string()))?;
self.list_by_url( self.list_by_url("manga", page, self.repository.manga_list_url(page))
"manga",
page,
self.repository.manga_list_url(page),
app_state,
)
.await .await
} }
pub async fn manhua_slug( pub async fn manhua_slug(
&self, &self,
page_slug: String, page_slug: String,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<GenreKomikResponse, AppError> {
let page = page_slug let page = page_slug
.parse::<u32>() .parse::<u32>()
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; .map_err(|_| DomainError::Validation("Invalid page number".to_string()))?;
self.list_by_url( self.list_by_url("manhua", page, self.repository.manhua_list_url(page))
"manhua",
page,
self.repository.manhua_list_url(page),
app_state,
)
.await .await
} }
pub async fn manhwa_slug( pub async fn manhwa_slug(
&self, &self,
page_slug: String, page_slug: String,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<GenreKomikResponse, AppError> {
let page = page_slug let page = page_slug
.parse::<u32>() .parse::<u32>()
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; .map_err(|_| DomainError::Validation("Invalid page number".to_string()))?;
self.list_by_url( self.list_by_url("manhwa", page, self.repository.manhwa_list_url(page))
"manhwa",
page,
self.repository.manhwa_list_url(page),
app_state,
)
.await .await
} }
pub async fn popular_slug( pub async fn popular_slug(
&self, &self,
page_slug: String, page_slug: String,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<GenreKomikResponse, AppError> {
let page = page_slug let page = page_slug
.parse::<u32>() .parse::<u32>()
.map_err(|_| AppError::ScraperError("Invalid page number".to_string()))?; .map_err(|_| DomainError::Validation("Invalid page number".to_string()))?;
self.list_by_url( self.list_by_url("popular", page, self.repository.popular_list_url(page))
"popular",
page,
self.repository.popular_list_url(page),
app_state,
)
.await .await
} }
@@ -293,12 +267,10 @@ impl KomikService {
list_name: &str, list_name: &str,
page: u32, page: u32,
url: String, url: String,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<GenreKomikResponse, AppError> {
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("komik:list:{}:{}:v2", list_name, page); let cache_key = format!("komik:list:{}:{}:v2", list_name, page);
cache self.cache()
.get_or_set(&cache_key, GENRE_CACHE_TTL, || async { .get_or_set(&cache_key, GENRE_CACHE_TTL, || async {
let html = self let html = self
.repository .repository
@@ -318,33 +290,26 @@ impl KomikService {
apply_cached_posters( apply_cached_posters(
&mut komik_list, &mut komik_list,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(GenreKomikResponse { Ok((komik_list, pagination))
status: "Ok".to_string(),
genre: list_name.to_string(),
data: komik_list,
pagination,
})
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn search_slug( pub async fn search_slug(
&self, &self,
query: String, query: String,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<SearchKomikResponse, AppError> { let page = 1u32;
let page = 1;
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("komik:search:{}:{}", query, page); let cache_key = format!("komik:search:{}:{}", query, page);
cache self.cache()
.get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async {
let url = self.repository.search_url(&query, page); let url = self.repository.search_url(&query, page);
let html = self let html = self
@@ -361,32 +326,26 @@ impl KomikService {
apply_cached_posters( apply_cached_posters(
&mut komik_list, &mut komik_list,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(SearchKomikResponse { Ok((komik_list, pagination))
status: "Ok".to_string(),
data: komik_list,
pagination,
})
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
pub async fn search_slug_page( pub async fn search_slug_page(
&self, &self,
query: String, query: String,
page: u32, page: u32,
app_state: Arc<AppState>, ) -> Result<(Vec<KomikItem>, Pagination), DomainError> {
) -> Result<SearchKomikResponse, AppError> {
let cache = Cache::new(&app_state.redis_pool);
let cache_key = format!("komik:search:{}:{}", query, page); let cache_key = format!("komik:search:{}:{}", query, page);
cache self.cache()
.get_or_set(&cache_key, SEARCH_CACHE_TTL, || async { .get_or_set(&cache_key, SEARCH_CACHE_TTL, || async {
let url = self.repository.search_url(&query, page); let url = self.repository.search_url(&query, page);
let html = self let html = self
@@ -403,19 +362,15 @@ impl KomikService {
apply_cached_posters( apply_cached_posters(
&mut komik_list, &mut komik_list,
app_state.db.clone(), self.db.clone(),
&app_state.redis_pool, &self.redis_pool,
Some(app_state.image_processing_semaphore.clone()), self.semaphore.clone(),
) )
.await; .await;
Ok(SearchKomikResponse { Ok((komik_list, pagination))
status: "Ok".to_string(),
data: komik_list,
pagination,
})
}) })
.await .await
.map_err(AppError::ScraperError) .map_err(|e| DomainError::Scraping(ScrapingError::Http(e)))
} }
} }
+4
View File
@@ -0,0 +1,4 @@
pub mod anime;
pub mod anime2;
pub mod komik;
pub mod proxy;
+1
View File
@@ -0,0 +1 @@
pub mod use_cases;
+236
View File
@@ -0,0 +1,236 @@
//! Proxy application use cases.
//!
//! Provides proxy fetch, image caching, and audit/repair operations.
//!
//! TODO: Move result types to `crate::presentation::dto::proxy`.
//! TODO: Add event bus integration for ImageRepaired events.
//! TODO: Replace `reqwest::Client::new()` with shared HTTP client from infrastructure.
use std::sync::Arc;
use axum::http::StatusCode;
use axum::response::Response;
use serde::Serialize;
use tracing::{error, info, warn};
use utoipa::ToSchema;
use crate::domain::error::*;
use crate::domain::repository::ImageCacheRepository;
use crate::infrastructure::repository::{ProxyRepository, SeaOrmImageCacheRepository};
use crate::infrastructure::services::images::cache::ImageCache;
// ============================================================================
// Result types — TEMPORARY: move to presentation layer
// ============================================================================
/// Result of caching an image URL.
/// TODO: Move to presentation::dto::proxy
#[derive(Debug, Serialize, ToSchema)]
pub struct ImageCacheResult {
pub success: bool,
pub original_url: String,
pub cdn_url: String,
pub from_cache: bool,
pub pending: Option<bool>,
}
/// Result of auditing/repairing a cached image URL.
/// TODO: Move to presentation::dto::proxy
#[derive(Debug, Serialize, ToSchema)]
pub struct AuditImageCacheResult {
pub success: bool,
pub original_url: String,
pub cdn_url: Option<String>,
pub was_accessible: bool,
pub re_uploaded: bool,
pub message: String,
}
// ============================================================================
// Use case struct
// ============================================================================
pub struct ProxyUseCases {
repository: ProxyRepository,
image_cache_repo: Arc<dyn ImageCacheRepository>,
}
impl ProxyUseCases {
pub fn new(
repository: ProxyRepository,
image_cache_repo: Arc<SeaOrmImageCacheRepository>,
) -> Self {
Self {
repository,
image_cache_repo,
}
}
fn build_image_cache(&self) -> ImageCache {
ImageCache::new(self.image_cache_repo.clone())
}
pub async fn fetch_with_proxy_only(&self, url: String) -> Result<Response, DomainError> {
let fetch_result = self
.repository
.fetch_with_proxy_url(&url)
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e.to_string())))?;
let mut builder = Response::builder().status(StatusCode::OK);
if let Some(content_type) = fetch_result.content_type {
builder = builder.header("Content-Type", content_type);
}
builder
.body(fetch_result.data.into())
.map_err(|e| DomainError::Repository(RepositoryError::Network(e.to_string())))
}
pub async fn image_cache(
&self,
url: String,
lazy: bool,
) -> Result<ImageCacheResult, DomainError> {
let cache = self.build_image_cache();
if let Some(cdn_url) = cache.get_cdn_url(&url).await {
return Ok(ImageCacheResult {
success: true,
original_url: url,
cdn_url,
from_cache: true,
pending: None,
});
}
if lazy {
let repo = self.image_cache_repo.clone();
let url_clone = url.clone();
tokio::spawn(async move {
let cache = ImageCache::new(repo);
match cache.get_or_cache(&url_clone).await {
Ok(cdn) => info!("[LazyCache] Cached {} -> {}", url_clone, cdn),
Err(e) => warn!("[LazyCache] Failed {}: {}", url_clone, e),
}
});
return Ok(ImageCacheResult {
success: true,
original_url: url.clone(),
cdn_url: url,
from_cache: false,
pending: Some(true),
});
}
match cache.get_or_cache(&url).await {
Ok(cdn_url) => Ok(ImageCacheResult {
success: true,
original_url: url,
cdn_url,
from_cache: false,
pending: None,
}),
Err(e) => {
error!("ImageCache error: {}", e);
Ok(ImageCacheResult {
success: false,
original_url: url.clone(),
cdn_url: url,
from_cache: false,
pending: None,
})
}
}
}
pub async fn audit_image_cache(
&self,
url: String,
) -> Result<AuditImageCacheResult, DomainError> {
let cache = self.build_image_cache();
let mut cdn_opt = cache.get_cdn_url(&url).await;
let mut original = url.clone();
if cdn_opt.is_none() {
if let Some(orig) = cache.find_original_from_cdn(&url).await {
info!("SmartAudit: {} recognized as CDN, original {}", url, orig);
original = orig;
cdn_opt = Some(url.clone());
}
}
if let Some(cdn_url) = cdn_opt {
let client = reqwest::Client::new();
let mut accessible = false;
match client.get(&cdn_url).send().await {
Ok(resp) if resp.status().is_success() => {
if let Ok(bytes) = resp.bytes().await {
if infer::get(&bytes)
.map(|k| k.mime_type().starts_with("image/"))
.unwrap_or(false)
{
accessible = true;
} else {
warn!("CDN {} returned non-image content", cdn_url);
}
}
}
Ok(resp) => warn!("CDN {} status {}", cdn_url, resp.status()),
Err(e) => warn!("CDN {} fetch error {}", cdn_url, e),
}
if accessible {
return Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(cdn_url),
was_accessible: true,
re_uploaded: false,
message: "CDN URL is accessible and the image is valid".to_string(),
});
}
info!("CDN {} inaccessible, purging and reuploading", cdn_url);
let _ = cache.invalidate(&original).await;
match cache.get_or_cache(&original).await {
Ok(new_cdn) => Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(new_cdn),
was_accessible: false,
re_uploaded: true,
message: "CDN URL was inaccessible, re-uploaded".to_string(),
}),
Err(e) => Ok(AuditImageCacheResult {
success: false,
original_url: original,
cdn_url: None,
was_accessible: false,
re_uploaded: false,
message: format!("Re-upload failed: {}", e),
}),
}
} else {
match cache.get_or_cache(&original).await {
Ok(new_cdn) => Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(new_cdn),
was_accessible: false,
re_uploaded: true,
message: "Cached newly".to_string(),
}),
Err(e) => Ok(AuditImageCacheResult {
success: false,
original_url: original,
cdn_url: None,
was_accessible: false,
re_uploaded: false,
message: format!("Cache failed: {}", e),
}),
}
}
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
use scraper::Selector; use scraper::Selector;
use scraper_service::shared::utils::parse_html; use scraper_service::infrastructure::scraping::parsing_utils::parse_html;
/// Capture html5ever tree_builder warning evidence by parsing problematic HTML. /// Capture html5ever tree_builder warning evidence by parsing problematic HTML.
/// ///
/// Build with: cargo build --bin capture_warning /// Build with: cargo build --bin capture_warning
+1 -1
View File
@@ -6,7 +6,7 @@
/// Uses shared fixture: src/bin/test_fixtures/foster_parenting_minimal.html /// Uses shared fixture: src/bin/test_fixtures/foster_parenting_minimal.html
/// Uses shared parser: src/helpers::parse_html() /// Uses shared parser: src/helpers::parse_html()
use scraper::Selector; use scraper::Selector;
use scraper_service::shared::utils::parse_html; use scraper_service::infrastructure::scraping::parsing_utils::parse_html;
use std::fs; use std::fs;
fn main() { fn main() {
@@ -55,7 +55,7 @@ fn generate_list_handler(resource: &str, model: &str) -> String {
use axum::{{Extension, Json, response::IntoResponse, Router}}; use axum::{{Extension, Json, response::IntoResponse, Router}};
use sea_orm::{{DatabaseConnection, EntityTrait}}; use sea_orm::{{DatabaseConnection, EntityTrait}};
use std::sync::Arc; use std::sync::Arc;
use crate::shared::state::AppState; use crate::presentation::state::AppState;
use crate::entities::{model_low}::{{Entity as {model}, Model}}; use crate::entities::{model_low}::{{Entity as {model}, Model}};
pub async fn list( pub async fn list(
@@ -88,7 +88,7 @@ fn generate_show_handler(resource: &str, model: &str) -> String {
use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}}; use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}};
use sea_orm::{{DatabaseConnection, EntityTrait}}; use sea_orm::{{DatabaseConnection, EntityTrait}};
use std::sync::Arc; use std::sync::Arc;
use crate::shared::state::AppState; use crate::presentation::state::AppState;
use crate::entities::{model_low}::{{Entity as {model}, Model}}; use crate::entities::{model_low}::{{Entity as {model}, Model}};
pub async fn show( pub async fn show(
@@ -125,7 +125,7 @@ use axum::{{Extension, Json, response::IntoResponse, Router}};
use sea_orm::{{ActiveModelTrait, DatabaseConnection, Set}}; use sea_orm::{{ActiveModelTrait, DatabaseConnection, Set}};
use serde::{{Deserialize, Serialize}}; use serde::{{Deserialize, Serialize}};
use std::sync::Arc; use std::sync::Arc;
use crate::shared::state::AppState; use crate::presentation::state::AppState;
use crate::entities::{model_low}::{{ActiveModel, Model}}; use crate::entities::{model_low}::{{ActiveModel, Model}};
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
@@ -172,7 +172,7 @@ use axum::{{Extension, Json, extract::Path, response::IntoResponse, Router}};
use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, Set}}; use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, Set}};
use serde::{{Deserialize, Serialize}}; use serde::{{Deserialize, Serialize}};
use std::sync::Arc; use std::sync::Arc;
use crate::shared::state::AppState; use crate::presentation::state::AppState;
use crate::entities::{model_low}::{{ActiveModel, Entity as {model}, Model}}; use crate::entities::{model_low}::{{ActiveModel, Entity as {model}, Model}};
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
@@ -228,7 +228,7 @@ fn generate_delete_handler(resource: &str, model: &str) -> String {
use axum::{{Extension, extract::Path, response::IntoResponse, Router}}; use axum::{{Extension, extract::Path, response::IntoResponse, Router}};
use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, IntoActiveModel}}; use sea_orm::{{ActiveModelTrait, DatabaseConnection, EntityTrait, IntoActiveModel}};
use std::sync::Arc; use std::sync::Arc;
use crate::shared::state::AppState; use crate::presentation::state::AppState;
use crate::entities::{model_low}::{{Entity as {model}}}; use crate::entities::{model_low}::{{Entity as {model}}};
pub async fn destroy( pub async fn destroy(
@@ -270,7 +270,7 @@ fn generate_basic_controller(api_dir: &Path, resource: &str) {
use axum::Router; use axum::Router;
use std::sync::Arc; use std::sync::Arc;
use crate::shared::state::AppState; use crate::presentation::state::AppState;
pub async fn index() -> &'static str {{ pub async fn index() -> &'static str {{
"{resource} endpoint" "{resource} endpoint"
+19 -10
View File
@@ -1,3 +1,5 @@
pub mod setup;
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::Arc; use std::sync::Arc;
use tokio::net::TcpListener; use tokio::net::TcpListener;
@@ -6,9 +8,9 @@ use axum::Router;
use sea_orm::Database; use sea_orm::Database;
use tracing_subscriber::EnvFilter; use tracing_subscriber::EnvFilter;
use crate::shared::config::CONFIG; use crate::config::CONFIG;
use crate::shared::database::get_redis_conn; use crate::infrastructure::cache::redis_pool::get_redis_conn;
use crate::shared::state::AppState; use crate::presentation::state::AppState;
pub struct Application { pub struct Application {
pub port: u16, pub port: u16,
@@ -27,7 +29,7 @@ impl Application {
tracing_subscriber::fmt().with_env_filter(env_filter).init(); tracing_subscriber::fmt().with_env_filter(env_filter).init();
// Initialize OpenTelemetry metrics // Initialize OpenTelemetry metrics
crate::shared::observability::metrics::init_otel_metrics(); crate::observability::metrics::init_otel_metrics();
tracing::info!("🚀 Scraper starting up..."); tracing::info!("🚀 Scraper starting up...");
tracing::info!(" Environment: {}", CONFIG.environment); tracing::info!(" Environment: {}", CONFIG.environment);
@@ -46,8 +48,8 @@ impl Application {
// Browser Pool // Browser Pool
tracing::info!("Initializing browser pool..."); tracing::info!("Initializing browser pool...");
let browser_config = crate::shared::browser::BrowserPoolConfig::default(); let browser_config = crate::infrastructure::browser::BrowserPoolConfig::default();
match crate::shared::browser::pool::init_browser_pool(browser_config).await { match crate::infrastructure::browser::pool::init_browser_pool(browser_config).await {
Ok(_) => tracing::info!("✓ Browser pool initialized"), Ok(_) => tracing::info!("✓ Browser pool initialized"),
Err(e) => tracing::error!("⚠️ Failed to initialize browser pool: {}", e), Err(e) => tracing::error!("⚠️ Failed to initialize browser pool: {}", e),
} }
@@ -76,7 +78,7 @@ impl Application {
tracing::info!("✓ SeaORM database connection established"); tracing::info!("✓ SeaORM database connection established");
// Schema & Seeding // Schema & Seeding
if let Err(e) = crate::shared::database::setup::init(&db).await { if let Err(e) = crate::bootstrap::setup::init(&db).await {
tracing::error!("Failed to init DB schema: {}", e); tracing::error!("Failed to init DB schema: {}", e);
} }
@@ -85,19 +87,26 @@ impl Application {
let image_processing_semaphore = Arc::new(tokio::sync::Semaphore::new( let image_processing_semaphore = Arc::new(tokio::sync::Semaphore::new(
CONFIG.image_processing_concurrency, CONFIG.image_processing_concurrency,
)); ));
let event_bus = Arc::new(crate::shared::events::bus::EventBus::new()); let event_bus = Arc::new(crate::events::bus::EventBus::new());
let redis_pool = crate::shared::database::redis_pool() let redis_pool = crate::infrastructure::cache::redis_pool::redis_pool()
.map_err(|e| anyhow::anyhow!("Failed to init Redis pool: {}", e))?; .map_err(|e| anyhow::anyhow!("Failed to init Redis pool: {}", e))?;
use crate::infrastructure::repository::SeaOrmImageCacheRepository;
let image_cache_repo = Arc::new(SeaOrmImageCacheRepository::new(
db_arc.clone(),
redis_pool.clone(),
));
let app_state = Arc::new(AppState { let app_state = Arc::new(AppState {
redis_pool, redis_pool,
db: db_arc.clone(), db: db_arc.clone(),
image_processing_semaphore, image_processing_semaphore,
event_bus: event_bus.clone(), event_bus: event_bus.clone(),
image_cache_repo,
}); });
let app = crate::app::build_router(app_state, db_arc.clone()).await?; let app = crate::presentation::router::build_router(app_state.clone())?;
// Listener // Listener
let port = CONFIG.server_port; let port = CONFIG.server_port;
@@ -1,7 +1,10 @@
use crate::shared::database::persistence::entities::image_cache; //! Database schema initialization.
use sea_orm::{ConnectionTrait, DatabaseConnection, Schema, Statement}; use sea_orm::{ConnectionTrait, DatabaseConnection, Schema, Statement};
use tracing::info; use tracing::info;
use crate::infrastructure::persistence::entities::image_cache;
pub async fn init(db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> { pub async fn init(db: &DatabaseConnection) -> Result<(), sea_orm::DbErr> {
info!("🚀 Initializing database schema..."); info!("🚀 Initializing database schema...");
let backend = db.get_database_backend(); let backend = db.get_database_backend();
@@ -6,9 +6,9 @@
//! - Supports hierarchical configuration (default -> environment-specific) //! - Supports hierarchical configuration (default -> environment-specific)
use config::{Config, ConfigError, Environment, File}; use config::{Config, ConfigError, Environment, File};
use once_cell::sync::Lazy;
use serde::Deserialize; use serde::Deserialize;
use std::env; use std::env;
use std::sync::LazyLock;
/// Application configuration loaded at startup. /// Application configuration loaded at startup.
/// All fields are required unless marked as `Option<T>`. /// All fields are required unless marked as `Option<T>`.
@@ -270,7 +270,7 @@ impl AppConfig {
/// Global configuration instance, loaded once at startup. /// Global configuration instance, loaded once at startup.
/// Panics if configuration is invalid - this is intentional for fail-fast behavior. /// Panics if configuration is invalid - this is intentional for fail-fast behavior.
pub static CONFIG: Lazy<AppConfig> = Lazy::new(|| { pub static CONFIG: LazyLock<AppConfig> = LazyLock::new(|| {
AppConfig::load().unwrap_or_else(|e| { AppConfig::load().unwrap_or_else(|e| {
eprintln!("❌ Failed to load configuration: {}", e); eprintln!("❌ Failed to load configuration: {}", e);
eprintln!(" Make sure all required environment variables are set:"); eprintln!(" Make sure all required environment variables are set:");
@@ -283,7 +283,7 @@ pub static CONFIG: Lazy<AppConfig> = Lazy::new(|| {
/// Global MinIO configuration, loaded from environment variables. /// Global MinIO configuration, loaded from environment variables.
/// Returns None if required MINIO_* variables are not set. /// Returns None if required MINIO_* variables are not set.
pub static MINIO_CONFIG: Lazy<Option<MinioConfig>> = Lazy::new(|| { pub static MINIO_CONFIG: LazyLock<Option<MinioConfig>> = LazyLock::new(|| {
let _ = dotenvy::dotenv(); let _ = dotenvy::dotenv();
MinioConfig::from_env() MinioConfig::from_env()
}); });
+404
View File
@@ -0,0 +1,404 @@
//! Domain entities for anime data.
//!
//! Pure domain structs with no framework dependencies beyond serde + utoipa.
//! These represent the scraped anime data model regardless of source site.
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
// ============================================================================
// PAGINATION
// ============================================================================
/// Common pagination structure shared across all endpoints
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Pagination {
pub current_page: u32,
pub last_visible_page: u32,
pub has_next_page: bool,
pub next_page: Option<u32>,
pub has_previous_page: bool,
pub previous_page: Option<u32>,
}
impl Pagination {
pub fn with_string_pages(
current_page: u32,
last_visible_page: u32,
has_next_page: bool,
next_page: Option<String>,
has_previous_page: bool,
previous_page: Option<String>,
) -> PaginationWithStringPages {
PaginationWithStringPages {
current_page,
last_visible_page,
has_next_page,
next_page,
has_previous_page,
previous_page,
}
}
}
/// Pagination variant with string-based page numbers (used in search endpoints)
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct PaginationWithStringPages {
pub current_page: u32,
pub last_visible_page: u32,
pub has_next_page: bool,
pub next_page: Option<String>,
pub has_previous_page: bool,
pub previous_page: Option<String>,
}
// ============================================================================
// OTakudesu (Anime Module) — Index Types
// ============================================================================
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct OngoingAnimeItem {
pub title: String,
pub slug: String,
pub poster: String,
pub current_episode: String,
pub anime_url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct CompleteAnimeItem {
pub title: String,
pub slug: String,
pub poster: String,
pub episode_count: String,
pub anime_url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct AnimeData {
pub ongoing_anime: Vec<OngoingAnimeItem>,
pub complete_anime: Vec<CompleteAnimeItem>,
}
// ============================================================================
// Genre Types
// ============================================================================
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Genre {
pub name: String,
pub slug: String,
pub url: String,
}
// ============================================================================
// Otakudesu Detail Types
// ============================================================================
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct DetailGenre {
pub name: String,
pub slug: String,
pub anime_url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct EpisodeList {
pub episode: String,
pub slug: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Recommendation {
pub title: String,
pub slug: String,
pub poster: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub status: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub r#type: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct AnimeDetailData {
pub title: String,
pub alternative_title: String,
pub poster: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub r#type: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub status: Option<String>,
pub release_date: String,
pub studio: String,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub genres: Vec<DetailGenre>,
pub synopsis: String,
pub episode_lists: Vec<EpisodeList>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub batch: Vec<EpisodeList>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub producers: Vec<String>,
pub recommendations: Vec<Recommendation>,
}
// ============================================================================
// Otakudesu List Page Types
// ============================================================================
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct CompleteAnimeListItem {
pub title: String,
pub slug: String,
pub poster: String,
pub episode_count: String,
pub anime_url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct OngoingAnimeListItem {
pub title: String,
pub slug: String,
pub poster: String,
pub score: String,
pub anime_url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct LatestAnimeItem {
pub title: String,
pub slug: String,
pub poster: String,
pub episode: String,
pub score: String,
pub anime_url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct SearchAnimeItem {
pub title: String,
pub slug: String,
pub poster: String,
pub episode: String,
pub anime_url: String,
pub genres: Vec<String>,
pub status: String,
pub rating: String,
pub description: String,
pub r#type: String,
pub season: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct GenreAnimeItem {
pub title: String,
pub slug: String,
pub poster: String,
pub episode: String,
pub score: String,
pub status: String,
pub anime_url: String,
}
// ============================================================================
// Otakudesu Full Episode Types
// ============================================================================
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct AnimeInfo {
pub slug: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct EpisodeInfo {
pub slug: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct DownloadLink {
pub server: String,
pub url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct AnimeFullData {
pub episode: String,
pub episode_number: String,
pub anime: AnimeInfo,
pub has_next_episode: bool,
pub next_episode: Option<EpisodeInfo>,
pub has_previous_episode: bool,
pub previous_episode: Option<EpisodeInfo>,
pub stream_url: String,
pub download_urls: std::collections::HashMap<String, Vec<DownloadLink>>,
pub image_url: String,
}
// ============================================================================
// ALQanime (Anime2 Module) — Index Types
// ============================================================================
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2Item {
pub title: String,
pub slug: String,
pub poster: String,
pub status: String,
pub r#type: String,
pub score: String,
pub anime_url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2Data {
pub ongoing_anime: Vec<Anime2Item>,
pub complete_anime: Vec<Anime2Item>,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2ItemDetail {
pub title: String,
pub slug: String,
pub poster: String,
pub poster2: String,
pub synopsis: String,
pub alternative_title: String,
pub r#type: String,
pub status: String,
pub score: String,
pub genres: Vec<DetailGenre>,
pub episodes: Vec<EpisodeList>,
pub recommendations: Vec<Recommendation>,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct OngoingAnimeItemWithScore {
pub title: String,
pub slug: String,
pub poster: String,
pub score: String,
pub anime_url: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct FilterAnimeItem {
pub title: String,
pub slug: String,
pub poster: String,
pub score: String,
pub status: String,
pub r#type: String,
pub anime_url: String,
}
// ============================================================================
// TRAITS
// ============================================================================
/// Trait for types that have a poster image URL
pub trait HasPoster {
fn poster(&self) -> &str;
fn set_poster(&mut self, url: String);
}
// ============================================================================
// TRAIT IMPLEMENTATIONS
// ============================================================================
impl HasPoster for OngoingAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for CompleteAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for CompleteAnimeListItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for OngoingAnimeListItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for LatestAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for SearchAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for GenreAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for Recommendation {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for Anime2Item {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for OngoingAnimeItemWithScore {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
impl HasPoster for FilterAnimeItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
+64
View File
@@ -0,0 +1,64 @@
//! Domain entities for komik (comic) data.
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use crate::domain::entity::anime::HasPoster;
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct KomikGenre {
pub name: String,
pub slug: String,
pub count: Option<u32>,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Chapter {
pub chapter: String,
pub date: String,
pub chapter_id: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct ChapterData {
pub title: String,
pub next_chapter_id: String,
pub prev_chapter_id: String,
pub list_chapter: String,
pub images: Vec<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct DetailData {
pub title: String,
pub poster: String,
pub description: String,
pub status: String,
pub r#type: String,
pub release_date: String,
pub author: String,
pub total_chapter: String,
pub updated_on: String,
pub genres: Vec<String>,
pub chapters: Vec<Chapter>,
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct KomikItem {
pub title: String,
pub slug: String,
pub poster: String,
pub chapter: String,
pub score: String,
pub r#type: String,
pub komik_url: String,
}
impl HasPoster for KomikItem {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
+2
View File
@@ -0,0 +1,2 @@
pub mod anime;
pub mod komik;
+55
View File
@@ -0,0 +1,55 @@
//! Domain-level error types.
//!
//! These are framework-agnostic errors that can be mapped to HTTP errors
//! at the presentation layer. Domain and application layers only use these.
use thiserror::Error;
/// Errors originating from repository operations (DB, HTTP, etc.)
#[derive(Error, Debug)]
pub enum RepositoryError {
#[error("Not found")]
NotFound,
#[error("Conflict: {0}")]
Conflict(String),
#[error("Database error: {0}")]
Database(String),
#[error("Network error: {0}")]
Network(String),
}
/// Errors originating from scraping/parsing operations
#[derive(Error, Debug)]
pub enum ScrapingError {
#[error("HTTP error: {0}")]
Http(String),
#[error("Parse error: {0}")]
Parse(String),
#[error("Empty response")]
EmptyResponse,
}
/// Generic domain error
#[derive(Error, Debug)]
pub enum DomainError {
#[error("Not found: {0}")]
NotFound(String),
#[error("Validation error: {0}")]
Validation(String),
#[error("Repository error: {0}")]
Repository(#[from] RepositoryError),
#[error("Scraping error: {0}")]
Scraping(#[from] ScrapingError),
}
impl From<String> for RepositoryError {
fn from(s: String) -> Self {
RepositoryError::Database(s)
}
}
impl From<&str> for RepositoryError {
fn from(s: &str) -> Self {
RepositoryError::Database(s.to_string())
}
}
+3
View File
@@ -0,0 +1,3 @@
pub mod entity;
pub mod error;
pub mod repository;
@@ -1,5 +1,8 @@
//! Repository trait for image cache (original → CDN mapping).
use async_trait::async_trait; use async_trait::async_trait;
/// Repository for managing cached image URL mappings.
#[async_trait] #[async_trait]
pub trait ImageCacheRepository: Send + Sync { pub trait ImageCacheRepository: Send + Sync {
async fn get_from_redis(&self, key: &str) -> Option<String>; async fn get_from_redis(&self, key: &str) -> Option<String>;
+5
View File
@@ -0,0 +1,5 @@
pub mod image_cache;
pub mod scraping;
pub use image_cache::ImageCacheRepository;
pub use scraping::ScrapingRepository;
+12
View File
@@ -0,0 +1,12 @@
//! Repository trait for scraping HTML from remote sources.
use async_trait::async_trait;
use crate::domain::error::ScrapingError;
/// Trait for repositories that fetch and scrape HTML content.
#[async_trait]
pub trait ScrapingRepository: Send + Sync {
/// Fetch raw HTML from a URL.
async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError>;
}
@@ -60,7 +60,7 @@ impl Default for BrowserPoolConfig {
let chrome_remote = std::env::var("CHROME_REMOTE_WS").ok(); let chrome_remote = std::env::var("CHROME_REMOTE_WS").ok();
let remote_websocket_url = if let Some(ext) = external { let remote_websocket_url = if let Some(ext) = external {
tracing::info!("🌐 Browser: using EXTERNAL_BROWSERLESS_WS"); tracing::info!(" Browser: using EXTERNAL_BROWSERLESS_WS");
ext ext
} else if let Some(ref cr) = chrome_remote { } else if let Some(ref cr) = chrome_remote {
if cr == "ws://browserless:3000" { if cr == "ws://browserless:3000" {
@@ -70,7 +70,7 @@ impl Default for BrowserPoolConfig {
); );
std::process::exit(1); std::process::exit(1);
} else { } else {
tracing::info!("🌐 Browser: using CHROME_REMOTE_WS"); tracing::info!(" Browser: using CHROME_REMOTE_WS");
cr.clone() cr.clone()
} }
} else { } else {
@@ -462,9 +462,9 @@ impl Drop for PooledTab {
} }
// Global browser pool instance // Global browser pool instance
use once_cell::sync::OnceCell; use std::sync::OnceLock;
static BROWSER_POOL: OnceCell<Arc<BrowserPool>> = OnceCell::new(); static BROWSER_POOL: OnceLock<Arc<BrowserPool>> = OnceLock::new();
/// Initialize the global browser pool. /// Initialize the global browser pool.
/// Call this once at application startup. /// Call this once at application startup.
+4
View File
@@ -0,0 +1,4 @@
pub mod redis;
pub mod redis_pool;
pub use redis::Cache;
@@ -1,13 +1,12 @@
//! Redis caching helpers. //! Redis caching helpers.
use crate::shared::utils::cache_ttl::CACHE_TTL_VERY_SHORT;
use deadpool_redis::redis::AsyncCommands; use deadpool_redis::redis::AsyncCommands;
use deadpool_redis::Pool; use deadpool_redis::Pool;
use serde::{de::DeserializeOwned, Serialize}; use serde::{de::DeserializeOwned, Serialize};
use tracing::{debug, error}; use tracing::{debug, error};
/// Default cache TTL in seconds (5 minutes). /// Default cache TTL in seconds (5 minutes).
pub const DEFAULT_CACHE_TTL: u64 = CACHE_TTL_VERY_SHORT; pub const DEFAULT_CACHE_TTL: u64 = 300;
/// Cache helper for Redis operations. /// Cache helper for Redis operations.
pub struct Cache<'a> { pub struct Cache<'a> {
@@ -15,12 +14,10 @@ pub struct Cache<'a> {
} }
impl<'a> Cache<'a> { impl<'a> Cache<'a> {
/// Create a new cache helper.
pub fn new(pool: &'a Pool) -> Self { pub fn new(pool: &'a Pool) -> Self {
Self { pool } Self { pool }
} }
/// Get a value from cache, deserializing JSON.
pub async fn get<T: DeserializeOwned>(&self, key: &str) -> Option<T> { pub async fn get<T: DeserializeOwned>(&self, key: &str) -> Option<T> {
let mut conn = match self.pool.get().await { let mut conn = match self.pool.get().await {
Ok(c) => c, Ok(c) => c,
@@ -41,8 +38,6 @@ impl<'a> Cache<'a> {
cached.and_then(|json| serde_json::from_str(&json).ok()) cached.and_then(|json| serde_json::from_str(&json).ok())
} }
/// Get multiple values from cache, deserializing JSON.
/// Returns a vector of Options, preserving order of keys.
pub async fn mget<T: DeserializeOwned>(&self, keys: &[String]) -> Vec<Option<T>> { pub async fn mget<T: DeserializeOwned>(&self, keys: &[String]) -> Vec<Option<T>> {
if keys.is_empty() { if keys.is_empty() {
return Vec::new(); return Vec::new();
@@ -56,7 +51,6 @@ impl<'a> Cache<'a> {
} }
}; };
// Use low-level cmd interface for MGET to ensure correct command usage
use deadpool_redis::redis::cmd; use deadpool_redis::redis::cmd;
let cached_values: Vec<Option<String>> = let cached_values: Vec<Option<String>> =
match cmd("MGET").arg(keys).query_async(&mut conn).await { match cmd("MGET").arg(keys).query_async(&mut conn).await {
@@ -73,12 +67,10 @@ impl<'a> Cache<'a> {
.collect() .collect()
} }
/// Set a value in cache with default TTL (5 minutes).
pub async fn set<T: Serialize>(&self, key: &str, value: &T) -> Result<(), String> { pub async fn set<T: Serialize>(&self, key: &str, value: &T) -> Result<(), String> {
self.set_with_ttl(key, value, DEFAULT_CACHE_TTL).await self.set_with_ttl(key, value, DEFAULT_CACHE_TTL).await
} }
/// Set a value in cache with custom TTL.
pub async fn set_with_ttl<T: Serialize>( pub async fn set_with_ttl<T: Serialize>(
&self, &self,
key: &str, key: &str,
@@ -86,18 +78,14 @@ impl<'a> Cache<'a> {
ttl_secs: u64, ttl_secs: u64,
) -> Result<(), String> { ) -> Result<(), String> {
let mut conn = self.pool.get().await.map_err(|e| e.to_string())?; let mut conn = self.pool.get().await.map_err(|e| e.to_string())?;
let json = serde_json::to_string(value).map_err(|e| e.to_string())?; let json = serde_json::to_string(value).map_err(|e| e.to_string())?;
conn.set_ex::<_, _, ()>(key, json, ttl_secs) conn.set_ex::<_, _, ()>(key, json, ttl_secs)
.await .await
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
debug!("Cache: set key {} with TTL {}s", key, ttl_secs); debug!("Cache: set key {} with TTL {}s", key, ttl_secs);
Ok(()) Ok(())
} }
/// Delete a key from cache.
pub async fn delete(&self, key: &str) -> Result<(), String> { pub async fn delete(&self, key: &str) -> Result<(), String> {
let mut conn = self.pool.get().await.map_err(|e| e.to_string())?; let mut conn = self.pool.get().await.map_err(|e| e.to_string())?;
conn.del::<_, ()>(key).await.map_err(|e| e.to_string())?; conn.del::<_, ()>(key).await.map_err(|e| e.to_string())?;
@@ -105,7 +93,6 @@ impl<'a> Cache<'a> {
Ok(()) Ok(())
} }
/// Check if key exists.
pub async fn exists(&self, key: &str) -> bool { pub async fn exists(&self, key: &str) -> bool {
let mut conn = match self.pool.get().await { let mut conn = match self.pool.get().await {
Ok(c) => c, Ok(c) => c,
@@ -126,20 +113,14 @@ impl<'a> Cache<'a> {
F: FnOnce() -> Fut, F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, String>>, Fut: std::future::Future<Output = Result<T, String>>,
{ {
// Try cache first
if let Some(cached) = self.get::<T>(key).await { if let Some(cached) = self.get::<T>(key).await {
debug!("Cache hit: {}", key); debug!("Cache hit: {}", key);
return Ok(cached); return Ok(cached);
} }
debug!("Cache miss: {}", key); debug!("Cache miss: {}", key);
// Compute the value
let value = compute().await?; let value = compute().await?;
// Store in cache
self.set_with_ttl(key, &value, ttl_secs).await?; self.set_with_ttl(key, &value, ttl_secs).await?;
Ok(value) Ok(value)
} }
} }
@@ -148,8 +129,3 @@ impl<'a> Cache<'a> {
pub fn cache_key(prefix: &str, id: &str) -> String { pub fn cache_key(prefix: &str, id: &str) -> String {
format!("{}:{}", prefix, id) format!("{}:{}", prefix, id)
} }
/// Create a cache key with multiple parts.
pub fn cache_key_multi(parts: &[&str]) -> String {
parts.join(":")
}
@@ -1,21 +1,19 @@
//! Redis connection utility with tracing for connection lifecycle and errors. //! Redis connection pool management.
//!
//! Uses the type-safe CONFIG for Redis connection parameters. use std::sync::LazyLock;
use crate::shared::config::CONFIG;
use crate::shared::errors::AppError;
use deadpool_redis::{Manager, Pool}; use deadpool_redis::{Manager, Pool};
use once_cell::sync::Lazy;
use tracing::{debug, error, info}; use tracing::{debug, error, info};
static REDIS_POOL_INIT: Lazy<Result<Pool, String>> = Lazy::new(|| { use crate::config::CONFIG;
static REDIS_POOL_INIT: LazyLock<Result<Pool, String>> = LazyLock::new(|| {
let redis_url = if !CONFIG.redis_url.is_empty() { let redis_url = if !CONFIG.redis_url.is_empty() {
CONFIG.redis_url.clone() CONFIG.redis_url.clone()
} else { } else {
let host = std::env::var("REDIS_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); let host = std::env::var("REDIS_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
let port = std::env::var("REDIS_PORT").unwrap_or_else(|_| "6379".to_string()); let port = std::env::var("REDIS_PORT").unwrap_or_else(|_| "6379".to_string());
let password = std::env::var("REDIS_PASSWORD").unwrap_or_default(); let password = std::env::var("REDIS_PASSWORD").unwrap_or_default();
if password.is_empty() { if password.is_empty() {
format!("redis://{}:{}", host, port) format!("redis://{}:{}", host, port)
} else { } else {
@@ -37,19 +35,16 @@ static REDIS_POOL_INIT: Lazy<Result<Pool, String>> = Lazy::new(|| {
}) })
}); });
/// Get the Redis connection pool (for internal use).
pub fn get_redis_pool() -> Result<&'static Pool, String> { pub fn get_redis_pool() -> Result<&'static Pool, String> {
REDIS_POOL_INIT.as_ref().map_err(|e| e.clone()) REDIS_POOL_INIT.as_ref().map_err(|e| e.clone())
} }
/// Get a cloned reference to the Redis pool.
pub fn redis_pool() -> Result<Pool, String> { pub fn redis_pool() -> Result<Pool, String> {
get_redis_pool().map(|p| (*p).clone()) get_redis_pool().cloned()
} }
/// Get an async connection from the pool with retry backoff. pub async fn get_redis_conn() -> Result<deadpool_redis::Connection, String> {
pub async fn get_redis_conn() -> Result<deadpool_redis::Connection, AppError> { let pool = get_redis_pool()?;
let pool = get_redis_pool().map_err(|e| AppError::Other(e))?;
let mut retries = 5; let mut retries = 5;
let mut wait = std::time::Duration::from_millis(100); let mut wait = std::time::Duration::from_millis(100);
@@ -62,7 +57,7 @@ pub async fn get_redis_conn() -> Result<deadpool_redis::Connection, AppError> {
Err(e) => { Err(e) => {
if retries <= 0 { if retries <= 0 {
error!("Failed to get Redis connection after retries: {:?}", e); error!("Failed to get Redis connection after retries: {:?}", e);
return Err(AppError::from(e)); return Err(format!("Redis connection failed: {}", e));
} }
debug!("Redis connection failed, retrying in {:?}: {:?}", wait, e); debug!("Redis connection failed, retrying in {:?}: {:?}", wait, e);
tokio::time::sleep(wait).await; tokio::time::sleep(wait).await;
+7
View File
@@ -0,0 +1,7 @@
pub mod browser;
pub mod cache;
pub mod persistence;
pub mod repository;
pub mod scraping;
pub mod services;
pub mod utils;
@@ -1,21 +1,18 @@
use crate::shared::database::traits::scraping_repository::ScrapingRepository; //! Alqanime (Anime2) scraping repository.
use crate::shared::errors::AppError;
use crate::shared::utils::web::proxy_fetch::fetch_with_proxy_only;
use async_trait::async_trait; use async_trait::async_trait;
use tracing::warn; use tracing::warn;
use crate::domain::error::ScrapingError;
use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::scraping::proxy_fetch::fetch_with_proxy_only;
const BASE_URL: &str = "https://alqanime.si"; const BASE_URL: &str = "https://alqanime.si";
const BASE_DETAIL_URL: &str = "https://alqanime.net"; const BASE_DETAIL_URL: &str = "https://alqanime.net";
pub struct Anime2Repository; pub struct AlqanimeRepository;
impl Default for Anime2Repository { impl AlqanimeRepository {
fn default() -> Self {
Self::new()
}
}
impl Anime2Repository {
pub fn new() -> Self { pub fn new() -> Self {
Self Self
} }
@@ -91,11 +88,13 @@ impl Anime2Repository {
} }
#[async_trait] #[async_trait]
impl ScrapingRepository for Anime2Repository { impl ScrapingRepository for AlqanimeRepository {
async fn fetch_html(&self, url: &str) -> Result<String, AppError> { async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
let response = fetch_with_proxy_only(url).await?; let response = fetch_with_proxy_only(url)
.await
.map_err(|e| ScrapingError::Http(format!("Alqanime fetch failed: {}", e)))?;
if response.data.trim().is_empty() { if response.data.trim().is_empty() {
warn!("Anime2 browserless fetch returned empty body for {}", url); warn!("Alqanime browserless fetch returned empty body for {}", url);
} }
Ok(response.data) Ok(response.data)
} }
@@ -1,12 +1,15 @@
use crate::shared::database::persistence::entities::image_cache; //! SeaORM-backed implementation of ImageCacheRepository.
use crate::shared::database::traits::image_cache::ImageCacheRepository;
use crate::shared::utils::Cache;
use async_trait::async_trait; use async_trait::async_trait;
use chrono::Utc; use chrono::Utc;
use deadpool_redis::Pool as RedisPool; use deadpool_redis::Pool as RedisPool;
use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, Set}; use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, Set};
use std::sync::Arc; use std::sync::Arc;
use crate::domain::repository::ImageCacheRepository;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::persistence::entities::image_cache;
pub struct SeaOrmImageCacheRepository { pub struct SeaOrmImageCacheRepository {
db: Arc<DatabaseConnection>, db: Arc<DatabaseConnection>,
redis: RedisPool, redis: RedisPool,
@@ -48,7 +51,6 @@ impl ImageCacheRepository for SeaOrmImageCacheRepository {
created_at: Set(Utc::now()), created_at: Set(Utc::now()),
expires_at: Set(None), expires_at: Set(None),
}; };
model model
.insert(self.db.as_ref()) .insert(self.db.as_ref())
.await .await
@@ -1,16 +1,13 @@
use crate::shared::database::traits::scraping_repository::ScrapingRepository; //! Komik site scraping repository.
use crate::shared::errors::AppError;
use crate::shared::utils::fetch_html_with_retry;
use crate::shared::utils::web::scraping_urls::{get_komik_api_url, get_komik_url};
use async_trait::async_trait; use async_trait::async_trait;
pub struct KomikRepository; use crate::domain::error::ScrapingError;
use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::scraping::html_fetcher::fetch_html_with_retry;
use crate::infrastructure::scraping::scraping_urls::{get_komik_api_url, get_komik_url};
impl Default for KomikRepository { pub struct KomikRepository;
fn default() -> Self {
Self::new()
}
}
impl KomikRepository { impl KomikRepository {
pub fn new() -> Self { pub fn new() -> Self {
@@ -72,7 +69,7 @@ impl KomikRepository {
#[async_trait] #[async_trait]
impl ScrapingRepository for KomikRepository { impl ScrapingRepository for KomikRepository {
async fn fetch_html(&self, url: &str) -> Result<String, AppError> { async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
fetch_html_with_retry(url).await fetch_html_with_retry(url).await
} }
} }
+12
View File
@@ -0,0 +1,12 @@
pub mod alqanime;
pub mod image_cache_seaorm;
pub mod komik;
pub mod otakudesu;
pub mod parsers;
pub mod proxy;
pub use alqanime::AlqanimeRepository;
pub use image_cache_seaorm::SeaOrmImageCacheRepository;
pub use komik::KomikRepository;
pub use otakudesu::OtakudesuRepository;
pub use proxy::ProxyRepository;
@@ -1,38 +1,31 @@
use crate::modules::anime::parser; //! Otakudesu anime scraping repository.
use crate::modules::anime::types::*;
use crate::shared::database::traits::scraping_repository::ScrapingRepository;
use crate::shared::errors::AppError;
use crate::shared::utils::web::proxy_fetch::fetch_with_proxy;
use crate::shared::utils::web::scraping_urls::{get_otakudesu_url, OTAKUDESU_BASE_URL};
use crate::shared::utils::{default_backoff, fetch_html_with_retry, transient};
use async_trait::async_trait; use async_trait::async_trait;
use backoff::future::retry; use backoff::future::retry;
use tracing::{info, warn}; use tracing::{info, warn};
pub struct AnimeRepository; use crate::domain::entity::anime::{
AnimeData, AnimeDetailData, AnimeFullData, CompleteAnimeListItem, Genre, GenreAnimeItem,
LatestAnimeItem, OngoingAnimeListItem, Pagination, SearchAnimeItem,
};
use crate::domain::error::ScrapingError;
use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::repository::parsers::otakudesu_parser;
use crate::infrastructure::scraping::html_fetcher::fetch_html_with_retry;
use crate::infrastructure::scraping::proxy_fetch::fetch_with_proxy;
use crate::infrastructure::scraping::retry::{default_backoff, transient};
impl Default for AnimeRepository { const OTAKUDESU_BASE_URL: &str = "https://otakudesu.cloud";
fn default() -> Self {
Self::new()
}
}
impl AnimeRepository { pub struct OtakudesuRepository;
impl OtakudesuRepository {
pub fn new() -> Self { pub fn new() -> Self {
Self Self
} }
}
#[async_trait] fn base_url(&self) -> String {
impl ScrapingRepository for AnimeRepository { "https://otakudesu.cloud".to_string()
async fn fetch_html(&self, url: &str) -> Result<String, AppError> {
fetch_html_with_retry(url).await
}
}
impl AnimeRepository {
pub fn base_url(&self) -> String {
get_otakudesu_url()
} }
pub fn index_urls(&self) -> (String, String) { pub fn index_urls(&self) -> (String, String) {
@@ -70,24 +63,36 @@ impl AnimeRepository {
pub fn full_episode_url(&self, slug: &str) -> String { pub fn full_episode_url(&self, slug: &str) -> String {
format!("{}/episode/{}", OTAKUDESU_BASE_URL, slug) format!("{}/episode/{}", OTAKUDESU_BASE_URL, slug)
} }
}
pub async fn fetch_anime_index(&self) -> Result<AnimeData, AppError> { #[async_trait]
impl ScrapingRepository for OtakudesuRepository {
async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
fetch_html_with_retry(url).await
}
}
impl OtakudesuRepository {
pub async fn fetch_anime_index(&self) -> Result<AnimeData, ScrapingError> {
let (ongoing_url, complete_url) = self.index_urls(); let (ongoing_url, complete_url) = self.index_urls();
let (ongoing_html, complete_html) = tokio::join!( let (ongoing_html, complete_html) = tokio::join!(
self.fetch_html(&ongoing_url), self.fetch_html(&ongoing_url),
self.fetch_html(&complete_url) self.fetch_html(&complete_url)
); );
let ongoing_html = ongoing_html?; let ongoing_html = ongoing_html?;
let complete_html = complete_html?; let complete_html = complete_html?;
let ongoing_anime = let ongoing_anime = tokio::task::spawn_blocking(move || {
tokio::task::spawn_blocking(move || parser::parse_ongoing_anime(&ongoing_html)) otakudesu_parser::parse_ongoing_anime(&ongoing_html)
.await??; })
let complete_anime = .await
tokio::task::spawn_blocking(move || parser::parse_complete_anime(&complete_html)) .map_err(|e| ScrapingError::Parse(e.to_string()))??;
.await??;
let complete_anime = tokio::task::spawn_blocking(move || {
otakudesu_parser::parse_complete_anime(&complete_html)
})
.await
.map_err(|e| ScrapingError::Parse(e.to_string()))??;
Ok(AnimeData { Ok(AnimeData {
ongoing_anime, ongoing_anime,
@@ -95,88 +100,103 @@ impl AnimeRepository {
}) })
} }
pub async fn fetch_genres(&self) -> Result<Vec<Genre>, AppError> { pub async fn fetch_genres(&self) -> Result<Vec<Genre>, ScrapingError> {
let html = self.fetch_html(&self.genres_url()).await?; let html = self.fetch_html(&self.genres_url()).await?;
tokio::task::spawn_blocking(move || parser::parse_genres(&html)).await? tokio::task::spawn_blocking(move || otakudesu_parser::parse_genres(&html))
.await
.map_err(|e| ScrapingError::Parse(e.to_string()))?
} }
pub async fn fetch_anime_detail(&self, slug: &str) -> Result<AnimeDetailData, AppError> { pub async fn fetch_anime_detail(&self, slug: &str) -> Result<AnimeDetailData, ScrapingError> {
let url = self.detail_url(slug); let url = self.detail_url(slug);
let html = self let html = self.fetch_with_proxy_retry(&url).await?;
.fetch_with_proxy_retry(&url) tokio::task::spawn_blocking(move || otakudesu_parser::parse_anime_detail_document(&html))
.await .await
.map_err(|e| AppError::ScraperError(e.to_string()))?; .map_err(|e| ScrapingError::Parse(e.to_string()))?
tokio::task::spawn_blocking(move || parser::parse_anime_detail_document(&html)).await?
} }
pub async fn fetch_complete_anime_page( pub async fn fetch_complete_anime_page(
&self, &self,
slug: &str, slug: &str,
) -> Result<(Vec<CompleteAnimeListItem>, Pagination), AppError> { ) -> Result<(Vec<CompleteAnimeListItem>, Pagination), ScrapingError> {
let url = self.page_url("complete-anime", slug); let url = self.page_url("complete-anime", slug);
let html = self.fetch_html(&url).await?; let html = self.fetch_html(&url).await?;
let slug_owned = slug.to_string(); let slug_owned = slug.to_string();
tokio::task::spawn_blocking(move || parser::parse_anime_page(&html, &slug_owned)).await? tokio::task::spawn_blocking(move || otakudesu_parser::parse_anime_page(&html, &slug_owned))
.await
.map_err(|e| ScrapingError::Parse(e.to_string()))?
} }
pub async fn fetch_ongoing_anime_page( pub async fn fetch_ongoing_anime_page(
&self, &self,
slug: &str, slug: &str,
) -> Result<(Vec<OngoingAnimeListItem>, Pagination), AppError> { ) -> Result<(Vec<OngoingAnimeListItem>, Pagination), ScrapingError> {
let url = self.page_url("ongoing-anime", slug); let url = self.page_url("ongoing-anime", slug);
let html = self.fetch_html(&url).await?; let html = self.fetch_html(&url).await?;
let slug_owned = slug.to_string(); let slug_owned = slug.to_string();
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
parser::parse_ongoing_anime_document(&html, &slug_owned) otakudesu_parser::parse_ongoing_anime_document(&html, &slug_owned)
}) })
.await? .await
.map_err(|e| ScrapingError::Parse(e.to_string()))?
} }
pub async fn fetch_latest_anime_page( pub async fn fetch_latest_anime_page(
&self, &self,
slug: &str, slug: &str,
) -> Result<(Vec<LatestAnimeItem>, Pagination), AppError> { ) -> Result<(Vec<LatestAnimeItem>, Pagination), ScrapingError> {
let url = self.page_url("latest-anime", slug); let url = self.page_url("latest-anime", slug);
let html = self.fetch_html(&url).await?; let html = self.fetch_html(&url).await?;
let slug_owned = slug.to_string(); let slug_owned = slug.to_string();
tokio::task::spawn_blocking(move || parser::parse_latest_anime_document(&html, &slug_owned)) tokio::task::spawn_blocking(move || {
.await? otakudesu_parser::parse_latest_anime_document(&html, &slug_owned)
})
.await
.map_err(|e| ScrapingError::Parse(e.to_string()))?
} }
pub async fn fetch_search_anime_page( pub async fn fetch_search_anime_page(
&self, &self,
slug: &str, slug: &str,
page: &str, page: &str,
) -> Result<(Vec<SearchAnimeItem>, Pagination), AppError> { ) -> Result<(Vec<SearchAnimeItem>, Pagination), ScrapingError> {
let url = self.search_url(slug, page); let url = self.search_url(slug, page);
let html = self.fetch_html(&url).await?; let html = self.fetch_html(&url).await?;
let page_owned = page.to_string(); let page_owned = page.to_string();
tokio::task::spawn_blocking(move || parser::parse_search_anime_document(&html, &page_owned)) tokio::task::spawn_blocking(move || {
.await? otakudesu_parser::parse_search_anime_document(&html, &page_owned)
})
.await
.map_err(|e| ScrapingError::Parse(e.to_string()))?
} }
pub async fn fetch_genre_anime_page( pub async fn fetch_genre_anime_page(
&self, &self,
genre_slug: &str, genre_slug: &str,
page: &str, page: &str,
) -> Result<(Vec<GenreAnimeItem>, Pagination), AppError> { ) -> Result<(Vec<GenreAnimeItem>, Pagination), ScrapingError> {
let url = self.genre_page_url(genre_slug, page); let url = self.genre_page_url(genre_slug, page);
let html = self.fetch_html(&url).await?; let html = self.fetch_html(&url).await?;
let page_owned = page.to_string(); let page_owned = page.to_string();
tokio::task::spawn_blocking(move || parser::parse_genre_anime_document(&html, &page_owned)) tokio::task::spawn_blocking(move || {
.await? otakudesu_parser::parse_genre_anime_document(&html, &page_owned)
})
.await
.map_err(|e| ScrapingError::Parse(e.to_string()))?
} }
pub async fn fetch_anime_full(&self, slug: &str) -> Result<AnimeFullData, AppError> { pub async fn fetch_anime_full(&self, slug: &str) -> Result<AnimeFullData, ScrapingError> {
let url = self.full_episode_url(slug); let url = self.full_episode_url(slug);
let html = self.fetch_html(&url).await?; let html = self.fetch_html(&url).await?;
let slug_owned = slug.to_string(); let slug_owned = slug.to_string();
tokio::task::spawn_blocking(move || parser::parse_anime_full_document(&html, &slug_owned)) tokio::task::spawn_blocking(move || {
.await? otakudesu_parser::parse_anime_full_document(&html, &slug_owned)
})
.await
.map_err(|e| ScrapingError::Parse(e.to_string()))?
} }
async fn fetch_with_proxy_retry(&self, url: &str) -> Result<String, AppError> { async fn fetch_with_proxy_retry(&self, url: &str) -> Result<String, ScrapingError> {
let backoff = default_backoff(); let backoff = default_backoff();
let url_owned = url.to_string(); let url_owned = url.to_string();
let fetch_op = || async { let fetch_op = || async {
@@ -188,12 +208,15 @@ impl AnimeRepository {
} }
Err(e) => { Err(e) => {
warn!("Failed to fetch URL: {}, error: {:?}", url_owned, e); warn!("Failed to fetch URL: {}, error: {:?}", url_owned, e);
Err(transient(e)) Err(transient(ScrapingError::Http(format!(
"Proxy fetch failed: {}",
e
))))
} }
} }
}; };
retry(backoff, fetch_op) retry(backoff, fetch_op)
.await .await
.map_err(|e| AppError::ScraperError(e.to_string())) .map_err(|e| ScrapingError::Http(e.to_string()))
} }
} }
@@ -1,43 +1,96 @@
use crate::shared::errors::AppError; use crate::domain::entity::anime::{
use crate::shared::utils::parse_html; CompleteAnimeItem, DetailGenre, FilterAnimeItem, Genre, GenreAnimeItem, HasPoster,
use crate::shared::utils::scraping::{attr, extract_slug, selector, text, text_from_or}; LatestAnimeItem, OngoingAnimeItem, OngoingAnimeItemWithScore, Pagination,
use once_cell::sync::Lazy; PaginationWithStringPages, SearchAnimeItem,
};
use crate::domain::error::ScrapingError;
use crate::infrastructure::scraping::parsing_utils::parse_html;
use crate::infrastructure::scraping::parsing_utils::{
attr, extract_slug, selector, text, text_from_or,
};
/// Parser-specific types for Alqanime detail data
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
pub struct AlqLink {
pub name: String,
pub url: String,
}
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
pub struct AlqDownloadItem {
pub resolution: String,
pub links: Vec<AlqLink>,
}
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
pub struct AlqRecommendation {
pub title: String,
pub slug: String,
pub poster: String,
pub status: String,
pub r#type: String,
}
impl HasPoster for AlqRecommendation {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
use regex::Regex; use regex::Regex;
use scraper::Selector; use scraper::Selector;
use std::sync::LazyLock;
static ITEM_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("article.bs").unwrap()); #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
static TITLE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".tt h2").unwrap()); pub struct AlqDetailData {
static IMG_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("img").unwrap()); pub title: String,
static SCORE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".numscore").unwrap()); pub alternative_title: String,
static STATUS_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".status").unwrap()); pub poster: String,
static TYPE_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".type").unwrap()); pub poster2: String,
static LINK_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse("a").unwrap()); pub r#type: String,
static PAGINATION_SELECTOR: Lazy<Selector> = pub release_date: String,
Lazy::new(|| Selector::parse(".pagination .page-numbers:not(.next)").unwrap()); pub status: String,
static NEXT_SELECTOR: Lazy<Selector> = Lazy::new(|| Selector::parse(".pagination .next").unwrap()); pub synopsis: String,
static SLUG_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"/([^/]+)/?$").unwrap()); pub studio: String,
static GENRE_SLUG_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"genre-(.+)$").unwrap()); pub genres: Vec<DetailGenre>,
pub fn parse_ongoing_anime( pub producers: Vec<String>,
html: &str, pub recommendations: Vec<AlqRecommendation>,
) -> Result<Vec<crate::shared::types::entities::anime::OngoingAnimeItem>, AppError> { pub batch: Vec<AlqDownloadItem>,
pub ova: Vec<AlqDownloadItem>,
pub downloads: Vec<AlqDownloadItem>,
}
static ITEM_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse("article.bs").unwrap());
static TITLE_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".tt h2").unwrap());
static IMG_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse("img").unwrap());
static SCORE_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".numscore").unwrap());
static STATUS_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".status").unwrap());
static TYPE_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".type").unwrap());
static LINK_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a").unwrap());
static PAGINATION_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".pagination .page-numbers:not(.next)").unwrap());
static NEXT_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse(".pagination .next").unwrap());
static SLUG_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"/([^/]+)/?$").unwrap());
static GENRE_SLUG_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"genre-(.+)$").unwrap());
pub fn parse_ongoing_anime(html: &str) -> Result<Vec<OngoingAnimeItem>, ScrapingError> {
let items = parse_ongoing_anime_with_score(html)?; let items = parse_ongoing_anime_with_score(html)?;
Ok(items Ok(items
.into_iter() .into_iter()
.map( .map(|item| OngoingAnimeItem {
|item| crate::shared::types::entities::anime::OngoingAnimeItem {
title: item.title, title: item.title,
slug: item.slug, slug: item.slug,
poster: item.poster, poster: item.poster,
current_episode: item.score, current_episode: item.score,
anime_url: item.anime_url, anime_url: item.anime_url,
}, })
)
.collect()) .collect())
} }
pub fn parse_complete_anime( pub fn parse_complete_anime(html: &str) -> Result<Vec<CompleteAnimeItem>, ScrapingError> {
html: &str,
) -> Result<Vec<crate::shared::types::entities::anime::CompleteAnimeItem>, AppError> {
let document = parse_html(html); let document = parse_html(html);
let mut complete_anime = Vec::new(); let mut complete_anime = Vec::new();
@@ -68,7 +121,7 @@ pub fn parse_complete_anime(
let episode_count = text_from_or(&element, &STATUS_SELECTOR, "N/A"); let episode_count = text_from_or(&element, &STATUS_SELECTOR, "N/A");
if !title.is_empty() { if !title.is_empty() {
complete_anime.push(crate::shared::types::entities::anime::CompleteAnimeItem { complete_anime.push(CompleteAnimeItem {
title, title,
slug, slug,
poster, poster,
@@ -81,11 +134,11 @@ pub fn parse_complete_anime(
Ok(complete_anime) Ok(complete_anime)
} }
pub fn parse_genres(html: &str) -> Result<Vec<crate::modules::anime2::types::Genre>, AppError> { pub fn parse_genres(html: &str) -> Result<Vec<Genre>, ScrapingError> {
let document = parse_html(html); let document = parse_html(html);
let mut genres = Vec::new(); let mut genres = Vec::new();
let genre_label_selector = selector("label[for^=\"genre-\"]").ok_or_else(|| { let genre_label_selector = selector("label[for^=\"genre-\"]").ok_or_else(|| {
AppError::ScraperError("Invalid selector: label[for^=\"genre-\"]".to_string()) ScrapingError::Parse("Invalid selector: label[for^=\"genre-\"]".to_string())
})?; })?;
for element in document.select(&genre_label_selector) { for element in document.select(&genre_label_selector) {
@@ -100,7 +153,11 @@ pub fn parse_genres(html: &str) -> Result<Vec<crate::modules::anime2::types::Gen
.to_string(); .to_string();
if !name.is_empty() && !slug.is_empty() { if !name.is_empty() && !slug.is_empty() {
genres.push(crate::modules::anime2::types::Genre { name, slug }); genres.push(Genre {
name,
slug,
url: String::new(),
});
} }
} }
@@ -110,13 +167,7 @@ pub fn parse_genres(html: &str) -> Result<Vec<crate::modules::anime2::types::Gen
pub fn parse_filter_page( pub fn parse_filter_page(
html: &str, html: &str,
current_page: u32, current_page: u32,
) -> Result< ) -> Result<(Vec<FilterAnimeItem>, Pagination), ScrapingError> {
(
Vec<crate::shared::types::entities::anime::FilterAnimeItem>,
crate::shared::types::entities::anime::Pagination,
),
AppError,
> {
let document = parse_html(html); let document = parse_html(html);
let mut anime_list = Vec::new(); let mut anime_list = Vec::new();
@@ -167,7 +218,7 @@ pub fn parse_filter_page(
.to_string(); .to_string();
if !title.is_empty() { if !title.is_empty() {
anime_list.push(crate::shared::types::entities::anime::FilterAnimeItem { anime_list.push(FilterAnimeItem {
title, title,
slug, slug,
poster, poster,
@@ -192,7 +243,7 @@ pub fn parse_filter_page(
.unwrap_or(1); .unwrap_or(1);
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
let pagination = crate::shared::types::entities::anime::Pagination { let pagination = Pagination {
current_page, current_page,
last_visible_page, last_visible_page,
has_next_page, has_next_page,
@@ -212,9 +263,7 @@ pub fn parse_filter_page(
Ok((anime_list, pagination)) Ok((anime_list, pagination))
} }
pub fn parse_genre_anime( pub fn parse_genre_anime(html: &str) -> Result<Vec<GenreAnimeItem>, ScrapingError> {
html: &str,
) -> Result<Vec<crate::shared::types::entities::anime::GenreAnimeItem>, AppError> {
let document = parse_html(html); let document = parse_html(html);
let mut anime_list = Vec::new(); let mut anime_list = Vec::new();
@@ -259,10 +308,11 @@ pub fn parse_genre_anime(
.to_string(); .to_string();
if !title.is_empty() { if !title.is_empty() {
anime_list.push(crate::shared::types::entities::anime::GenreAnimeItem { anime_list.push(GenreAnimeItem {
title, title,
slug, slug,
poster, poster,
episode: String::new(),
score, score,
status, status,
anime_url, anime_url,
@@ -273,9 +323,7 @@ pub fn parse_genre_anime(
Ok(anime_list) Ok(anime_list)
} }
pub fn parse_search_anime( pub fn parse_search_anime(html: &str) -> Result<Vec<SearchAnimeItem>, ScrapingError> {
html: &str,
) -> Result<Vec<crate::shared::types::entities::anime::SearchAnimeItem>, AppError> {
let document = parse_html(html); let document = parse_html(html);
let mut anime_list = Vec::new(); let mut anime_list = Vec::new();
@@ -308,14 +356,16 @@ pub fn parse_search_anime(
.to_string(); .to_string();
if !title.is_empty() { if !title.is_empty() {
anime_list.push(crate::shared::types::entities::anime::SearchAnimeItem { anime_list.push(SearchAnimeItem {
title, title,
slug, slug,
poster, poster,
description: String::new(), episode: String::new(),
anime_url, anime_url,
genres: Vec::new(), genres: Vec::new(),
status: String::new(),
rating: "N/A".to_string(), rating: "N/A".to_string(),
description: String::new(),
r#type: "Unknown".to_string(), r#type: "Unknown".to_string(),
season: "Unknown".to_string(), season: "Unknown".to_string(),
}); });
@@ -325,9 +375,7 @@ pub fn parse_search_anime(
Ok(anime_list) Ok(anime_list)
} }
pub fn parse_latest_anime( pub fn parse_latest_anime(html: &str) -> Result<Vec<LatestAnimeItem>, ScrapingError> {
html: &str,
) -> Result<Vec<crate::shared::types::entities::anime::LatestAnimeItem>, AppError> {
let document = parse_html(html); let document = parse_html(html);
let mut anime_list = Vec::new(); let mut anime_list = Vec::new();
@@ -366,11 +414,11 @@ pub fn parse_latest_anime(
.to_string(); .to_string();
if !title.is_empty() { if !title.is_empty() {
anime_list.push(crate::shared::types::entities::anime::LatestAnimeItem { anime_list.push(LatestAnimeItem {
title, title,
slug, slug,
poster, poster,
current_episode: "N/A".to_string(), episode: "N/A".to_string(),
score, score,
anime_url, anime_url,
}); });
@@ -382,7 +430,7 @@ pub fn parse_latest_anime(
pub fn parse_ongoing_anime_with_score( pub fn parse_ongoing_anime_with_score(
html: &str, html: &str,
) -> Result<Vec<crate::shared::types::entities::anime::OngoingAnimeItemWithScore>, AppError> { ) -> Result<Vec<OngoingAnimeItemWithScore>, ScrapingError> {
let document = parse_html(html); let document = parse_html(html);
let mut anime_list = Vec::new(); let mut anime_list = Vec::new();
@@ -421,15 +469,13 @@ pub fn parse_ongoing_anime_with_score(
.to_string(); .to_string();
if !title.is_empty() { if !title.is_empty() {
anime_list.push( anime_list.push(OngoingAnimeItemWithScore {
crate::shared::types::entities::anime::OngoingAnimeItemWithScore {
title, title,
slug, slug,
poster, poster,
score, score,
anime_url, anime_url,
}, });
);
} }
} }
@@ -439,7 +485,7 @@ pub fn parse_ongoing_anime_with_score(
pub fn parse_pagination( pub fn parse_pagination(
document: &scraper::Html, document: &scraper::Html,
current_page: u32, current_page: u32,
) -> Result<crate::shared::types::entities::anime::Pagination, String> { ) -> Result<Pagination, ScrapingError> {
let last_visible_page = document let last_visible_page = document
.select(&PAGINATION_SELECTOR) .select(&PAGINATION_SELECTOR)
.next_back() .next_back()
@@ -453,7 +499,7 @@ pub fn parse_pagination(
.unwrap_or(1); .unwrap_or(1);
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
let pagination = crate::shared::types::entities::anime::Pagination { let pagination = Pagination {
current_page, current_page,
last_visible_page, last_visible_page,
has_next_page, has_next_page,
@@ -476,7 +522,7 @@ pub fn parse_pagination(
pub fn parse_pagination_with_string( pub fn parse_pagination_with_string(
document: &scraper::Html, document: &scraper::Html,
current_page: u32, current_page: u32,
) -> Result<crate::shared::types::entities::anime::PaginationWithStringPages, String> { ) -> Result<PaginationWithStringPages, ScrapingError> {
let last_visible_page = document let last_visible_page = document
.select(&PAGINATION_SELECTOR) .select(&PAGINATION_SELECTOR)
.next_back() .next_back()
@@ -490,7 +536,7 @@ pub fn parse_pagination_with_string(
.unwrap_or(1); .unwrap_or(1);
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some(); let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
let pagination = crate::shared::types::entities::anime::PaginationWithStringPages { let pagination = PaginationWithStringPages {
current_page, current_page,
last_visible_page, last_visible_page,
has_next_page, has_next_page,
@@ -510,55 +556,53 @@ pub fn parse_pagination_with_string(
Ok(pagination) Ok(pagination)
} }
pub fn parse_anime_detail( pub fn parse_anime_detail(html: &str) -> Result<AlqDetailData, ScrapingError> {
html: &str,
) -> Result<crate::modules::anime2::types::AnimeDetailData, AppError> {
let document = parse_html(html); let document = parse_html(html);
let title_selector = selector(".entry-title") let title_selector = selector(".entry-title")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .entry-title".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .entry-title".to_string()))?;
let alt_title_selector = selector(".alter") let alt_title_selector = selector(".alter")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .alter".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .alter".to_string()))?;
let poster_selector = selector(".thumb img, .thumbook img, .wp-post-image, .ts-post-image") let poster_selector = selector(".thumb img, .thumbook img, .wp-post-image, .ts-post-image")
.ok_or_else(|| { .ok_or_else(|| {
AppError::ScraperError( ScrapingError::Parse(
"Invalid selector: .thumb img, .thumbook img, .wp-post-image, .ts-post-image" "Invalid selector: .thumb img, .thumbook img, .wp-post-image, .ts-post-image"
.to_string(), .to_string(),
) )
})?; })?;
let poster2_selector = selector(".bigcover img, .bixbox.animefull .bigcover .ime img") let poster2_selector = selector(".bigcover img, .bixbox.animefull .bigcover .ime img")
.ok_or_else(|| { .ok_or_else(|| {
AppError::ScraperError( ScrapingError::Parse(
"Invalid selector: .bigcover img, .bixbox.animefull .bigcover .ime img".to_string(), "Invalid selector: .bigcover img, .bixbox.animefull .bigcover .ime img".to_string(),
) )
})?; })?;
let spe_span_selector = selector(".info-content .spe span").ok_or_else(|| { let spe_span_selector = selector(".info-content .spe span").ok_or_else(|| {
AppError::ScraperError("Invalid selector: .info-content .spe span".to_string()) ScrapingError::Parse("Invalid selector: .info-content .spe span".to_string())
})?; })?;
let a_selector = let a_selector =
selector("a").ok_or_else(|| AppError::ScraperError("Invalid selector: a".to_string()))?; selector("a").ok_or_else(|| ScrapingError::Parse("Invalid selector: a".to_string()))?;
let synopsis_selector = selector(".entry-content p") let synopsis_selector = selector(".entry-content p")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .entry-content p".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .entry-content p".to_string()))?;
let genre_selector = selector(".genxed a") let genre_selector = selector(".genxed a")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .genxed a".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .genxed a".to_string()))?;
let download_container_selector = selector(".soraddl.dlone") let download_container_selector = selector(".soraddl.dlone")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .soraddl.dlone".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .soraddl.dlone".to_string()))?;
let resolution_selector = selector(".res") let resolution_selector = selector(".res")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .res".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .res".to_string()))?;
let link_selector = selector(".slink a") let link_selector = selector(".slink a")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .slink a".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .slink a".to_string()))?;
let h3_selector = let h3_selector =
selector("h3").ok_or_else(|| AppError::ScraperError("Invalid selector: h3".to_string()))?; selector("h3").ok_or_else(|| ScrapingError::Parse("Invalid selector: h3".to_string()))?;
let recommendation_selector = selector(".listupd .bs") let recommendation_selector = selector(".listupd .bs")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .listupd .bs".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .listupd .bs".to_string()))?;
let rec_title_selector = selector(".ntitle") let rec_title_selector = selector(".ntitle")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .ntitle".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .ntitle".to_string()))?;
let rec_img_selector = selector("img") let rec_img_selector =
.ok_or_else(|| AppError::ScraperError("Invalid selector: img".to_string()))?; selector("img").ok_or_else(|| ScrapingError::Parse("Invalid selector: img".to_string()))?;
let status_selector = selector(".status") let status_selector = selector(".status")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .status".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .status".to_string()))?;
let type_selector = selector(".typez") let type_selector = selector(".typez")
.ok_or_else(|| AppError::ScraperError("Invalid selector: .typez".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: .typez".to_string()))?;
let title = text_from_or(&document.root_element(), &title_selector, ""); let title = text_from_or(&document.root_element(), &title_selector, "");
let alternative_title = text_from_or(&document.root_element(), &alt_title_selector, ""); let alternative_title = text_from_or(&document.root_element(), &alt_title_selector, "");
@@ -616,7 +660,7 @@ pub fn parse_anime_detail(
let name = text(&element); let name = text(&element);
let anime_url = attr(&element, "href").unwrap_or_default(); let anime_url = attr(&element, "href").unwrap_or_default();
let genre_slug = extract_slug(&anime_url); let genre_slug = extract_slug(&anime_url);
genres.push(crate::modules::anime2::types::DetailGenre { genres.push(DetailGenre {
name, name,
slug: genre_slug, slug: genre_slug,
anime_url, anime_url,
@@ -641,7 +685,7 @@ pub fn parse_anime_detail(
let mut all_links = Vec::new(); let mut all_links = Vec::new();
let row_selector = selector("table tr") let row_selector = selector("table tr")
.ok_or_else(|| AppError::ScraperError("Invalid selector: table tr".to_string()))?; .ok_or_else(|| ScrapingError::Parse("Invalid selector: table tr".to_string()))?;
for row in element.select(&row_selector) { for row in element.select(&row_selector) {
let resolution = text_from_or(&row, &resolution_selector, ""); let resolution = text_from_or(&row, &resolution_selector, "");
@@ -655,11 +699,11 @@ pub fn parse_anime_detail(
provider provider
}; };
all_links.push(crate::modules::anime2::types::Link { name, url }); all_links.push(AlqLink { name, url });
} }
} }
let download_item = crate::modules::anime2::types::DownloadItem { let download_item = AlqDownloadItem {
resolution: title, resolution: title,
links: all_links, links: all_links,
}; };
@@ -695,7 +739,7 @@ pub fn parse_anime_detail(
let r#type = text_from_or(&element, &type_selector, ""); let r#type = text_from_or(&element, &type_selector, "");
recommendations.push(crate::modules::anime2::types::Recommendation { recommendations.push(AlqRecommendation {
title, title,
slug: rec_slug, slug: rec_slug,
poster, poster,
@@ -704,7 +748,7 @@ pub fn parse_anime_detail(
}); });
} }
Ok(crate::modules::anime2::types::AnimeDetailData { Ok(AlqDetailData {
title, title,
alternative_title, alternative_title,
poster, poster,
@@ -726,13 +770,7 @@ pub fn parse_anime_detail(
pub fn parse_genre_page( pub fn parse_genre_page(
html: &str, html: &str,
current_page: u32, current_page: u32,
) -> Result< ) -> Result<(Vec<GenreAnimeItem>, Pagination), ScrapingError> {
(
Vec<crate::shared::types::entities::anime::GenreAnimeItem>,
crate::shared::types::entities::anime::Pagination,
),
AppError,
> {
let document = parse_html(html); let document = parse_html(html);
let anime_list = parse_genre_anime(html)?; let anime_list = parse_genre_anime(html)?;
let pagination = parse_pagination(&document, current_page)?; let pagination = parse_pagination(&document, current_page)?;
@@ -742,13 +780,7 @@ pub fn parse_genre_page(
pub fn parse_search_page( pub fn parse_search_page(
html: &str, html: &str,
current_page: u32, current_page: u32,
) -> Result< ) -> Result<(Vec<SearchAnimeItem>, PaginationWithStringPages), ScrapingError> {
(
Vec<crate::shared::types::entities::anime::SearchAnimeItem>,
crate::shared::types::entities::anime::PaginationWithStringPages,
),
AppError,
> {
let document = parse_html(html); let document = parse_html(html);
let data = parse_search_anime(html)?; let data = parse_search_anime(html)?;
let pagination = parse_pagination_with_string(&document, current_page)?; let pagination = parse_pagination_with_string(&document, current_page)?;
@@ -758,13 +790,7 @@ pub fn parse_search_page(
pub fn parse_latest_page( pub fn parse_latest_page(
html: &str, html: &str,
current_page: u32, current_page: u32,
) -> Result< ) -> Result<(Vec<LatestAnimeItem>, Pagination), ScrapingError> {
(
Vec<crate::shared::types::entities::anime::LatestAnimeItem>,
crate::shared::types::entities::anime::Pagination,
),
AppError,
> {
let document = parse_html(html); let document = parse_html(html);
let anime_list = parse_latest_anime(html)?; let anime_list = parse_latest_anime(html)?;
let pagination = parse_pagination(&document, current_page)?; let pagination = parse_pagination(&document, current_page)?;
@@ -774,13 +800,7 @@ pub fn parse_latest_page(
pub fn parse_ongoing_page( pub fn parse_ongoing_page(
html: &str, html: &str,
current_page: u32, current_page: u32,
) -> Result< ) -> Result<(Vec<OngoingAnimeItemWithScore>, Pagination), ScrapingError> {
(
Vec<crate::shared::types::entities::anime::OngoingAnimeItemWithScore>,
crate::shared::types::entities::anime::Pagination,
),
AppError,
> {
let document = parse_html(html); let document = parse_html(html);
let anime_list = parse_ongoing_anime_with_score(html)?; let anime_list = parse_ongoing_anime_with_score(html)?;
let pagination = parse_pagination(&document, current_page)?; let pagination = parse_pagination(&document, current_page)?;
@@ -790,13 +810,7 @@ pub fn parse_ongoing_page(
pub fn parse_complete_page( pub fn parse_complete_page(
html: &str, html: &str,
current_page: u32, current_page: u32,
) -> Result< ) -> Result<(Vec<CompleteAnimeItem>, Pagination), ScrapingError> {
(
Vec<crate::shared::types::entities::anime::CompleteAnimeItem>,
crate::shared::types::entities::anime::Pagination,
),
AppError,
> {
let document = parse_html(html); let document = parse_html(html);
let anime_list = parse_complete_anime(html)?; let anime_list = parse_complete_anime(html)?;
let pagination = parse_pagination(&document, current_page)?; let pagination = parse_pagination(&document, current_page)?;
@@ -1,41 +1,51 @@
use crate::modules::komik::types::{ChapterData, DetailData, Genre, KomikItem, Pagination}; use crate::domain::entity::anime::Pagination;
use crate::shared::utils::parse_html; use crate::domain::entity::komik::{Chapter, ChapterData, DetailData, KomikGenre, KomikItem};
use crate::shared::utils::scraping::{attr, attr_from, attr_from_or, selector, text, text_from_or}; use crate::domain::error::ScrapingError;
use once_cell::sync::Lazy; use crate::infrastructure::scraping::parsing_utils::parse_html;
use crate::infrastructure::scraping::parsing_utils::{
attr, attr_from, attr_from_or, selector, text, text_from_or,
};
use rayon::prelude::*; use rayon::prelude::*;
use regex::Regex; use regex::Regex;
use std::sync::LazyLock;
use tracing::info; use tracing::info;
static TD_LAST_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("td:last-child").unwrap()); static TD_LAST_SELECTOR: LazyLock<scraper::Selector> =
static TITLE_SELECTOR: Lazy<scraper::Selector> = LazyLock::new(|| selector("td:last-child").unwrap());
Lazy::new(|| selector("div#Judul h1 span[itemprop=\"name\"]").unwrap()); static TITLE_SELECTOR: LazyLock<scraper::Selector> =
static H1_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("h1").unwrap()); LazyLock::new(|| selector("div#Judul h1 span[itemprop=\"name\"]").unwrap());
static TITLE_TAG_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("title").unwrap()); static H1_SELECTOR: LazyLock<scraper::Selector> = LazyLock::new(|| selector("h1").unwrap());
static INFO_ROW_SELECTOR: Lazy<scraper::Selector> = static TITLE_TAG_SELECTOR: LazyLock<scraper::Selector> =
Lazy::new(|| selector("table.inftable tr").unwrap()); LazyLock::new(|| selector("title").unwrap());
static POSTER_SELECTOR: Lazy<scraper::Selector> = static INFO_ROW_SELECTOR: LazyLock<scraper::Selector> =
Lazy::new(|| selector("section#Informasi .ims img").unwrap()); LazyLock::new(|| selector("table.inftable tr").unwrap());
static DESC_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("p.desc").unwrap()); static POSTER_SELECTOR: LazyLock<scraper::Selector> =
static CHAPTER_LIST_SELECTOR: Lazy<scraper::Selector> = LazyLock::new(|| selector("section#Informasi .ims img").unwrap());
Lazy::new(|| selector("#Daftar_Chapter tr, tbody#daftarChapter tr").unwrap()); static DESC_SELECTOR: LazyLock<scraper::Selector> = LazyLock::new(|| selector("p.desc").unwrap());
static DATE_LINK_SELECTOR: Lazy<scraper::Selector> = static CHAPTER_LIST_SELECTOR: LazyLock<scraper::Selector> =
Lazy::new(|| selector("td.tanggalseries, .tanggalseries").unwrap()); LazyLock::new(|| selector("#Daftar_Chapter tr, tbody#daftarChapter tr").unwrap());
static JUDUL2_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("div.judul2").unwrap()); static DATE_LINK_SELECTOR: LazyLock<scraper::Selector> =
static GENRE_SELECTOR: Lazy<scraper::Selector> = Lazy::new(|| selector("ul.genre li a").unwrap()); LazyLock::new(|| selector("td.tanggalseries, .tanggalseries").unwrap());
static CHAPTER_LINK_SELECTOR: Lazy<scraper::Selector> = static JUDUL2_SELECTOR: LazyLock<scraper::Selector> =
Lazy::new(|| selector("td.judulseries a").unwrap()); LazyLock::new(|| selector("div.judul2").unwrap());
static CHAPTER_TITLE_REGEX: Lazy<Regex> = static GENRE_SELECTOR: LazyLock<scraper::Selector> =
Lazy::new(|| Regex::new(r"(?i)(?:chapter|ch\.?)\s*([\d\.]+)").unwrap()); LazyLock::new(|| selector("ul.genre li a").unwrap());
static CHAPTER_NUMBER_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"([\d\.]+)").unwrap()); static CHAPTER_LINK_SELECTOR: LazyLock<scraper::Selector> =
LazyLock::new(|| selector("td.judulseries a").unwrap());
static CHAPTER_TITLE_REGEX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)(?:chapter|ch\.?)\s*([\d\.]+)").unwrap());
static CHAPTER_NUMBER_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"([\d\.]+)").unwrap());
pub fn parse_genres(html: &str) -> Result<Vec<Genre>, String> { pub fn parse_genres(html: &str) -> Result<Vec<KomikGenre>, ScrapingError> {
let document = parse_html(html); let document = parse_html(html);
let mut genres = Vec::new(); let mut genres = Vec::new();
let genre_selector = let genre_selector = selector("#Genre .ls3, section#Genre .ls3, .ls3")
selector("#Genre .ls3, section#Genre .ls3, .ls3").ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let genre_name_selector = selector(".ls3p h4, h4").ok_or("Selector error".to_string())?; let genre_name_selector =
let genre_link_selector = selector("a[href*='/genre/']").ok_or("Selector error".to_string())?; selector(".ls3p h4, h4").ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let genre_link_selector =
selector("a[href*='/genre/']").ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let slug_regex = Regex::new(r"/genre/([^/]+)").unwrap(); let slug_regex = Regex::new(r"/genre/([^/]+)").unwrap();
for element in document.select(&genre_selector) { for element in document.select(&genre_selector) {
@@ -50,7 +60,7 @@ pub fn parse_genres(html: &str) -> Result<Vec<Genre>, String> {
.to_string(); .to_string();
if !name.is_empty() && !slug.is_empty() { if !name.is_empty() && !slug.is_empty() {
genres.push(Genre { genres.push(KomikGenre {
name, name,
slug, slug,
count: None, count: None,
@@ -62,22 +72,26 @@ pub fn parse_genres(html: &str) -> Result<Vec<Genre>, String> {
Ok(genres) Ok(genres)
} }
pub fn parse_komik_chapter_document(html: &str, chapter_url: &str) -> Result<ChapterData, String> { pub fn parse_komik_chapter_document(
html: &str,
chapter_url: &str,
) -> Result<ChapterData, ScrapingError> {
let document = parse_html(html); let document = parse_html(html);
let _start_time = std::time::Instant::now(); let _start_time = std::time::Instant::now();
info!("Starting to parse komik chapter document"); info!("Starting to parse komik chapter document");
let title_selector = selector("title").ok_or("Selector error".to_string())?; let title_selector =
selector("title").ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let prev_chapter_selector = selector( let prev_chapter_selector = selector(
"a[aria-label='Prev'][href*='chapter'], .nxpr a:not(.rl):not([href*='#Chapter']), .chprev a, a.prev", "a[aria-label='Prev'][href*='chapter'], .nxpr a:not(.rl):not([href*='#Chapter']), .chprev a, a.prev",
) )
.ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let next_chapter_selector = selector( let next_chapter_selector = selector(
"a[aria-label='Next'][href*='chapter'], .nxpr a.rl, .nxpr a.next, .chnext a, a.next", "a[aria-label='Next'][href*='chapter'], .nxpr a.rl, .nxpr a.next, .chnext a, a.next",
) )
.ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let image_selector = let image_selector = selector("#Baca_Komik img, img.klazy.ww")
selector("#Baca_Komik img, img.klazy.ww").ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let title = document let title = document
.select(&title_selector) .select(&title_selector)
@@ -243,7 +257,7 @@ fn find_table_row_with_text<'a>(
}) })
} }
pub fn parse_komik_detail_document(html: &str) -> Result<DetailData, String> { pub fn parse_komik_detail_document(html: &str) -> Result<DetailData, ScrapingError> {
let start_time = std::time::Instant::now(); let start_time = std::time::Instant::now();
info!("Starting to parse komik detail document"); info!("Starting to parse komik detail document");
@@ -446,7 +460,7 @@ pub fn parse_komik_detail_document(html: &str) -> Result<DetailData, String> {
}) })
.collect(); .collect();
let chapters: Vec<crate::modules::komik::types::Chapter> = raw_chapter_data let chapters: Vec<Chapter> = raw_chapter_data
.par_iter() .par_iter()
.filter_map(|(chapter_text, date_text, href_text)| { .filter_map(|(chapter_text, date_text, href_text)| {
let chapter = { let chapter = {
@@ -474,7 +488,7 @@ pub fn parse_komik_detail_document(html: &str) -> Result<DetailData, String> {
.to_string(); .to_string();
if !chapter_id.is_empty() { if !chapter_id.is_empty() {
Some(crate::modules::komik::types::Chapter { Some(Chapter {
chapter, chapter,
date, date,
chapter_id, chapter_id,
@@ -506,21 +520,26 @@ pub fn parse_komik_detail_document(html: &str) -> Result<DetailData, String> {
pub fn parse_genre_page( pub fn parse_genre_page(
html: &str, html: &str,
current_page: u32, current_page: u32,
) -> Result<(Vec<KomikItem>, Pagination), String> { ) -> Result<(Vec<KomikItem>, Pagination), ScrapingError> {
let document = parse_html(html); let document = parse_html(html);
let mut komik_list = Vec::new(); let mut komik_list = Vec::new();
let item_selector = let item_selector = selector(".bge, article, .ls4, .ls2")
selector(".bge, article, .ls4, .ls2").ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let title_selector = selector(".kan h3, h3 a, h4 a").ok_or("Selector error".to_string())?; let title_selector = selector(".kan h3, h3 a, h4 a")
let img_selector = selector(".bgei img, img.lazy, img").ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let img_selector = selector(".bgei img, img.lazy, img")
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let chapter_selector = selector(".new1:last-of-type a span:last-child, .new1 a:last-child span:last-child, .ls4s a, .ls24, .ls2l a") let chapter_selector = selector(".new1:last-of-type a span:last-child, .new1 a:last-child span:last-child, .ls4s a, .ls24, .ls2l a")
.ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let score_selector = selector(".up, .numscore, .epx").ok_or("Selector error".to_string())?; let score_selector = selector(".up, .numscore, .epx")
let type_selector = selector(".tpe1_inf, .ls3p, .type").ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let link_selector = let type_selector = selector(".tpe1_inf, .ls3p, .type")
selector(".kan h3 a, .bgei a, h3 a, h4 a, a").ok_or("Selector error".to_string())?; .ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let next_selector = selector("span[hx-get]").ok_or("Selector error".to_string())?; let link_selector = selector(".kan h3 a, .bgei a, h3 a, h4 a, a")
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let next_selector =
selector("span[hx-get]").ok_or(ScrapingError::Parse("Selector error".to_string()))?;
let slug_regex = Regex::new(r"/([^/]+)/?$").unwrap(); let slug_regex = Regex::new(r"/([^/]+)/?$").unwrap();
for element in document.select(&item_selector) { for element in document.select(&item_selector) {
@@ -0,0 +1,3 @@
pub mod alqanime_parser;
pub mod komik_parser;
pub mod otakudesu_parser;
@@ -0,0 +1,621 @@
//! Otakudesu HTML parser — native implementation using infrastructure utilities.
//!
//! Parses Otakudesu HTML pages into domain types. All parsing runs in
//! `spawn_blocking` (called from the repository layer).
use crate::domain::entity::anime::*;
use crate::domain::error::ScrapingError;
use crate::infrastructure::scraping::parsing_utils::{
attr, attr_from, attr_from_or, extract_slug, parse_html, selector, text, text_from_or,
};
// ============================================================================
// INDEX (Ongoing + Complete Anime)
// ============================================================================
pub fn parse_ongoing_anime(html: &str) -> Result<Vec<OngoingAnimeItem>, ScrapingError> {
let document = parse_html(html);
let mut items = Vec::new();
let venz_sel = selector(".venz ul li")
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
let title_sel = selector(".thumbz h2.jdlflm")
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let img_sel = selector("img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
let ep_sel = selector(".epz")
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
for element in document.select(&venz_sel) {
let title = text_from_or(&element, &title_sel, "");
let href = attr_from(&element, &link_sel, "href").unwrap_or_default();
let slug = extract_slug(&href);
let poster = attr_from_or(&element, &img_sel, "src", "");
let current_episode = text_from_or(&element, &ep_sel, "N/A");
let anime_url = attr_from_or(&element, &link_sel, "href", "");
if !title.is_empty() {
items.push(OngoingAnimeItem {
title,
slug,
poster,
current_episode,
anime_url,
});
}
}
Ok(items)
}
pub fn parse_complete_anime(html: &str) -> Result<Vec<CompleteAnimeItem>, ScrapingError> {
let document = parse_html(html);
let mut items = Vec::new();
let venz_sel = selector(".venz ul li")
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
let title_sel = selector(".thumbz h2.jdlflm")
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let img_sel = selector("img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
let ep_sel = selector(".epz")
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
for element in document.select(&venz_sel) {
let title = text_from_or(&element, &title_sel, "");
let href = attr_from(&element, &link_sel, "href").unwrap_or_default();
let slug = extract_slug(&href);
let poster = attr_from_or(&element, &img_sel, "src", "");
let episode_count = text_from_or(&element, &ep_sel, "N/A");
let anime_url = attr_from_or(&element, &link_sel, "href", "");
if !title.is_empty() {
items.push(CompleteAnimeItem {
title,
slug,
poster,
episode_count,
anime_url,
});
}
}
Ok(items)
}
// ============================================================================
// GENRES
// ============================================================================
pub fn parse_genres(html: &str) -> Result<Vec<Genre>, ScrapingError> {
let document = parse_html(html);
let mut genres = Vec::new();
let genre_sel = selector(".genres li a, .genre-list a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse genre selector".into()))?;
for element in document.select(&genre_sel) {
let name = text(&element);
let url = attr(&element, "href").unwrap_or_default();
let slug = extract_slug(&url);
if !name.is_empty() && !slug.is_empty() {
genres.push(Genre { name, slug, url });
}
}
Ok(genres)
}
// ============================================================================
// DETAIL
// ============================================================================
pub fn parse_anime_detail_document(html: &str) -> Result<AnimeDetailData, ScrapingError> {
let document = parse_html(html);
let info_sel = selector(".infozingle p")
.ok_or_else(|| ScrapingError::Parse("Failed to parse info selector".into()))?;
let poster_sel = selector(".fotoanime img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse poster selector".into()))?;
let synopsis_sel = selector(".sinopc")
.ok_or_else(|| ScrapingError::Parse("Failed to parse synopsis selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let ep_list_sel = selector(".episodelist ul li a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse episode list selector".into()))?;
let rec_sel = selector("#recommend-anime-series .isi-anime")
.ok_or_else(|| ScrapingError::Parse("Failed to parse recommendation selector".into()))?;
let rec_title_sel = selector(".judul-anime a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse rec title selector".into()))?;
let rec_img_sel = selector("img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse rec img selector".into()))?;
let mut title = String::new();
let mut alternative_title = String::new();
let mut r#type: Option<String> = None;
let mut status: Option<String> = None;
let mut release_date = String::new();
let mut studio = String::new();
for element in document.select(&info_sel) {
let text = text(&element);
if text.contains("Judul:") {
title = text.replace("Judul:", "").trim().to_string();
} else if text.contains("Japanese:") {
alternative_title = text.replace("Japanese:", "").trim().to_string();
} else if text.contains("Type:") {
let type_str = text.replace("Type:", "").trim().to_string();
if !type_str.is_empty() {
r#type = Some(type_str);
}
} else if text.contains("Status:") {
let status_str = text.replace("Status:", "").trim().to_string();
if !status_str.is_empty() {
status = Some(status_str);
}
} else if text.contains("Tanggal Rilis:") {
release_date = text.replace("Tanggal Rilis:", "").trim().to_string();
} else if text.contains("Studio:") {
studio = text.replace("Studio:", "").trim().to_string();
}
}
let poster = document
.select(&poster_sel)
.next()
.and_then(|e| e.value().attr("src"))
.unwrap_or("")
.to_string();
let synopsis = text_from_or(&document.root_element(), &synopsis_sel, "");
let mut genres = Vec::new();
if let Some(genres_element) = document
.select(&info_sel)
.find(|e| text(e).contains("Genres:"))
{
for genre_link in genres_element.select(&link_sel) {
let gname = text(&genre_link);
let anine_url = attr(&genre_link, "href").unwrap_or_default();
let genre_slug = extract_slug(&anine_url);
genres.push(DetailGenre {
name: gname,
slug: genre_slug,
anime_url: anine_url,
});
}
}
let mut episode_lists = Vec::new();
for element in document.select(&ep_list_sel) {
let episode = text(&element);
let href = attr(&element, "href").unwrap_or_default();
let slug = extract_slug(&href);
episode_lists.push(EpisodeList { episode, slug });
}
let mut recommendations = Vec::new();
for element in document.select(&rec_sel) {
let rtitle = text_from_or(&element, &rec_title_sel, "");
let rposter = attr_from_or(&element, &rec_img_sel, "src", "");
let rhref = element
.select(&link_sel)
.next()
.and_then(|e| e.value().attr("href"))
.unwrap_or("");
let rslug = extract_slug(rhref);
recommendations.push(Recommendation {
title: rtitle,
slug: rslug,
poster: rposter,
status: None,
r#type: None,
});
}
Ok(AnimeDetailData {
title,
alternative_title,
poster,
r#type,
status,
release_date,
studio,
genres,
synopsis,
episode_lists,
batch: vec![],
producers: vec![],
recommendations,
})
}
// ============================================================================
// PAGINATION HELPER
// ============================================================================
fn parse_pagination(slug: &str, document: &scraper::Html) -> Result<Pagination, ScrapingError> {
let pagination_sel = selector(".pagenavix .page-numbers:not(.next)")
.ok_or_else(|| ScrapingError::Parse("Failed to parse pagination selector".into()))?;
let next_sel = selector(".pagenavix .next.page-numbers")
.ok_or_else(|| ScrapingError::Parse("Failed to parse next selector".into()))?;
let current_page = slug.parse::<u32>().unwrap_or(1);
let last_visible_page = document
.select(&pagination_sel)
.next_back()
.and_then(|e| e.text().collect::<String>().trim().parse::<u32>().ok())
.unwrap_or(1);
let has_next_page = document.select(&next_sel).next().is_some();
let next_page = if has_next_page {
Some(current_page + 1)
} else {
None
};
let has_previous_page = current_page > 1;
let previous_page = if has_previous_page {
Some(current_page - 1)
} else {
None
};
Ok(Pagination {
current_page,
last_visible_page,
has_next_page,
next_page,
has_previous_page,
previous_page,
})
}
// ============================================================================
// COMPLETE ANIME PAGE
// ============================================================================
pub fn parse_anime_page(
html: &str,
slug: &str,
) -> Result<(Vec<CompleteAnimeListItem>, Pagination), ScrapingError> {
let document = parse_html(html);
let mut items = Vec::new();
let venz_sel = selector(".venz ul li")
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
let title_sel = selector(".thumbz h2.jdlflm")
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let img_sel = selector("img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
let ep_sel = selector(".epz")
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
for element in document.select(&venz_sel) {
let title = text_from_or(&element, &title_sel, "");
let anime_url = attr_from_or(&element, &link_sel, "href", "");
let slug = extract_slug(&anime_url);
let poster = attr_from_or(&element, &img_sel, "src", "");
let episode_count = text_from_or(&element, &ep_sel, "N/A");
if !title.is_empty() {
items.push(CompleteAnimeListItem {
title,
slug,
poster,
episode_count,
anime_url,
});
}
}
let pagination = parse_pagination(slug, &document)?;
Ok((items, pagination))
}
// ============================================================================
// ONGOING ANIME PAGE
// ============================================================================
pub fn parse_ongoing_anime_document(
html: &str,
slug: &str,
) -> Result<(Vec<OngoingAnimeListItem>, Pagination), ScrapingError> {
let document = parse_html(html);
let mut items = Vec::new();
let venz_sel = selector(".venz ul li")
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
let title_sel = selector(".thumbz h2.jdlflm")
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let img_sel = selector("img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
let score_sel = selector(".epz")
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
for element in document.select(&venz_sel) {
let title = text_from_or(&element, &title_sel, "");
let anime_url = attr_from_or(&element, &link_sel, "href", "");
let slug = extract_slug(&anime_url);
let poster = attr_from_or(&element, &img_sel, "src", "");
let score = text_from_or(&element, &score_sel, "N/A");
if !title.is_empty() {
items.push(OngoingAnimeListItem {
title,
slug,
poster,
score,
anime_url,
});
}
}
let pagination = parse_pagination(slug, &document)?;
Ok((items, pagination))
}
// ============================================================================
// LATEST ANIME PAGE
// ============================================================================
pub fn parse_latest_anime_document(
html: &str,
slug: &str,
) -> Result<(Vec<LatestAnimeItem>, Pagination), ScrapingError> {
let document = parse_html(html);
let mut items = Vec::new();
let venz_sel = selector(".venz ul li")
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
let title_sel = selector(".thumbz h2.jdlflm")
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let img_sel = selector("img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
let ep_sel = selector(".epz")
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
for element in document.select(&venz_sel) {
let title = text_from_or(&element, &title_sel, "");
let anime_url = attr_from_or(&element, &link_sel, "href", "");
let slug = extract_slug(&anime_url);
let poster = attr_from_or(&element, &img_sel, "src", "");
let episode = text_from_or(&element, &ep_sel, "N/A");
if !title.is_empty() {
items.push(LatestAnimeItem {
title,
slug,
poster,
episode,
score: String::new(),
anime_url,
});
}
}
let pagination = parse_pagination(slug, &document)?;
Ok((items, pagination))
}
// ============================================================================
// SEARCH ANIME PAGE
// ============================================================================
pub fn parse_search_anime_document(
html: &str,
page: &str,
) -> Result<(Vec<SearchAnimeItem>, Pagination), ScrapingError> {
let document = parse_html(html);
let mut items = Vec::new();
let venz_sel = selector(".venz ul li")
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
let title_sel = selector(".thumbz h2.jdlflm")
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let img_sel = selector("img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
let ep_sel = selector(".epz")
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
let genre_sel = selector(".genre-tag")
.ok_or_else(|| ScrapingError::Parse("Failed to parse genre-tag selector".into()))?;
let status_sel = selector(".status")
.ok_or_else(|| ScrapingError::Parse("Failed to parse status selector".into()))?;
let rating_sel = selector(".rating")
.ok_or_else(|| ScrapingError::Parse("Failed to parse rating selector".into()))?;
for element in document.select(&venz_sel) {
let title = text_from_or(&element, &title_sel, "");
let anime_url = attr_from_or(&element, &link_sel, "href", "");
let slug = extract_slug(&anime_url);
let poster = attr_from_or(&element, &img_sel, "src", "");
let episode = text_from_or(&element, &ep_sel, "N/A");
let mut genres = Vec::new();
for genre_elem in element.select(&genre_sel) {
genres.push(text(&genre_elem));
}
let status = text_from_or(&element, &status_sel, "");
let rating = text_from_or(&element, &rating_sel, "");
if !title.is_empty() {
items.push(SearchAnimeItem {
title,
slug,
poster,
episode,
anime_url,
genres,
status,
rating,
description: String::new(),
r#type: String::new(),
season: String::new(),
});
}
}
let pagination = parse_pagination(page, &document)?;
Ok((items, pagination))
}
// ============================================================================
// GENRE ANIME PAGE
// ============================================================================
pub fn parse_genre_anime_document(
html: &str,
page: &str,
) -> Result<(Vec<GenreAnimeItem>, Pagination), ScrapingError> {
let document = parse_html(html);
let mut items = Vec::new();
let venz_sel = selector(".venz ul li")
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
let title_sel = selector(".thumbz h2.jdlflm")
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let img_sel = selector("img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
let ep_sel = selector(".epz")
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
for element in document.select(&venz_sel) {
let title = text_from_or(&element, &title_sel, "");
let anime_url = attr_from_or(&element, &link_sel, "href", "");
let slug = extract_slug(&anime_url);
let poster = attr_from_or(&element, &img_sel, "src", "");
let episode = text_from_or(&element, &ep_sel, "N/A");
if !title.is_empty() {
items.push(GenreAnimeItem {
title,
slug,
poster,
episode,
score: String::new(),
status: String::new(),
anime_url,
});
}
}
let pagination = parse_pagination(page, &document)?;
Ok((items, pagination))
}
// ============================================================================
// FULL EPISODE PAGE
// ============================================================================
pub fn parse_anime_full_document(html: &str, slug: &str) -> Result<AnimeFullData, ScrapingError> {
let document = parse_html(html);
let ep_title_sel = selector("h1.posttl")
.ok_or_else(|| ScrapingError::Parse("Failed to parse h1.posttl selector".into()))?;
let img_sel = selector(".cukder img")
.ok_or_else(|| ScrapingError::Parse("Failed to parse .cukder img selector".into()))?;
let stream_sel = selector("#embed_holder iframe")
.ok_or_else(|| ScrapingError::Parse("Failed to parse embed_holder selector".into()))?;
let dl_item_sel = selector(".download ul li")
.ok_or_else(|| ScrapingError::Parse("Failed to parse download selector".into()))?;
let res_sel = selector("strong")
.ok_or_else(|| ScrapingError::Parse("Failed to parse strong selector".into()))?;
let link_sel = selector("a")
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
let next_ep_sel = selector(".flir a[title*='Episode Selanjutnya']")
.ok_or_else(|| ScrapingError::Parse("Failed to parse next episode selector".into()))?;
let prev_ep_sel = selector(".flir a[title*='Episode Sebelumnya']")
.ok_or_else(|| ScrapingError::Parse("Failed to parse prev episode selector".into()))?;
let episode = document
.select(&ep_title_sel)
.next()
.map(|e| text(&e))
.unwrap_or_default();
let episode_number = episode
.split("Episode")
.nth(1)
.map(|s| s.trim().to_string())
.unwrap_or_default();
let image_url = document
.select(&img_sel)
.next()
.and_then(|e| attr(&e, "src"))
.unwrap_or_default();
let stream_url = document
.select(&stream_sel)
.next()
.and_then(|e| attr(&e, "src"))
.unwrap_or_default();
let mut download_urls = std::collections::HashMap::new();
for element in document.select(&dl_item_sel) {
let resolution = element
.select(&res_sel)
.next()
.map(|e| text(&e))
.unwrap_or_default();
let mut links = Vec::new();
for link_element in element.select(&link_sel) {
let server = text(&link_element);
let url = attr(&link_element, "href").unwrap_or_default();
links.push(DownloadLink { server, url });
}
if !resolution.is_empty() && !links.is_empty() {
download_urls.insert(resolution, links);
}
}
let next_episode_element = document.select(&next_ep_sel).next();
let previous_episode_element = document.select(&prev_ep_sel).next();
let next_episode_slug = next_episode_element
.and_then(|e| attr(&e, "href"))
.and_then(|href| {
href.split('/')
.nth(href.split('/').count().saturating_sub(2))
.map(|s| s.to_string() + "/")
});
let previous_episode_slug = previous_episode_element
.and_then(|e| attr(&e, "href"))
.and_then(|href| {
href.split('/')
.nth(href.split('/').count().saturating_sub(2))
.map(|s| s.to_string() + "/")
});
Ok(AnimeFullData {
episode,
episode_number,
anime: AnimeInfo {
slug: slug.to_string(),
},
has_next_episode: next_episode_slug.is_some(),
next_episode: next_episode_slug.map(|s| EpisodeInfo { slug: s }),
has_previous_episode: previous_episode_slug.is_some(),
previous_episode: previous_episode_slug.map(|s| EpisodeInfo { slug: s }),
stream_url,
download_urls,
image_url,
})
}
+28
View File
@@ -0,0 +1,28 @@
//! Proxy fetching repository.
use async_trait::async_trait;
use crate::domain::error::ScrapingError;
use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::scraping::proxy_fetch::{self, FetchResult};
pub struct ProxyRepository;
impl ProxyRepository {
pub fn new() -> Self {
Self
}
pub async fn fetch_with_proxy_url(&self, url: &str) -> Result<FetchResult, ScrapingError> {
proxy_fetch::fetch_with_proxy(url)
.await
.map_err(|e| ScrapingError::Http(format!("Proxy fetch failed: {}", e)))
}
}
#[async_trait]
impl ScrapingRepository for ProxyRepository {
async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
self.fetch_with_proxy_url(url).await.map(|r| r.data)
}
}
@@ -0,0 +1,11 @@
//! HTML scraping helpers — re-exports from retry and parsing_utils modules.
//!
//! This module consolidates common scraping utilities for convenient single-path imports.
//! Downstream code should use `crate::infrastructure::scraping::html_fetcher::*`.
// Re-export retry utilities
pub use crate::infrastructure::scraping::retry::{
custom_backoff, default_backoff, permanent, quick_backoff, retry, slow_backoff, transient,
};
// Re-export common scraping helpers (fetch_html_with_retry, parse_html, selector, text, attr, etc.)
pub use crate::infrastructure::scraping::parsing_utils::*;
+5
View File
@@ -0,0 +1,5 @@
pub mod html_fetcher;
pub mod parsing_utils;
pub mod proxy_fetch;
pub mod retry;
pub mod scraping_urls;
@@ -1,16 +1,18 @@
//! HTML scraping helpers using scraper crate. //! HTML scraping helpers using scraper crate.
//!
//! Adapted from shared/utils/web/scraping.rs with domain-level errors.
use crate::shared::errors::AppError; use crate::domain::error::ScrapingError;
use crate::shared::utils::web::proxy_fetch::fetch_with_proxy; use crate::infrastructure::scraping::proxy_fetch::fetch_with_proxy;
use crate::shared::utils::{default_backoff, transient}; use crate::infrastructure::scraping::retry::{default_backoff, transient};
use backoff::future::retry; use backoff::future::retry;
use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use scraper::{ElementRef, Html, Selector}; use scraper::{ElementRef, Html, Selector};
use std::sync::LazyLock;
use tracing::{info, warn}; use tracing::{info, warn};
/// Fetch HTML from URL with retry backoff and proxy support. /// Fetch HTML from URL with retry backoff and proxy support.
pub async fn fetch_html_with_retry(url: &str) -> Result<String, AppError> { pub async fn fetch_html_with_retry(url: &str) -> Result<String, ScrapingError> {
let backoff = default_backoff(); let backoff = default_backoff();
let fetch_operation = || async { let fetch_operation = || async {
info!("Fetching: {}", url); info!("Fetching: {}", url);
@@ -28,7 +30,7 @@ pub async fn fetch_html_with_retry(url: &str) -> Result<String, AppError> {
retry(backoff, fetch_operation) retry(backoff, fetch_operation)
.await .await
.map_err(|e| AppError::ScraperError(e.to_string())) .map_err(|e| ScrapingError::Http(e.to_string()))
} }
/// Parse HTML string into a document. /// Parse HTML string into a document.
@@ -105,7 +107,8 @@ pub fn select_all<'a>(document: &'a Html, css: &str) -> Vec<ElementRef<'a>> {
/// Extract slug from URL (last path segment). /// Extract slug from URL (last path segment).
pub fn extract_slug(url: &str) -> String { pub fn extract_slug(url: &str) -> String {
static SLUG_REGEX: Lazy<Result<Regex, regex::Error>> = Lazy::new(|| Regex::new(r"/([^/]+)/?$")); static SLUG_REGEX: LazyLock<Result<Regex, regex::Error>> =
LazyLock::new(|| Regex::new(r"/([^/]+)/?$"));
SLUG_REGEX SLUG_REGEX
.as_ref() .as_ref()
@@ -118,7 +121,8 @@ pub fn extract_slug(url: &str) -> String {
/// Remove HTML tags from string. /// Remove HTML tags from string.
pub fn strip_tags(html: &str) -> String { pub fn strip_tags(html: &str) -> String {
static TAG_REGEX: Lazy<Result<Regex, regex::Error>> = Lazy::new(|| Regex::new(r"<[^>]+>")); static TAG_REGEX: LazyLock<Result<Regex, regex::Error>> =
LazyLock::new(|| Regex::new(r"<[^>]+>"));
TAG_REGEX TAG_REGEX
.as_ref() .as_ref()
.map(|r| r.replace_all(html, "").trim().to_string()) .map(|r| r.replace_all(html, "").trim().to_string())
@@ -127,7 +131,7 @@ pub fn strip_tags(html: &str) -> String {
/// Extract number from text. /// Extract number from text.
pub fn extract_number(text: &str) -> Option<i64> { pub fn extract_number(text: &str) -> Option<i64> {
static NUM_REGEX: Lazy<Result<Regex, regex::Error>> = Lazy::new(|| Regex::new(r"\d+")); static NUM_REGEX: LazyLock<Result<Regex, regex::Error>> = LazyLock::new(|| Regex::new(r"\d+"));
NUM_REGEX NUM_REGEX
.as_ref() .as_ref()
.ok() .ok()
@@ -137,8 +141,8 @@ pub fn extract_number(text: &str) -> Option<i64> {
/// Extract text inside parentheses. /// Extract text inside parentheses.
pub fn extract_parentheses(text: &str) -> Option<String> { pub fn extract_parentheses(text: &str) -> Option<String> {
static PAREN_REGEX: Lazy<Result<Regex, regex::Error>> = static PAREN_REGEX: LazyLock<Result<Regex, regex::Error>> =
Lazy::new(|| Regex::new(r"\(([^)]+)\)")); LazyLock::new(|| Regex::new(r"\(([^)]+)\)"));
PAREN_REGEX PAREN_REGEX
.as_ref() .as_ref()
.ok() .ok()

Some files were not shown because too many files have changed in this diff Show More