feat: full clean architecture refactor — domain → application → infrastructure → presentation
- Restructure from Modular MVC to Clean Architecture with 4 strict layers - Domain: entities (anime/komik/proxy), Repository traits, typed errors - Application: use case classes with proper error propagation - Infrastructure: repository impls, parsers, Redis cache, HTTP/scraping, browser - Presentation: Axum handlers, DTOs, AppState, router, AppError+IntoResponse - Migrate all parsers (otakudesu, alqanime, komik) to native infra implementations - Replace once_cell::sync::Lazy/OnceCell with std::sync::LazyLock/OnceLock - Remove 150+ old files in modules/ and shared/ directories - Remove once_cell from Cargo.toml dependencies - Fix test/debug binaries to use new import paths - Zero new clippy warnings Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
asepharyanaandClaude Opus 4.8 committed
1 parent
80c96eaa42
commit
776fa828d8
205 files changed
+4241
-14569
No files matched your search
@@ -0,0 +1,101 @@
|
||||
//! Alqanime (Anime2) scraping repository.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::domain::repository::ScrapingRepository;
|
||||
use crate::infrastructure::scraping::proxy_fetch::fetch_with_proxy_only;
|
||||
|
||||
const BASE_URL: &str = "https://alqanime.si";
|
||||
const BASE_DETAIL_URL: &str = "https://alqanime.net";
|
||||
|
||||
pub struct AlqanimeRepository;
|
||||
|
||||
impl AlqanimeRepository {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
pub fn index_ongoing_url(&self) -> String {
|
||||
format!("{}/anime/?status=ongoing&type=&order=update", BASE_URL)
|
||||
}
|
||||
|
||||
pub fn index_complete_url(&self) -> String {
|
||||
format!("{}/anime/?status=completed&type=&order=update", BASE_URL)
|
||||
}
|
||||
|
||||
pub fn genre_list_url(&self) -> String {
|
||||
format!("{}/anime/", BASE_URL)
|
||||
}
|
||||
|
||||
pub fn filter_url(&self, page: u32, order: &str) -> String {
|
||||
if page > 1 {
|
||||
format!("{}/anime/page/{}/?order={}", BASE_URL, page, order)
|
||||
} else {
|
||||
format!("{}/anime/?order={}", BASE_URL, order)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detail_url(&self, slug: &str) -> String {
|
||||
format!("{}/{}/", BASE_DETAIL_URL, slug)
|
||||
}
|
||||
|
||||
pub fn detail_image_url(&self, slug: &str) -> String {
|
||||
format!("{}/anime/{}/", BASE_URL, slug)
|
||||
}
|
||||
|
||||
pub fn genre_page_url(&self, genre_slug: &str, page: u32) -> String {
|
||||
if page > 1 {
|
||||
format!(
|
||||
"{}/anime/page/{}/?genre[]={}&order=update",
|
||||
BASE_URL, page, genre_slug
|
||||
)
|
||||
} else {
|
||||
format!("{}/anime/?genre[]={}&order=update", BASE_URL, genre_slug)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn search_url(&self, query: &str, page: u32) -> String {
|
||||
let encoded = urlencoding::encode(query);
|
||||
if page == 1 {
|
||||
format!("{}/?s={}", BASE_URL, encoded)
|
||||
} else {
|
||||
format!("{}/page/{}/?s={}", BASE_URL, page, encoded)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn latest_url(&self, page: u32) -> String {
|
||||
format!(
|
||||
"{}/anime/page/{}/?status=&type=&order=latest",
|
||||
BASE_URL, page
|
||||
)
|
||||
}
|
||||
|
||||
pub fn ongoing_url(&self, page: u32) -> String {
|
||||
format!(
|
||||
"{}/anime/page/{}/?status=ongoing&type=&order=update",
|
||||
BASE_URL, page
|
||||
)
|
||||
}
|
||||
|
||||
pub fn complete_url(&self, page: u32) -> String {
|
||||
format!(
|
||||
"{}/anime/page/{}/?status=completed&order=update",
|
||||
BASE_URL, page
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ScrapingRepository for AlqanimeRepository {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
|
||||
let response = fetch_with_proxy_only(url)
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("Alqanime fetch failed: {}", e)))?;
|
||||
if response.data.trim().is_empty() {
|
||||
warn!("Alqanime browserless fetch returned empty body for {}", url);
|
||||
}
|
||||
Ok(response.data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
//! SeaORM-backed implementation of ImageCacheRepository.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use deadpool_redis::Pool as RedisPool;
|
||||
use sea_orm::{ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, Set};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::domain::repository::ImageCacheRepository;
|
||||
use crate::infrastructure::cache::redis::Cache;
|
||||
use crate::infrastructure::persistence::entities::image_cache;
|
||||
|
||||
pub struct SeaOrmImageCacheRepository {
|
||||
db: Arc<DatabaseConnection>,
|
||||
redis: RedisPool,
|
||||
}
|
||||
|
||||
impl SeaOrmImageCacheRepository {
|
||||
pub fn new(db: Arc<DatabaseConnection>, redis: RedisPool) -> Self {
|
||||
Self { db, redis }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ImageCacheRepository for SeaOrmImageCacheRepository {
|
||||
async fn get_from_redis(&self, key: &str) -> Option<String> {
|
||||
Cache::new(&self.redis).get::<String>(key).await
|
||||
}
|
||||
|
||||
async fn set_in_redis(&self, key: &str, value: &str, ttl: u64) -> Result<(), String> {
|
||||
Cache::new(&self.redis)
|
||||
.set_with_ttl(key, &value, ttl)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
async fn get_from_db(&self, original_url: &str) -> Result<Option<String>, String> {
|
||||
let entry = image_cache::Entity::find()
|
||||
.filter(image_cache::Column::OriginalUrl.eq(original_url))
|
||||
.one(self.db.as_ref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(entry.map(|m| m.cdn_url))
|
||||
}
|
||||
|
||||
async fn save_to_db(&self, original_url: &str, cdn_url: &str) -> Result<(), String> {
|
||||
let model = image_cache::ActiveModel {
|
||||
id: Set(uuid::Uuid::new_v4().to_string()),
|
||||
original_url: Set(original_url.to_string()),
|
||||
cdn_url: Set(cdn_url.to_string()),
|
||||
created_at: Set(Utc::now()),
|
||||
expires_at: Set(None),
|
||||
};
|
||||
model
|
||||
.insert(self.db.as_ref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn find_original_from_cdn(&self, cdn_url: &str) -> Result<Option<String>, String> {
|
||||
let entry = image_cache::Entity::find()
|
||||
.filter(image_cache::Column::CdnUrl.eq(cdn_url))
|
||||
.one(self.db.as_ref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(entry.map(|m| m.original_url))
|
||||
}
|
||||
|
||||
async fn delete_from_db(&self, original_url: &str) -> Result<(), String> {
|
||||
image_cache::Entity::delete_many()
|
||||
.filter(image_cache::Column::OriginalUrl.eq(original_url))
|
||||
.exec(self.db.as_ref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_from_redis(&self, key: &str) -> Result<(), String> {
|
||||
Cache::new(&self.redis)
|
||||
.delete(key)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
async fn get_lock(&self, key: &str) -> bool {
|
||||
Cache::new(&self.redis).get::<bool>(key).await.is_some()
|
||||
}
|
||||
|
||||
async fn set_lock(&self, key: &str, ttl: u64) -> Result<(), String> {
|
||||
Cache::new(&self.redis)
|
||||
.set_with_ttl(key, &true, ttl)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
async fn release_lock(&self, key: &str) -> Result<(), String> {
|
||||
self.delete_from_redis(key).await
|
||||
}
|
||||
|
||||
async fn invalidate_api_caches(&self, patterns: Vec<&str>) -> Result<(), String> {
|
||||
use deadpool_redis::redis::{cmd, AsyncCommands};
|
||||
|
||||
let mut conn = self.redis.get().await.map_err(|e| e.to_string())?;
|
||||
|
||||
for pattern in patterns {
|
||||
let mut cursor: u64 = 0;
|
||||
loop {
|
||||
let (new_cursor, keys): (u64, Vec<String>) = cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(pattern)
|
||||
.arg("COUNT")
|
||||
.arg(100)
|
||||
.query_async(&mut *conn)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
if !keys.is_empty() {
|
||||
let _: usize = conn.del(&keys).await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
|
||||
cursor = new_cursor;
|
||||
if cursor == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Komik site scraping repository.
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::domain::repository::ScrapingRepository;
|
||||
use crate::infrastructure::scraping::html_fetcher::fetch_html_with_retry;
|
||||
use crate::infrastructure::scraping::scraping_urls::{get_komik_api_url, get_komik_url};
|
||||
|
||||
pub struct KomikRepository;
|
||||
|
||||
impl KomikRepository {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
pub fn api_url(&self) -> String {
|
||||
get_komik_api_url()
|
||||
}
|
||||
|
||||
pub fn base_url(&self) -> String {
|
||||
get_komik_url()
|
||||
}
|
||||
|
||||
pub fn genre_url(&self, genre_slug: &str, page: u32) -> String {
|
||||
if page == 1 {
|
||||
format!("{}/genre/{}/", self.api_url(), genre_slug)
|
||||
} else {
|
||||
format!("{}/genre/{}/page/{}/", self.api_url(), genre_slug, page)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn search_url(&self, query: &str, page: u32) -> String {
|
||||
if page == 1 {
|
||||
format!("{}/search/{}/", self.api_url(), query)
|
||||
} else {
|
||||
format!("{}/search/{}/page/{}/", self.api_url(), query, page)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn manga_list_url(&self, page: u32) -> String {
|
||||
format!("{}/manga/page/{}/?tipe=manga", self.api_url(), page)
|
||||
}
|
||||
|
||||
pub fn manhua_list_url(&self, page: u32) -> String {
|
||||
format!("{}/manga/page/{}/?tipe=manhua", self.api_url(), page)
|
||||
}
|
||||
|
||||
pub fn manhwa_list_url(&self, page: u32) -> String {
|
||||
format!("{}/manga/page/{}/?tipe=manhwa", self.api_url(), page)
|
||||
}
|
||||
|
||||
pub fn popular_list_url(&self, page: u32) -> String {
|
||||
format!(
|
||||
"{}/manga/page/{}/?orderby=meta_value_num",
|
||||
self.api_url(),
|
||||
page
|
||||
)
|
||||
}
|
||||
|
||||
pub fn detail_url(&self, slug: &str) -> String {
|
||||
format!("{}/manga/{}/", self.base_url(), slug)
|
||||
}
|
||||
|
||||
pub fn chapter_url(&self, chapter_url: &str) -> String {
|
||||
format!("{}/{}", self.base_url(), chapter_url)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ScrapingRepository for KomikRepository {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
|
||||
fetch_html_with_retry(url).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
pub mod alqanime;
|
||||
pub mod image_cache_seaorm;
|
||||
pub mod komik;
|
||||
pub mod otakudesu;
|
||||
pub mod parsers;
|
||||
pub mod proxy;
|
||||
|
||||
pub use alqanime::AlqanimeRepository;
|
||||
pub use image_cache_seaorm::SeaOrmImageCacheRepository;
|
||||
pub use komik::KomikRepository;
|
||||
pub use otakudesu::OtakudesuRepository;
|
||||
pub use proxy::ProxyRepository;
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Otakudesu anime scraping repository.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use backoff::future::retry;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::domain::entity::anime::{
|
||||
AnimeData, AnimeDetailData, AnimeFullData, CompleteAnimeListItem, Genre, GenreAnimeItem,
|
||||
LatestAnimeItem, OngoingAnimeListItem, Pagination, SearchAnimeItem,
|
||||
};
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::domain::repository::ScrapingRepository;
|
||||
use crate::infrastructure::repository::parsers::otakudesu_parser;
|
||||
use crate::infrastructure::scraping::html_fetcher::fetch_html_with_retry;
|
||||
use crate::infrastructure::scraping::proxy_fetch::fetch_with_proxy;
|
||||
use crate::infrastructure::scraping::retry::{default_backoff, transient};
|
||||
|
||||
const OTAKUDESU_BASE_URL: &str = "https://otakudesu.cloud";
|
||||
|
||||
pub struct OtakudesuRepository;
|
||||
|
||||
impl OtakudesuRepository {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
fn base_url(&self) -> String {
|
||||
"https://otakudesu.cloud".to_string()
|
||||
}
|
||||
|
||||
pub fn index_urls(&self) -> (String, String) {
|
||||
let base = self.base_url();
|
||||
(
|
||||
format!("{}/ongoing-anime/", base),
|
||||
format!("{}/complete-anime/", base),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn genres_url(&self) -> String {
|
||||
format!("{}/genre-list/", self.base_url())
|
||||
}
|
||||
|
||||
pub fn detail_url(&self, slug: &str) -> String {
|
||||
format!("{}/anime/{}", OTAKUDESU_BASE_URL, slug)
|
||||
}
|
||||
|
||||
pub fn page_url(&self, category: &str, page: &str) -> String {
|
||||
format!("{}/{}/page/{}/", OTAKUDESU_BASE_URL, category, page)
|
||||
}
|
||||
|
||||
pub fn search_url(&self, query: &str, page: &str) -> String {
|
||||
if page == "1" {
|
||||
format!("{}/search/{}/", self.base_url(), query)
|
||||
} else {
|
||||
format!("{}/search/{}/page/{}/", self.base_url(), query, page)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn genre_page_url(&self, genre_slug: &str, page: &str) -> String {
|
||||
format!("{}/genre/{}/page/{}/", self.base_url(), genre_slug, page)
|
||||
}
|
||||
|
||||
pub fn full_episode_url(&self, slug: &str) -> String {
|
||||
format!("{}/episode/{}", OTAKUDESU_BASE_URL, slug)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ScrapingRepository for OtakudesuRepository {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
|
||||
fetch_html_with_retry(url).await
|
||||
}
|
||||
}
|
||||
|
||||
impl OtakudesuRepository {
|
||||
pub async fn fetch_anime_index(&self) -> Result<AnimeData, ScrapingError> {
|
||||
let (ongoing_url, complete_url) = self.index_urls();
|
||||
let (ongoing_html, complete_html) = tokio::join!(
|
||||
self.fetch_html(&ongoing_url),
|
||||
self.fetch_html(&complete_url)
|
||||
);
|
||||
let ongoing_html = ongoing_html?;
|
||||
let complete_html = complete_html?;
|
||||
|
||||
let ongoing_anime = tokio::task::spawn_blocking(move || {
|
||||
otakudesu_parser::parse_ongoing_anime(&ongoing_html)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))??;
|
||||
|
||||
let complete_anime = tokio::task::spawn_blocking(move || {
|
||||
otakudesu_parser::parse_complete_anime(&complete_html)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))??;
|
||||
|
||||
Ok(AnimeData {
|
||||
ongoing_anime,
|
||||
complete_anime,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn fetch_genres(&self) -> Result<Vec<Genre>, ScrapingError> {
|
||||
let html = self.fetch_html(&self.genres_url()).await?;
|
||||
tokio::task::spawn_blocking(move || otakudesu_parser::parse_genres(&html))
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))?
|
||||
}
|
||||
|
||||
pub async fn fetch_anime_detail(&self, slug: &str) -> Result<AnimeDetailData, ScrapingError> {
|
||||
let url = self.detail_url(slug);
|
||||
let html = self.fetch_with_proxy_retry(&url).await?;
|
||||
tokio::task::spawn_blocking(move || otakudesu_parser::parse_anime_detail_document(&html))
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))?
|
||||
}
|
||||
|
||||
pub async fn fetch_complete_anime_page(
|
||||
&self,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<CompleteAnimeListItem>, Pagination), ScrapingError> {
|
||||
let url = self.page_url("complete-anime", slug);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let slug_owned = slug.to_string();
|
||||
tokio::task::spawn_blocking(move || otakudesu_parser::parse_anime_page(&html, &slug_owned))
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))?
|
||||
}
|
||||
|
||||
pub async fn fetch_ongoing_anime_page(
|
||||
&self,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<OngoingAnimeListItem>, Pagination), ScrapingError> {
|
||||
let url = self.page_url("ongoing-anime", slug);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let slug_owned = slug.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
otakudesu_parser::parse_ongoing_anime_document(&html, &slug_owned)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))?
|
||||
}
|
||||
|
||||
pub async fn fetch_latest_anime_page(
|
||||
&self,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<LatestAnimeItem>, Pagination), ScrapingError> {
|
||||
let url = self.page_url("latest-anime", slug);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let slug_owned = slug.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
otakudesu_parser::parse_latest_anime_document(&html, &slug_owned)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))?
|
||||
}
|
||||
|
||||
pub async fn fetch_search_anime_page(
|
||||
&self,
|
||||
slug: &str,
|
||||
page: &str,
|
||||
) -> Result<(Vec<SearchAnimeItem>, Pagination), ScrapingError> {
|
||||
let url = self.search_url(slug, page);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let page_owned = page.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
otakudesu_parser::parse_search_anime_document(&html, &page_owned)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))?
|
||||
}
|
||||
|
||||
pub async fn fetch_genre_anime_page(
|
||||
&self,
|
||||
genre_slug: &str,
|
||||
page: &str,
|
||||
) -> Result<(Vec<GenreAnimeItem>, Pagination), ScrapingError> {
|
||||
let url = self.genre_page_url(genre_slug, page);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let page_owned = page.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
otakudesu_parser::parse_genre_anime_document(&html, &page_owned)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))?
|
||||
}
|
||||
|
||||
pub async fn fetch_anime_full(&self, slug: &str) -> Result<AnimeFullData, ScrapingError> {
|
||||
let url = self.full_episode_url(slug);
|
||||
let html = self.fetch_html(&url).await?;
|
||||
let slug_owned = slug.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
otakudesu_parser::parse_anime_full_document(&html, &slug_owned)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Parse(e.to_string()))?
|
||||
}
|
||||
|
||||
async fn fetch_with_proxy_retry(&self, url: &str) -> Result<String, ScrapingError> {
|
||||
let backoff = default_backoff();
|
||||
let url_owned = url.to_string();
|
||||
let fetch_op = || async {
|
||||
info!("Fetching URL: {}", url_owned);
|
||||
match fetch_with_proxy(&url_owned).await {
|
||||
Ok(response) => {
|
||||
info!("Successfully fetched URL: {}", url_owned);
|
||||
Ok(response.data)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to fetch URL: {}, error: {:?}", url_owned, e);
|
||||
Err(transient(ScrapingError::Http(format!(
|
||||
"Proxy fetch failed: {}",
|
||||
e
|
||||
))))
|
||||
}
|
||||
}
|
||||
};
|
||||
retry(backoff, fetch_op)
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(e.to_string()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,818 @@
|
||||
use crate::domain::entity::anime::{
|
||||
CompleteAnimeItem, DetailGenre, FilterAnimeItem, Genre, GenreAnimeItem, HasPoster,
|
||||
LatestAnimeItem, OngoingAnimeItem, OngoingAnimeItemWithScore, Pagination,
|
||||
PaginationWithStringPages, SearchAnimeItem,
|
||||
};
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::infrastructure::scraping::parsing_utils::parse_html;
|
||||
use crate::infrastructure::scraping::parsing_utils::{
|
||||
attr, extract_slug, selector, text, text_from_or,
|
||||
};
|
||||
|
||||
/// Parser-specific types for Alqanime detail data
|
||||
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
|
||||
pub struct AlqLink {
|
||||
pub name: String,
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
|
||||
pub struct AlqDownloadItem {
|
||||
pub resolution: String,
|
||||
pub links: Vec<AlqLink>,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
|
||||
pub struct AlqRecommendation {
|
||||
pub title: String,
|
||||
pub slug: String,
|
||||
pub poster: String,
|
||||
pub status: String,
|
||||
pub r#type: String,
|
||||
}
|
||||
|
||||
impl HasPoster for AlqRecommendation {
|
||||
fn poster(&self) -> &str {
|
||||
&self.poster
|
||||
}
|
||||
fn set_poster(&mut self, url: String) {
|
||||
self.poster = url;
|
||||
}
|
||||
}
|
||||
|
||||
use regex::Regex;
|
||||
use scraper::Selector;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema, Debug, Clone)]
|
||||
pub struct AlqDetailData {
|
||||
pub title: String,
|
||||
pub alternative_title: String,
|
||||
pub poster: String,
|
||||
pub poster2: String,
|
||||
pub r#type: String,
|
||||
pub release_date: String,
|
||||
pub status: String,
|
||||
pub synopsis: String,
|
||||
pub studio: String,
|
||||
pub genres: Vec<DetailGenre>,
|
||||
pub producers: Vec<String>,
|
||||
pub recommendations: Vec<AlqRecommendation>,
|
||||
pub batch: Vec<AlqDownloadItem>,
|
||||
pub ova: Vec<AlqDownloadItem>,
|
||||
pub downloads: Vec<AlqDownloadItem>,
|
||||
}
|
||||
|
||||
static ITEM_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse("article.bs").unwrap());
|
||||
static TITLE_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".tt h2").unwrap());
|
||||
static IMG_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse("img").unwrap());
|
||||
static SCORE_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".numscore").unwrap());
|
||||
static STATUS_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".status").unwrap());
|
||||
static TYPE_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse(".type").unwrap());
|
||||
static LINK_SELECTOR: LazyLock<Selector> = LazyLock::new(|| Selector::parse("a").unwrap());
|
||||
static PAGINATION_SELECTOR: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse(".pagination .page-numbers:not(.next)").unwrap());
|
||||
static NEXT_SELECTOR: LazyLock<Selector> =
|
||||
LazyLock::new(|| Selector::parse(".pagination .next").unwrap());
|
||||
static SLUG_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"/([^/]+)/?$").unwrap());
|
||||
static GENRE_SLUG_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"genre-(.+)$").unwrap());
|
||||
pub fn parse_ongoing_anime(html: &str) -> Result<Vec<OngoingAnimeItem>, ScrapingError> {
|
||||
let items = parse_ongoing_anime_with_score(html)?;
|
||||
Ok(items
|
||||
.into_iter()
|
||||
.map(|item| OngoingAnimeItem {
|
||||
title: item.title,
|
||||
slug: item.slug,
|
||||
poster: item.poster,
|
||||
current_episode: item.score,
|
||||
anime_url: item.anime_url,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn parse_complete_anime(html: &str) -> Result<Vec<CompleteAnimeItem>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut complete_anime = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let episode_count = text_from_or(&element, &STATUS_SELECTOR, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
complete_anime.push(CompleteAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode_count,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(complete_anime)
|
||||
}
|
||||
|
||||
pub fn parse_genres(html: &str) -> Result<Vec<Genre>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut genres = Vec::new();
|
||||
let genre_label_selector = selector("label[for^=\"genre-\"]").ok_or_else(|| {
|
||||
ScrapingError::Parse("Invalid selector: label[for^=\"genre-\"]".to_string())
|
||||
})?;
|
||||
|
||||
for element in document.select(&genre_label_selector) {
|
||||
let name = text(&element).trim().to_string();
|
||||
let for_attr = attr(&element, "for").unwrap_or_default();
|
||||
|
||||
let slug = GENRE_SLUG_REGEX
|
||||
.captures(&for_attr)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !name.is_empty() && !slug.is_empty() {
|
||||
genres.push(Genre {
|
||||
name,
|
||||
slug,
|
||||
url: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(genres)
|
||||
}
|
||||
|
||||
pub fn parse_filter_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<(Vec<FilterAnimeItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let score = element
|
||||
.select(&SCORE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("N/A".to_string());
|
||||
|
||||
let status = element
|
||||
.select(&STATUS_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("Unknown".to_string());
|
||||
|
||||
let anime_type = element
|
||||
.select(&TYPE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("Unknown".to_string());
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(FilterAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
status,
|
||||
r#type: anime_type,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let last_visible_page = document
|
||||
.select(&PAGINATION_SELECTOR)
|
||||
.next_back()
|
||||
.map(|e| {
|
||||
e.text()
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.unwrap_or(1)
|
||||
})
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page: if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
has_previous_page: current_page > 1,
|
||||
previous_page: if current_page > 1 {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_genre_anime(html: &str) -> Result<Vec<GenreAnimeItem>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let score = element
|
||||
.select(&SCORE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("N/A".to_string());
|
||||
|
||||
let status = element
|
||||
.select(&STATUS_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("Unknown".to_string());
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(GenreAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode: String::new(),
|
||||
score,
|
||||
status,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(anime_list)
|
||||
}
|
||||
|
||||
pub fn parse_search_anime(html: &str) -> Result<Vec<SearchAnimeItem>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(SearchAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode: String::new(),
|
||||
anime_url,
|
||||
genres: Vec::new(),
|
||||
status: String::new(),
|
||||
rating: "N/A".to_string(),
|
||||
description: String::new(),
|
||||
r#type: "Unknown".to_string(),
|
||||
season: "Unknown".to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(anime_list)
|
||||
}
|
||||
|
||||
pub fn parse_latest_anime(html: &str) -> Result<Vec<LatestAnimeItem>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let score = element
|
||||
.select(&SCORE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("N/A".to_string());
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(LatestAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode: "N/A".to_string(),
|
||||
score,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(anime_list)
|
||||
}
|
||||
|
||||
pub fn parse_ongoing_anime_with_score(
|
||||
html: &str,
|
||||
) -> Result<Vec<OngoingAnimeItemWithScore>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut anime_list = Vec::new();
|
||||
|
||||
for element in document.select(&ITEM_SELECTOR) {
|
||||
let title = element
|
||||
.select(&TITLE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = element
|
||||
.select(&IMG_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src").or(e.value().attr("data-src")))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let score = element
|
||||
.select(&SCORE_SELECTOR)
|
||||
.next()
|
||||
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||
.unwrap_or("N/A".to_string());
|
||||
|
||||
let anime_url = element
|
||||
.select(&LINK_SELECTOR)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let slug = SLUG_REGEX
|
||||
.captures(&anime_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
anime_list.push(OngoingAnimeItemWithScore {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(anime_list)
|
||||
}
|
||||
|
||||
pub fn parse_pagination(
|
||||
document: &scraper::Html,
|
||||
current_page: u32,
|
||||
) -> Result<Pagination, ScrapingError> {
|
||||
let last_visible_page = document
|
||||
.select(&PAGINATION_SELECTOR)
|
||||
.next_back()
|
||||
.map(|e| {
|
||||
e.text()
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.unwrap_or(1)
|
||||
})
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page: if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
has_previous_page: current_page > 1,
|
||||
previous_page: if current_page > 1 {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
Ok(pagination)
|
||||
}
|
||||
|
||||
pub fn parse_pagination_with_string(
|
||||
document: &scraper::Html,
|
||||
current_page: u32,
|
||||
) -> Result<PaginationWithStringPages, ScrapingError> {
|
||||
let last_visible_page = document
|
||||
.select(&PAGINATION_SELECTOR)
|
||||
.next_back()
|
||||
.map(|e| {
|
||||
e.text()
|
||||
.collect::<String>()
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.unwrap_or(1)
|
||||
})
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&NEXT_SELECTOR).next().is_some();
|
||||
let pagination = PaginationWithStringPages {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page: if has_next_page {
|
||||
Some((current_page + 1).to_string())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
has_previous_page: current_page > 1,
|
||||
previous_page: if current_page > 1 {
|
||||
Some((current_page - 1).to_string())
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
Ok(pagination)
|
||||
}
|
||||
|
||||
pub fn parse_anime_detail(html: &str) -> Result<AlqDetailData, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
|
||||
let title_selector = selector(".entry-title")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .entry-title".to_string()))?;
|
||||
let alt_title_selector = selector(".alter")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .alter".to_string()))?;
|
||||
let poster_selector = selector(".thumb img, .thumbook img, .wp-post-image, .ts-post-image")
|
||||
.ok_or_else(|| {
|
||||
ScrapingError::Parse(
|
||||
"Invalid selector: .thumb img, .thumbook img, .wp-post-image, .ts-post-image"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
let poster2_selector = selector(".bigcover img, .bixbox.animefull .bigcover .ime img")
|
||||
.ok_or_else(|| {
|
||||
ScrapingError::Parse(
|
||||
"Invalid selector: .bigcover img, .bixbox.animefull .bigcover .ime img".to_string(),
|
||||
)
|
||||
})?;
|
||||
let spe_span_selector = selector(".info-content .spe span").ok_or_else(|| {
|
||||
ScrapingError::Parse("Invalid selector: .info-content .spe span".to_string())
|
||||
})?;
|
||||
let a_selector =
|
||||
selector("a").ok_or_else(|| ScrapingError::Parse("Invalid selector: a".to_string()))?;
|
||||
let synopsis_selector = selector(".entry-content p")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .entry-content p".to_string()))?;
|
||||
let genre_selector = selector(".genxed a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .genxed a".to_string()))?;
|
||||
let download_container_selector = selector(".soraddl.dlone")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .soraddl.dlone".to_string()))?;
|
||||
let resolution_selector = selector(".res")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .res".to_string()))?;
|
||||
let link_selector = selector(".slink a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .slink a".to_string()))?;
|
||||
let h3_selector =
|
||||
selector("h3").ok_or_else(|| ScrapingError::Parse("Invalid selector: h3".to_string()))?;
|
||||
let recommendation_selector = selector(".listupd .bs")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .listupd .bs".to_string()))?;
|
||||
let rec_title_selector = selector(".ntitle")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .ntitle".to_string()))?;
|
||||
let rec_img_selector =
|
||||
selector("img").ok_or_else(|| ScrapingError::Parse("Invalid selector: img".to_string()))?;
|
||||
let status_selector = selector(".status")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .status".to_string()))?;
|
||||
let type_selector = selector(".typez")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: .typez".to_string()))?;
|
||||
|
||||
let title = text_from_or(&document.root_element(), &title_selector, "");
|
||||
let alternative_title = text_from_or(&document.root_element(), &alt_title_selector, "");
|
||||
|
||||
let poster = document
|
||||
.select(&poster_selector)
|
||||
.next()
|
||||
.and_then(|e| {
|
||||
attr(&e, "src")
|
||||
.or_else(|| attr(&e, "data-src"))
|
||||
.or_else(|| attr(&e, "data-lazy-src"))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster2 = document
|
||||
.select(&poster2_selector)
|
||||
.next()
|
||||
.and_then(|e| {
|
||||
attr(&e, "src")
|
||||
.or_else(|| attr(&e, "data-src"))
|
||||
.or_else(|| attr(&e, "data-lazy-src"))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let r#type = document
|
||||
.select(&spe_span_selector)
|
||||
.find(|e| text(&e).contains("Tipe:"))
|
||||
.and_then(|span| span.select(&a_selector).next())
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let release_date = document
|
||||
.select(&spe_span_selector)
|
||||
.find(|e| text(&e).contains("Dirilis:"))
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let status = document
|
||||
.select(&spe_span_selector)
|
||||
.find(|e| text(&e).contains("Status:"))
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let synopsis = text_from_or(&document.root_element(), &synopsis_selector, "");
|
||||
|
||||
let studio = document
|
||||
.select(&spe_span_selector)
|
||||
.find(|e| text(&e).contains("Studio:"))
|
||||
.and_then(|span| span.select(&a_selector).next())
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut genres = Vec::new();
|
||||
for element in document.select(&genre_selector) {
|
||||
let name = text(&element);
|
||||
let anime_url = attr(&element, "href").unwrap_or_default();
|
||||
let genre_slug = extract_slug(&anime_url);
|
||||
genres.push(DetailGenre {
|
||||
name,
|
||||
slug: genre_slug,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
|
||||
let mut batch = Vec::new();
|
||||
let mut ova = Vec::new();
|
||||
let mut downloads = Vec::new();
|
||||
|
||||
for element in document.select(&download_container_selector) {
|
||||
let title = element
|
||||
.select(&h3_selector)
|
||||
.next()
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_else(|| "Unknown".to_string());
|
||||
|
||||
let category = title.to_lowercase();
|
||||
let is_batch = category.contains("batch");
|
||||
let is_ova = category.contains("ova");
|
||||
|
||||
let mut all_links = Vec::new();
|
||||
|
||||
let row_selector = selector("table tr")
|
||||
.ok_or_else(|| ScrapingError::Parse("Invalid selector: table tr".to_string()))?;
|
||||
for row in element.select(&row_selector) {
|
||||
let resolution = text_from_or(&row, &resolution_selector, "");
|
||||
|
||||
for link_element in row.select(&link_selector) {
|
||||
let provider = text(&link_element);
|
||||
let url = attr(&link_element, "href").unwrap_or_default();
|
||||
|
||||
let name = if !resolution.is_empty() {
|
||||
format!("{} - {}", resolution, provider)
|
||||
} else {
|
||||
provider
|
||||
};
|
||||
|
||||
all_links.push(AlqLink { name, url });
|
||||
}
|
||||
}
|
||||
|
||||
let download_item = AlqDownloadItem {
|
||||
resolution: title,
|
||||
links: all_links,
|
||||
};
|
||||
|
||||
if is_batch {
|
||||
batch.push(download_item);
|
||||
} else if is_ova {
|
||||
ova.push(download_item);
|
||||
} else {
|
||||
downloads.push(download_item);
|
||||
}
|
||||
}
|
||||
|
||||
let mut recommendations = Vec::new();
|
||||
for element in document.select(&recommendation_selector) {
|
||||
let title = text_from_or(&element, &rec_title_selector, "");
|
||||
|
||||
let anime_url = element
|
||||
.select(&a_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.unwrap_or_default();
|
||||
|
||||
let rec_slug = extract_slug(&anime_url);
|
||||
|
||||
let poster = element
|
||||
.select(&rec_img_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "data-src").or_else(|| attr(&e, "src")))
|
||||
.unwrap_or_default();
|
||||
|
||||
let status = text_from_or(&element, &status_selector, "");
|
||||
|
||||
let r#type = text_from_or(&element, &type_selector, "");
|
||||
|
||||
recommendations.push(AlqRecommendation {
|
||||
title,
|
||||
slug: rec_slug,
|
||||
poster,
|
||||
status,
|
||||
r#type,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(AlqDetailData {
|
||||
title,
|
||||
alternative_title,
|
||||
poster,
|
||||
poster2,
|
||||
r#type,
|
||||
release_date,
|
||||
status,
|
||||
synopsis,
|
||||
studio,
|
||||
genres,
|
||||
producers: vec![],
|
||||
recommendations,
|
||||
batch,
|
||||
ova,
|
||||
downloads,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_genre_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<(Vec<GenreAnimeItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let anime_list = parse_genre_anime(html)?;
|
||||
let pagination = parse_pagination(&document, current_page)?;
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_search_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<(Vec<SearchAnimeItem>, PaginationWithStringPages), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let data = parse_search_anime(html)?;
|
||||
let pagination = parse_pagination_with_string(&document, current_page)?;
|
||||
Ok((data, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_latest_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<(Vec<LatestAnimeItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let anime_list = parse_latest_anime(html)?;
|
||||
let pagination = parse_pagination(&document, current_page)?;
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_ongoing_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<(Vec<OngoingAnimeItemWithScore>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let anime_list = parse_ongoing_anime_with_score(html)?;
|
||||
let pagination = parse_pagination(&document, current_page)?;
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
|
||||
pub fn parse_complete_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<(Vec<CompleteAnimeItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let anime_list = parse_complete_anime(html)?;
|
||||
let pagination = parse_pagination(&document, current_page)?;
|
||||
Ok((anime_list, pagination))
|
||||
}
|
||||
@@ -0,0 +1,611 @@
|
||||
use crate::domain::entity::anime::Pagination;
|
||||
use crate::domain::entity::komik::{Chapter, ChapterData, DetailData, KomikGenre, KomikItem};
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::infrastructure::scraping::parsing_utils::parse_html;
|
||||
use crate::infrastructure::scraping::parsing_utils::{
|
||||
attr, attr_from, attr_from_or, selector, text, text_from_or,
|
||||
};
|
||||
use rayon::prelude::*;
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
use tracing::info;
|
||||
|
||||
static TD_LAST_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("td:last-child").unwrap());
|
||||
static TITLE_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("div#Judul h1 span[itemprop=\"name\"]").unwrap());
|
||||
static H1_SELECTOR: LazyLock<scraper::Selector> = LazyLock::new(|| selector("h1").unwrap());
|
||||
static TITLE_TAG_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("title").unwrap());
|
||||
static INFO_ROW_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("table.inftable tr").unwrap());
|
||||
static POSTER_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("section#Informasi .ims img").unwrap());
|
||||
static DESC_SELECTOR: LazyLock<scraper::Selector> = LazyLock::new(|| selector("p.desc").unwrap());
|
||||
static CHAPTER_LIST_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("#Daftar_Chapter tr, tbody#daftarChapter tr").unwrap());
|
||||
static DATE_LINK_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("td.tanggalseries, .tanggalseries").unwrap());
|
||||
static JUDUL2_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("div.judul2").unwrap());
|
||||
static GENRE_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("ul.genre li a").unwrap());
|
||||
static CHAPTER_LINK_SELECTOR: LazyLock<scraper::Selector> =
|
||||
LazyLock::new(|| selector("td.judulseries a").unwrap());
|
||||
static CHAPTER_TITLE_REGEX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)(?:chapter|ch\.?)\s*([\d\.]+)").unwrap());
|
||||
static CHAPTER_NUMBER_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"([\d\.]+)").unwrap());
|
||||
|
||||
pub fn parse_genres(html: &str) -> Result<Vec<KomikGenre>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut genres = Vec::new();
|
||||
|
||||
let genre_selector = selector("#Genre .ls3, section#Genre .ls3, .ls3")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let genre_name_selector =
|
||||
selector(".ls3p h4, h4").ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let genre_link_selector =
|
||||
selector("a[href*='/genre/']").ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let slug_regex = Regex::new(r"/genre/([^/]+)").unwrap();
|
||||
|
||||
for element in document.select(&genre_selector) {
|
||||
let name = text_from_or(&element, &genre_name_selector, "");
|
||||
let href = attr_from(&element, &genre_link_selector, "href").unwrap_or_default();
|
||||
|
||||
let slug = slug_regex
|
||||
.captures(&href)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !name.is_empty() && !slug.is_empty() {
|
||||
genres.push(KomikGenre {
|
||||
name,
|
||||
slug,
|
||||
count: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
info!("Parsed {} genres", genres.len());
|
||||
Ok(genres)
|
||||
}
|
||||
|
||||
pub fn parse_komik_chapter_document(
|
||||
html: &str,
|
||||
chapter_url: &str,
|
||||
) -> Result<ChapterData, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let _start_time = std::time::Instant::now();
|
||||
info!("Starting to parse komik chapter document");
|
||||
|
||||
let title_selector =
|
||||
selector("title").ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let prev_chapter_selector = selector(
|
||||
"a[aria-label='Prev'][href*='chapter'], .nxpr a:not(.rl):not([href*='#Chapter']), .chprev a, a.prev",
|
||||
)
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let next_chapter_selector = selector(
|
||||
"a[aria-label='Next'][href*='chapter'], .nxpr a.rl, .nxpr a.next, .chnext a, a.next",
|
||||
)
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let image_selector = selector("#Baca_Komik img, img.klazy.ww")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
|
||||
let title = document
|
||||
.select(&title_selector)
|
||||
.next()
|
||||
.map(|e| {
|
||||
let full_title = text(&e);
|
||||
if let Some(start) = full_title.find("Komik ") {
|
||||
if let Some(end) = full_title.find(" - Komiku") {
|
||||
full_title[start + 6..end].trim().to_string()
|
||||
} else {
|
||||
full_title
|
||||
}
|
||||
} else {
|
||||
full_title
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let next_chapter_id = document
|
||||
.select(&next_chapter_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.map(|href| {
|
||||
href.trim_end_matches('/')
|
||||
.split('/')
|
||||
.filter(|s| !s.is_empty())
|
||||
.next_back()
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
fn get_previous_chapter_id(chapter_url: &str) -> String {
|
||||
const CHAPTER_PATTERN: &str = "chapter-";
|
||||
|
||||
if let Some(pattern_pos) = chapter_url.rfind(CHAPTER_PATTERN) {
|
||||
let prefix = &chapter_url[0..pattern_pos];
|
||||
let suffix = &chapter_url[pattern_pos + CHAPTER_PATTERN.len()..];
|
||||
|
||||
let chapter_num = suffix
|
||||
.chars()
|
||||
.take_while(|c| c.is_ascii_digit())
|
||||
.collect::<String>();
|
||||
|
||||
if let Ok(num) = chapter_num.parse::<u32>() {
|
||||
let prev_num = num.saturating_sub(1);
|
||||
|
||||
let formatted_num = if chapter_num.starts_with('0') {
|
||||
format!("{:0width$}", prev_num, width = chapter_num.len())
|
||||
} else {
|
||||
prev_num.to_string()
|
||||
};
|
||||
|
||||
return format!("{}{}{}", prefix, CHAPTER_PATTERN, formatted_num);
|
||||
}
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
let prev_chapter_id_from_url = get_previous_chapter_id(chapter_url);
|
||||
let prev_chapter_id = if !prev_chapter_id_from_url.is_empty() {
|
||||
prev_chapter_id_from_url
|
||||
} else {
|
||||
document
|
||||
.select(&prev_chapter_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.map(|href| {
|
||||
href.trim_end_matches('/')
|
||||
.split('/')
|
||||
.filter(|s| !s.is_empty())
|
||||
.next_back()
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
};
|
||||
|
||||
fn get_list_chapter_from_url(chapter_url: &str) -> String {
|
||||
if let Some(pos) = chapter_url.rfind("-chapter-") {
|
||||
chapter_url[..pos].to_string()
|
||||
} else {
|
||||
chapter_url.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
let list_chapter = get_list_chapter_from_url(chapter_url);
|
||||
|
||||
let mut images = Vec::new();
|
||||
let forbidden_images = [
|
||||
"https://flagcdn.com/32x24/jp.png",
|
||||
"https://flagcdn.com/32x24/kr.png",
|
||||
"https://flagcdn.com/32x24/cn.png",
|
||||
"https://www.gstatic.com/firebasejs/ui/2.0.0/images/auth/google.svg",
|
||||
"https://www.gravatar.com/avatar/?d=mp&s=80",
|
||||
"/asset/img/komikuplus2.jpg",
|
||||
"https://komiku.org/asset/img/Loading.gif",
|
||||
];
|
||||
for el in document.select(&image_selector) {
|
||||
if let Some(src) = attr(&el, "src")
|
||||
.or_else(|| attr(&el, "data-src"))
|
||||
.or_else(|| attr(&el, "data-lazy-src"))
|
||||
.or_else(|| {
|
||||
attr(&el, "srcset").and_then(|s| s.split_whitespace().next().map(|s| s.to_string()))
|
||||
})
|
||||
{
|
||||
if !forbidden_images.contains(&src.as_str()) {
|
||||
images.push(src);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ChapterData {
|
||||
title,
|
||||
next_chapter_id,
|
||||
prev_chapter_id,
|
||||
list_chapter,
|
||||
images,
|
||||
})
|
||||
}
|
||||
|
||||
fn clean_text(text: String) -> String {
|
||||
text.replace(['\n', '\t'], " ").trim().to_string()
|
||||
}
|
||||
|
||||
fn extract_value_after_keyword(full_text: &str, keywords: &[&str], default_index: usize) -> String {
|
||||
let lower_text = full_text.to_lowercase();
|
||||
for keyword in keywords {
|
||||
if let Some(pos) = lower_text.find(&format!("{}:", keyword)) {
|
||||
return clean_text(full_text[pos + keyword.len() + 1..].to_string());
|
||||
} else if let Some(pos) = lower_text.find(&format!("{} ", keyword)) {
|
||||
return clean_text(full_text[pos + keyword.len() + 1..].to_string());
|
||||
}
|
||||
}
|
||||
if let Some(colon_pos) = full_text.find(':') {
|
||||
return clean_text(full_text[colon_pos + 1..].to_string());
|
||||
}
|
||||
clean_text(full_text[default_index..].to_string())
|
||||
}
|
||||
|
||||
fn find_table_row_with_text<'a>(
|
||||
info_rows: &[scraper::ElementRef<'a>],
|
||||
text_fragments: &[&str],
|
||||
) -> Option<String> {
|
||||
let lower_text_fragments: Vec<String> =
|
||||
text_fragments.iter().map(|&s| s.to_lowercase()).collect();
|
||||
let td_last_selector = &*TD_LAST_SELECTOR;
|
||||
|
||||
info_rows
|
||||
.iter()
|
||||
.find(|row| {
|
||||
let row_text = text(row).to_lowercase();
|
||||
lower_text_fragments
|
||||
.iter()
|
||||
.any(|fragment| row_text.contains(fragment))
|
||||
})
|
||||
.and_then(|row| {
|
||||
text_from_or(row, td_last_selector, "")
|
||||
.trim()
|
||||
.to_string()
|
||||
.into()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_komik_detail_document(html: &str) -> Result<DetailData, ScrapingError> {
|
||||
let start_time = std::time::Instant::now();
|
||||
info!("Starting to parse komik detail document");
|
||||
|
||||
let document = parse_html(html);
|
||||
|
||||
let title_selector = &*TITLE_SELECTOR;
|
||||
let h1_selector = &*H1_SELECTOR;
|
||||
let title_tag_selector = &*TITLE_TAG_SELECTOR;
|
||||
let info_row_selector = &*INFO_ROW_SELECTOR;
|
||||
let poster_selector = &*POSTER_SELECTOR;
|
||||
let desc_selector = &*DESC_SELECTOR;
|
||||
let chapter_list_selector = &*CHAPTER_LIST_SELECTOR;
|
||||
let date_link_selector = &*DATE_LINK_SELECTOR;
|
||||
let judul2_selector = &*JUDUL2_SELECTOR;
|
||||
let genre_selector = &*GENRE_SELECTOR;
|
||||
let chapter_link_selector = &*CHAPTER_LINK_SELECTOR;
|
||||
|
||||
let title = document
|
||||
.select(&title_selector)
|
||||
.next()
|
||||
.map(|e| {
|
||||
let text = clean_text(text(&e));
|
||||
text.replace("Komik ", "")
|
||||
.replace("Manga ", "")
|
||||
.replace("Manhua ", "")
|
||||
.replace("Manhwa ", "")
|
||||
.trim()
|
||||
.to_string()
|
||||
})
|
||||
.or_else(|| {
|
||||
document
|
||||
.select(&h1_selector)
|
||||
.next()
|
||||
.map(|e| clean_text(text(&e)))
|
||||
})
|
||||
.or_else(|| {
|
||||
document.select(&title_tag_selector).next().map(|e| {
|
||||
let text = clean_text(text(&e));
|
||||
if text.contains("Komik ") {
|
||||
text.replace("Komik ", "").trim().to_string()
|
||||
} else {
|
||||
text.trim().to_string()
|
||||
}
|
||||
})
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let info_rows_vec: Vec<scraper::ElementRef> = document.select(&info_row_selector).collect();
|
||||
let info_rows = &info_rows_vec[..];
|
||||
|
||||
let status = info_rows
|
||||
.iter()
|
||||
.find_map(|&row| {
|
||||
let full_text = text(&row);
|
||||
if full_text.to_lowercase().contains("status") {
|
||||
Some(
|
||||
extract_value_after_keyword(&full_text, &["status"], 0)
|
||||
.replace("Status", "")
|
||||
.replace("Jenis Komik", "")
|
||||
.replace("Type", "")
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let r#type = info_rows
|
||||
.iter()
|
||||
.find_map(|&row| {
|
||||
let full_text = text(&row);
|
||||
if full_text.to_lowercase().contains("jenis komik")
|
||||
|| full_text.to_lowercase().contains("type")
|
||||
{
|
||||
Some(
|
||||
extract_value_after_keyword(&full_text, &["jenis komik", "type"], 0)
|
||||
.replace("Jenis Komik", "")
|
||||
.replace("Type", "")
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let author = info_rows
|
||||
.iter()
|
||||
.find_map(|&row| {
|
||||
let full_text = text(&row);
|
||||
if full_text.to_lowercase().contains("pengarang")
|
||||
|| full_text.to_lowercase().contains("author")
|
||||
|| full_text.to_lowercase().contains("artist")
|
||||
{
|
||||
Some(
|
||||
extract_value_after_keyword(&full_text, &["pengarang", "author", "artist"], 0)
|
||||
.replace("Pengarang", "")
|
||||
.replace("Author", "")
|
||||
.replace("pengarang", "")
|
||||
.replace("author", "")
|
||||
.replace("Artist", "")
|
||||
.replace("artist", "")
|
||||
.trim()
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let poster = document
|
||||
.select(&poster_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "src"))
|
||||
.map(|s| s.split('?').next().unwrap_or(&s).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let description = document
|
||||
.select(&desc_selector)
|
||||
.map(|e| clean_text(text(&e)))
|
||||
.filter(|t| t.len() > 50)
|
||||
.collect::<Vec<String>>()
|
||||
.join("\n")
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
let release_date = find_table_row_with_text(info_rows, &["tanggal rilis", "release date"])
|
||||
.map(clean_text)
|
||||
.unwrap_or_else(|| {
|
||||
document
|
||||
.select(&chapter_list_selector)
|
||||
.next_back()
|
||||
.and_then(|last| last.select(&date_link_selector).next())
|
||||
.map(|e| clean_text(text(&e)))
|
||||
.unwrap_or_default()
|
||||
});
|
||||
|
||||
let total_chapter = find_table_row_with_text(info_rows, &["total chapter", "total chapters"])
|
||||
.unwrap_or_else(|| {
|
||||
let count = document.select(&chapter_list_selector).count();
|
||||
if count > 0 {
|
||||
count.to_string()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
});
|
||||
|
||||
let updated_on = find_table_row_with_text(info_rows, &["diperbarui", "updated"])
|
||||
.or_else(|| {
|
||||
document.select(&judul2_selector).next().map(|e| {
|
||||
let text_str = clean_text(text(&e));
|
||||
text_str.split("• ").nth(1).unwrap_or("").trim().to_string()
|
||||
})
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
document
|
||||
.select(&chapter_list_selector)
|
||||
.next()
|
||||
.and_then(|first| first.select(&date_link_selector).next())
|
||||
.map(|e| clean_text(text(&e)))
|
||||
.unwrap_or_default()
|
||||
});
|
||||
|
||||
let mut genres = Vec::new();
|
||||
for element in document.select(&genre_selector) {
|
||||
let genre = clean_text(text(&element));
|
||||
if !genre.is_empty() {
|
||||
genres.push(genre);
|
||||
}
|
||||
}
|
||||
|
||||
let raw_chapter_data: Vec<(String, String, String)> = document
|
||||
.select(&chapter_list_selector)
|
||||
.filter_map(|el| {
|
||||
let chapter_link_element = el.select(&chapter_link_selector).next();
|
||||
let date_element = el.select(&date_link_selector).next();
|
||||
|
||||
let chapter_text = chapter_link_element
|
||||
.as_ref()
|
||||
.map(|e| clean_text(text(e)))
|
||||
.unwrap_or_default();
|
||||
|
||||
let date_text = date_element
|
||||
.map(|e| clean_text(text(&e)))
|
||||
.unwrap_or_default();
|
||||
|
||||
let href_text = chapter_link_element
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.unwrap_or_default();
|
||||
|
||||
if !chapter_text.is_empty() || !date_text.is_empty() || !href_text.is_empty() {
|
||||
Some((chapter_text, date_text, href_text))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let chapters: Vec<Chapter> = raw_chapter_data
|
||||
.par_iter()
|
||||
.filter_map(|(chapter_text, date_text, href_text)| {
|
||||
let chapter = {
|
||||
let trimmed_chapter_text = chapter_text.trim();
|
||||
if let Some(captures) = CHAPTER_TITLE_REGEX.captures(trimmed_chapter_text) {
|
||||
captures
|
||||
.get(1)
|
||||
.map_or(trimmed_chapter_text.to_string(), |m| m.as_str().to_string())
|
||||
} else if let Some(captures) = CHAPTER_NUMBER_REGEX.captures(trimmed_chapter_text) {
|
||||
captures
|
||||
.get(1)
|
||||
.map_or(trimmed_chapter_text.to_string(), |m| m.as_str().to_string())
|
||||
} else {
|
||||
trimmed_chapter_text.to_string()
|
||||
}
|
||||
};
|
||||
|
||||
let date = date_text.trim().to_string();
|
||||
|
||||
let chapter_id = href_text
|
||||
.split('/')
|
||||
.filter(|s| !s.is_empty())
|
||||
.next_back()
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !chapter_id.is_empty() {
|
||||
Some(Chapter {
|
||||
chapter,
|
||||
date,
|
||||
chapter_id,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let duration = start_time.elapsed();
|
||||
info!("Parsed komik detail document in {:?}", duration);
|
||||
|
||||
Ok(DetailData {
|
||||
title,
|
||||
poster,
|
||||
description,
|
||||
status,
|
||||
r#type,
|
||||
release_date,
|
||||
author,
|
||||
total_chapter,
|
||||
updated_on,
|
||||
genres,
|
||||
chapters,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_genre_page(
|
||||
html: &str,
|
||||
current_page: u32,
|
||||
) -> Result<(Vec<KomikItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut komik_list = Vec::new();
|
||||
|
||||
let item_selector = selector(".bge, article, .ls4, .ls2")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let title_selector = selector(".kan h3, h3 a, h4 a")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let img_selector = selector(".bgei img, img.lazy, img")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let chapter_selector = selector(".new1:last-of-type a span:last-child, .new1 a:last-child span:last-child, .ls4s a, .ls24, .ls2l a")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let score_selector = selector(".up, .numscore, .epx")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let type_selector = selector(".tpe1_inf, .ls3p, .type")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let link_selector = selector(".kan h3 a, .bgei a, h3 a, h4 a, a")
|
||||
.ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let next_selector =
|
||||
selector("span[hx-get]").ok_or(ScrapingError::Parse("Selector error".to_string()))?;
|
||||
let slug_regex = Regex::new(r"/([^/]+)/?$").unwrap();
|
||||
|
||||
for element in document.select(&item_selector) {
|
||||
let title = text_from_or(&element, &title_selector, "");
|
||||
|
||||
let poster = element
|
||||
.select(&img_selector)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "data-src").or(attr(&e, "src")))
|
||||
.unwrap_or_else(|| "".to_string())
|
||||
.to_string();
|
||||
|
||||
let chapter = text_from_or(&element, &chapter_selector, "N/A");
|
||||
|
||||
let score = text_from_or(&element, &score_selector, "N/A");
|
||||
|
||||
let komik_type = text_from_or(&element, &type_selector, "Unknown")
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.unwrap_or("Unknown")
|
||||
.to_string();
|
||||
|
||||
let komik_url = attr_from_or(&element, &link_selector, "href", "");
|
||||
|
||||
let slug = slug_regex
|
||||
.captures(&komik_url)
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if !title.is_empty() {
|
||||
komik_list.push(KomikItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
chapter,
|
||||
score,
|
||||
r#type: komik_type,
|
||||
komik_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let has_next_page = document.select(&next_selector).next().is_some();
|
||||
let last_visible_page = if has_next_page {
|
||||
current_page + 1
|
||||
} else {
|
||||
current_page
|
||||
};
|
||||
let pagination = Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page: if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
has_previous_page: current_page > 1,
|
||||
previous_page: if current_page > 1 {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
Ok((komik_list, pagination))
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod alqanime_parser;
|
||||
pub mod komik_parser;
|
||||
pub mod otakudesu_parser;
|
||||
@@ -0,0 +1,621 @@
|
||||
//! Otakudesu HTML parser — native implementation using infrastructure utilities.
|
||||
//!
|
||||
//! Parses Otakudesu HTML pages into domain types. All parsing runs in
|
||||
//! `spawn_blocking` (called from the repository layer).
|
||||
|
||||
use crate::domain::entity::anime::*;
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::infrastructure::scraping::parsing_utils::{
|
||||
attr, attr_from, attr_from_or, extract_slug, parse_html, selector, text, text_from_or,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// INDEX (Ongoing + Complete Anime)
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_ongoing_anime(html: &str) -> Result<Vec<OngoingAnimeItem>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut items = Vec::new();
|
||||
|
||||
let venz_sel = selector(".venz ul li")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
|
||||
let title_sel = selector(".thumbz h2.jdlflm")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let img_sel = selector("img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
|
||||
let ep_sel = selector(".epz")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
|
||||
|
||||
for element in document.select(&venz_sel) {
|
||||
let title = text_from_or(&element, &title_sel, "");
|
||||
let href = attr_from(&element, &link_sel, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&href);
|
||||
let poster = attr_from_or(&element, &img_sel, "src", "");
|
||||
let current_episode = text_from_or(&element, &ep_sel, "N/A");
|
||||
let anime_url = attr_from_or(&element, &link_sel, "href", "");
|
||||
|
||||
if !title.is_empty() {
|
||||
items.push(OngoingAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
current_episode,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
pub fn parse_complete_anime(html: &str) -> Result<Vec<CompleteAnimeItem>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut items = Vec::new();
|
||||
|
||||
let venz_sel = selector(".venz ul li")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
|
||||
let title_sel = selector(".thumbz h2.jdlflm")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let img_sel = selector("img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
|
||||
let ep_sel = selector(".epz")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
|
||||
|
||||
for element in document.select(&venz_sel) {
|
||||
let title = text_from_or(&element, &title_sel, "");
|
||||
let href = attr_from(&element, &link_sel, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&href);
|
||||
let poster = attr_from_or(&element, &img_sel, "src", "");
|
||||
let episode_count = text_from_or(&element, &ep_sel, "N/A");
|
||||
let anime_url = attr_from_or(&element, &link_sel, "href", "");
|
||||
|
||||
if !title.is_empty() {
|
||||
items.push(CompleteAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode_count,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GENRES
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_genres(html: &str) -> Result<Vec<Genre>, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut genres = Vec::new();
|
||||
let genre_sel = selector(".genres li a, .genre-list a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse genre selector".into()))?;
|
||||
|
||||
for element in document.select(&genre_sel) {
|
||||
let name = text(&element);
|
||||
let url = attr(&element, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&url);
|
||||
|
||||
if !name.is_empty() && !slug.is_empty() {
|
||||
genres.push(Genre { name, slug, url });
|
||||
}
|
||||
}
|
||||
Ok(genres)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// DETAIL
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_anime_detail_document(html: &str) -> Result<AnimeDetailData, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
|
||||
let info_sel = selector(".infozingle p")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse info selector".into()))?;
|
||||
let poster_sel = selector(".fotoanime img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse poster selector".into()))?;
|
||||
let synopsis_sel = selector(".sinopc")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse synopsis selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let ep_list_sel = selector(".episodelist ul li a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse episode list selector".into()))?;
|
||||
let rec_sel = selector("#recommend-anime-series .isi-anime")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse recommendation selector".into()))?;
|
||||
let rec_title_sel = selector(".judul-anime a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse rec title selector".into()))?;
|
||||
let rec_img_sel = selector("img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse rec img selector".into()))?;
|
||||
|
||||
let mut title = String::new();
|
||||
let mut alternative_title = String::new();
|
||||
let mut r#type: Option<String> = None;
|
||||
let mut status: Option<String> = None;
|
||||
let mut release_date = String::new();
|
||||
let mut studio = String::new();
|
||||
|
||||
for element in document.select(&info_sel) {
|
||||
let text = text(&element);
|
||||
if text.contains("Judul:") {
|
||||
title = text.replace("Judul:", "").trim().to_string();
|
||||
} else if text.contains("Japanese:") {
|
||||
alternative_title = text.replace("Japanese:", "").trim().to_string();
|
||||
} else if text.contains("Type:") {
|
||||
let type_str = text.replace("Type:", "").trim().to_string();
|
||||
if !type_str.is_empty() {
|
||||
r#type = Some(type_str);
|
||||
}
|
||||
} else if text.contains("Status:") {
|
||||
let status_str = text.replace("Status:", "").trim().to_string();
|
||||
if !status_str.is_empty() {
|
||||
status = Some(status_str);
|
||||
}
|
||||
} else if text.contains("Tanggal Rilis:") {
|
||||
release_date = text.replace("Tanggal Rilis:", "").trim().to_string();
|
||||
} else if text.contains("Studio:") {
|
||||
studio = text.replace("Studio:", "").trim().to_string();
|
||||
}
|
||||
}
|
||||
|
||||
let poster = document
|
||||
.select(&poster_sel)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("src"))
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let synopsis = text_from_or(&document.root_element(), &synopsis_sel, "");
|
||||
|
||||
let mut genres = Vec::new();
|
||||
if let Some(genres_element) = document
|
||||
.select(&info_sel)
|
||||
.find(|e| text(e).contains("Genres:"))
|
||||
{
|
||||
for genre_link in genres_element.select(&link_sel) {
|
||||
let gname = text(&genre_link);
|
||||
let anine_url = attr(&genre_link, "href").unwrap_or_default();
|
||||
let genre_slug = extract_slug(&anine_url);
|
||||
genres.push(DetailGenre {
|
||||
name: gname,
|
||||
slug: genre_slug,
|
||||
anime_url: anine_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let mut episode_lists = Vec::new();
|
||||
for element in document.select(&ep_list_sel) {
|
||||
let episode = text(&element);
|
||||
let href = attr(&element, "href").unwrap_or_default();
|
||||
let slug = extract_slug(&href);
|
||||
episode_lists.push(EpisodeList { episode, slug });
|
||||
}
|
||||
|
||||
let mut recommendations = Vec::new();
|
||||
for element in document.select(&rec_sel) {
|
||||
let rtitle = text_from_or(&element, &rec_title_sel, "");
|
||||
let rposter = attr_from_or(&element, &rec_img_sel, "src", "");
|
||||
let rhref = element
|
||||
.select(&link_sel)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr("href"))
|
||||
.unwrap_or("");
|
||||
|
||||
let rslug = extract_slug(rhref);
|
||||
|
||||
recommendations.push(Recommendation {
|
||||
title: rtitle,
|
||||
slug: rslug,
|
||||
poster: rposter,
|
||||
status: None,
|
||||
r#type: None,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(AnimeDetailData {
|
||||
title,
|
||||
alternative_title,
|
||||
poster,
|
||||
r#type,
|
||||
status,
|
||||
release_date,
|
||||
studio,
|
||||
genres,
|
||||
synopsis,
|
||||
episode_lists,
|
||||
batch: vec![],
|
||||
producers: vec![],
|
||||
recommendations,
|
||||
})
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// PAGINATION HELPER
|
||||
// ============================================================================
|
||||
|
||||
fn parse_pagination(slug: &str, document: &scraper::Html) -> Result<Pagination, ScrapingError> {
|
||||
let pagination_sel = selector(".pagenavix .page-numbers:not(.next)")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse pagination selector".into()))?;
|
||||
let next_sel = selector(".pagenavix .next.page-numbers")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse next selector".into()))?;
|
||||
|
||||
let current_page = slug.parse::<u32>().unwrap_or(1);
|
||||
let last_visible_page = document
|
||||
.select(&pagination_sel)
|
||||
.next_back()
|
||||
.and_then(|e| e.text().collect::<String>().trim().parse::<u32>().ok())
|
||||
.unwrap_or(1);
|
||||
|
||||
let has_next_page = document.select(&next_sel).next().is_some();
|
||||
let next_page = if has_next_page {
|
||||
Some(current_page + 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let has_previous_page = current_page > 1;
|
||||
let previous_page = if has_previous_page {
|
||||
Some(current_page - 1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Pagination {
|
||||
current_page,
|
||||
last_visible_page,
|
||||
has_next_page,
|
||||
next_page,
|
||||
has_previous_page,
|
||||
previous_page,
|
||||
})
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// COMPLETE ANIME PAGE
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_anime_page(
|
||||
html: &str,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<CompleteAnimeListItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut items = Vec::new();
|
||||
|
||||
let venz_sel = selector(".venz ul li")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
|
||||
let title_sel = selector(".thumbz h2.jdlflm")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let img_sel = selector("img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
|
||||
let ep_sel = selector(".epz")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
|
||||
|
||||
for element in document.select(&venz_sel) {
|
||||
let title = text_from_or(&element, &title_sel, "");
|
||||
let anime_url = attr_from_or(&element, &link_sel, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_sel, "src", "");
|
||||
let episode_count = text_from_or(&element, &ep_sel, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
items.push(CompleteAnimeListItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode_count,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let pagination = parse_pagination(slug, &document)?;
|
||||
Ok((items, pagination))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ONGOING ANIME PAGE
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_ongoing_anime_document(
|
||||
html: &str,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<OngoingAnimeListItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut items = Vec::new();
|
||||
|
||||
let venz_sel = selector(".venz ul li")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
|
||||
let title_sel = selector(".thumbz h2.jdlflm")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let img_sel = selector("img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
|
||||
let score_sel = selector(".epz")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
|
||||
|
||||
for element in document.select(&venz_sel) {
|
||||
let title = text_from_or(&element, &title_sel, "");
|
||||
let anime_url = attr_from_or(&element, &link_sel, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_sel, "src", "");
|
||||
let score = text_from_or(&element, &score_sel, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
items.push(OngoingAnimeListItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
score,
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let pagination = parse_pagination(slug, &document)?;
|
||||
Ok((items, pagination))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// LATEST ANIME PAGE
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_latest_anime_document(
|
||||
html: &str,
|
||||
slug: &str,
|
||||
) -> Result<(Vec<LatestAnimeItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut items = Vec::new();
|
||||
|
||||
let venz_sel = selector(".venz ul li")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
|
||||
let title_sel = selector(".thumbz h2.jdlflm")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let img_sel = selector("img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
|
||||
let ep_sel = selector(".epz")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
|
||||
|
||||
for element in document.select(&venz_sel) {
|
||||
let title = text_from_or(&element, &title_sel, "");
|
||||
let anime_url = attr_from_or(&element, &link_sel, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_sel, "src", "");
|
||||
let episode = text_from_or(&element, &ep_sel, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
items.push(LatestAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode,
|
||||
score: String::new(),
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let pagination = parse_pagination(slug, &document)?;
|
||||
Ok((items, pagination))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SEARCH ANIME PAGE
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_search_anime_document(
|
||||
html: &str,
|
||||
page: &str,
|
||||
) -> Result<(Vec<SearchAnimeItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut items = Vec::new();
|
||||
|
||||
let venz_sel = selector(".venz ul li")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
|
||||
let title_sel = selector(".thumbz h2.jdlflm")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let img_sel = selector("img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
|
||||
let ep_sel = selector(".epz")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
|
||||
let genre_sel = selector(".genre-tag")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse genre-tag selector".into()))?;
|
||||
let status_sel = selector(".status")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse status selector".into()))?;
|
||||
let rating_sel = selector(".rating")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse rating selector".into()))?;
|
||||
|
||||
for element in document.select(&venz_sel) {
|
||||
let title = text_from_or(&element, &title_sel, "");
|
||||
let anime_url = attr_from_or(&element, &link_sel, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_sel, "src", "");
|
||||
let episode = text_from_or(&element, &ep_sel, "N/A");
|
||||
|
||||
let mut genres = Vec::new();
|
||||
for genre_elem in element.select(&genre_sel) {
|
||||
genres.push(text(&genre_elem));
|
||||
}
|
||||
|
||||
let status = text_from_or(&element, &status_sel, "");
|
||||
let rating = text_from_or(&element, &rating_sel, "");
|
||||
|
||||
if !title.is_empty() {
|
||||
items.push(SearchAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode,
|
||||
anime_url,
|
||||
genres,
|
||||
status,
|
||||
rating,
|
||||
description: String::new(),
|
||||
r#type: String::new(),
|
||||
season: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let pagination = parse_pagination(page, &document)?;
|
||||
Ok((items, pagination))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GENRE ANIME PAGE
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_genre_anime_document(
|
||||
html: &str,
|
||||
page: &str,
|
||||
) -> Result<(Vec<GenreAnimeItem>, Pagination), ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
let mut items = Vec::new();
|
||||
|
||||
let venz_sel = selector(".venz ul li")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse .venz ul li selector".into()))?;
|
||||
let title_sel = selector(".thumbz h2.jdlflm")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse title selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let img_sel = selector("img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse img selector".into()))?;
|
||||
let ep_sel = selector(".epz")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse epz selector".into()))?;
|
||||
|
||||
for element in document.select(&venz_sel) {
|
||||
let title = text_from_or(&element, &title_sel, "");
|
||||
let anime_url = attr_from_or(&element, &link_sel, "href", "");
|
||||
let slug = extract_slug(&anime_url);
|
||||
let poster = attr_from_or(&element, &img_sel, "src", "");
|
||||
let episode = text_from_or(&element, &ep_sel, "N/A");
|
||||
|
||||
if !title.is_empty() {
|
||||
items.push(GenreAnimeItem {
|
||||
title,
|
||||
slug,
|
||||
poster,
|
||||
episode,
|
||||
score: String::new(),
|
||||
status: String::new(),
|
||||
anime_url,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let pagination = parse_pagination(page, &document)?;
|
||||
Ok((items, pagination))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// FULL EPISODE PAGE
|
||||
// ============================================================================
|
||||
|
||||
pub fn parse_anime_full_document(html: &str, slug: &str) -> Result<AnimeFullData, ScrapingError> {
|
||||
let document = parse_html(html);
|
||||
|
||||
let ep_title_sel = selector("h1.posttl")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse h1.posttl selector".into()))?;
|
||||
let img_sel = selector(".cukder img")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse .cukder img selector".into()))?;
|
||||
let stream_sel = selector("#embed_holder iframe")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse embed_holder selector".into()))?;
|
||||
let dl_item_sel = selector(".download ul li")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse download selector".into()))?;
|
||||
let res_sel = selector("strong")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse strong selector".into()))?;
|
||||
let link_sel = selector("a")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse link selector".into()))?;
|
||||
let next_ep_sel = selector(".flir a[title*='Episode Selanjutnya']")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse next episode selector".into()))?;
|
||||
let prev_ep_sel = selector(".flir a[title*='Episode Sebelumnya']")
|
||||
.ok_or_else(|| ScrapingError::Parse("Failed to parse prev episode selector".into()))?;
|
||||
|
||||
let episode = document
|
||||
.select(&ep_title_sel)
|
||||
.next()
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let episode_number = episode
|
||||
.split("Episode")
|
||||
.nth(1)
|
||||
.map(|s| s.trim().to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let image_url = document
|
||||
.select(&img_sel)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "src"))
|
||||
.unwrap_or_default();
|
||||
|
||||
let stream_url = document
|
||||
.select(&stream_sel)
|
||||
.next()
|
||||
.and_then(|e| attr(&e, "src"))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut download_urls = std::collections::HashMap::new();
|
||||
|
||||
for element in document.select(&dl_item_sel) {
|
||||
let resolution = element
|
||||
.select(&res_sel)
|
||||
.next()
|
||||
.map(|e| text(&e))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut links = Vec::new();
|
||||
for link_element in element.select(&link_sel) {
|
||||
let server = text(&link_element);
|
||||
let url = attr(&link_element, "href").unwrap_or_default();
|
||||
links.push(DownloadLink { server, url });
|
||||
}
|
||||
|
||||
if !resolution.is_empty() && !links.is_empty() {
|
||||
download_urls.insert(resolution, links);
|
||||
}
|
||||
}
|
||||
|
||||
let next_episode_element = document.select(&next_ep_sel).next();
|
||||
let previous_episode_element = document.select(&prev_ep_sel).next();
|
||||
|
||||
let next_episode_slug = next_episode_element
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.and_then(|href| {
|
||||
href.split('/')
|
||||
.nth(href.split('/').count().saturating_sub(2))
|
||||
.map(|s| s.to_string() + "/")
|
||||
});
|
||||
|
||||
let previous_episode_slug = previous_episode_element
|
||||
.and_then(|e| attr(&e, "href"))
|
||||
.and_then(|href| {
|
||||
href.split('/')
|
||||
.nth(href.split('/').count().saturating_sub(2))
|
||||
.map(|s| s.to_string() + "/")
|
||||
});
|
||||
|
||||
Ok(AnimeFullData {
|
||||
episode,
|
||||
episode_number,
|
||||
anime: AnimeInfo {
|
||||
slug: slug.to_string(),
|
||||
},
|
||||
has_next_episode: next_episode_slug.is_some(),
|
||||
next_episode: next_episode_slug.map(|s| EpisodeInfo { slug: s }),
|
||||
has_previous_episode: previous_episode_slug.is_some(),
|
||||
previous_episode: previous_episode_slug.map(|s| EpisodeInfo { slug: s }),
|
||||
stream_url,
|
||||
download_urls,
|
||||
image_url,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
//! Proxy fetching repository.
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::domain::repository::ScrapingRepository;
|
||||
use crate::infrastructure::scraping::proxy_fetch::{self, FetchResult};
|
||||
|
||||
pub struct ProxyRepository;
|
||||
|
||||
impl ProxyRepository {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
pub async fn fetch_with_proxy_url(&self, url: &str) -> Result<FetchResult, ScrapingError> {
|
||||
proxy_fetch::fetch_with_proxy(url)
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("Proxy fetch failed: {}", e)))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ScrapingRepository for ProxyRepository {
|
||||
async fn fetch_html(&self, url: &str) -> Result<String, ScrapingError> {
|
||||
self.fetch_with_proxy_url(url).await.map(|r| r.data)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user