feat: full clean architecture refactor — domain → application → infrastructure → presentation
- Restructure from Modular MVC to Clean Architecture with 4 strict layers - Domain: entities (anime/komik/proxy), Repository traits, typed errors - Application: use case classes with proper error propagation - Infrastructure: repository impls, parsers, Redis cache, HTTP/scraping, browser - Presentation: Axum handlers, DTOs, AppState, router, AppError+IntoResponse - Migrate all parsers (otakudesu, alqanime, komik) to native infra implementations - Replace once_cell::sync::Lazy/OnceCell with std::sync::LazyLock/OnceLock - Remove 150+ old files in modules/ and shared/ directories - Remove once_cell from Cargo.toml dependencies - Fix test/debug binaries to use new import paths - Zero new clippy warnings Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
asepharyana
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
80c96eaa42
commit
776fa828d8
@@ -0,0 +1,11 @@
|
||||
//! HTML scraping helpers — re-exports from retry and parsing_utils modules.
|
||||
//!
|
||||
//! This module consolidates common scraping utilities for convenient single-path imports.
|
||||
//! Downstream code should use `crate::infrastructure::scraping::html_fetcher::*`.
|
||||
|
||||
// Re-export retry utilities
|
||||
pub use crate::infrastructure::scraping::retry::{
|
||||
custom_backoff, default_backoff, permanent, quick_backoff, retry, slow_backoff, transient,
|
||||
};
|
||||
// Re-export common scraping helpers (fetch_html_with_retry, parse_html, selector, text, attr, etc.)
|
||||
pub use crate::infrastructure::scraping::parsing_utils::*;
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod html_fetcher;
|
||||
pub mod parsing_utils;
|
||||
pub mod proxy_fetch;
|
||||
pub mod retry;
|
||||
pub mod scraping_urls;
|
||||
@@ -0,0 +1,193 @@
|
||||
//! HTML scraping helpers using scraper crate.
|
||||
//!
|
||||
//! Adapted from shared/utils/web/scraping.rs with domain-level errors.
|
||||
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::infrastructure::scraping::proxy_fetch::fetch_with_proxy;
|
||||
use crate::infrastructure::scraping::retry::{default_backoff, transient};
|
||||
use backoff::future::retry;
|
||||
use regex::Regex;
|
||||
use scraper::{ElementRef, Html, Selector};
|
||||
use std::sync::LazyLock;
|
||||
use tracing::{info, warn};
|
||||
|
||||
/// Fetch HTML from URL with retry backoff and proxy support.
|
||||
pub async fn fetch_html_with_retry(url: &str) -> Result<String, ScrapingError> {
|
||||
let backoff = default_backoff();
|
||||
let fetch_operation = || async {
|
||||
info!("Fetching: {}", url);
|
||||
match fetch_with_proxy(url).await {
|
||||
Ok(response) => {
|
||||
info!("Successfully fetched: {}", url);
|
||||
Ok(response.data)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to fetch: {}, error: {:?}", url, e);
|
||||
Err(transient(e))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
retry(backoff, fetch_operation)
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(e.to_string()))
|
||||
}
|
||||
|
||||
/// Parse HTML string into a document.
|
||||
pub fn parse_html(html: &str) -> Html {
|
||||
Html::parse_document(html)
|
||||
}
|
||||
|
||||
/// Safely create a CSS selector.
|
||||
pub fn selector(css: &str) -> Option<Selector> {
|
||||
Selector::parse(css).ok()
|
||||
}
|
||||
|
||||
/// Extract text content from an element, trimmed.
|
||||
pub fn text(element: &ElementRef) -> String {
|
||||
element.text().collect::<String>().trim().to_string()
|
||||
}
|
||||
|
||||
/// Extract text from first matching element.
|
||||
pub fn select_text(element: &ElementRef, css: &str) -> Option<String> {
|
||||
let sel = selector(css)?;
|
||||
element.select(&sel).next().map(|e| text(&e))
|
||||
}
|
||||
|
||||
/// Extract text from first matching element using a pre-compiled selector.
|
||||
pub fn text_from(element: &ElementRef, selector: &Selector) -> Option<String> {
|
||||
element.select(selector).next().map(|e| text(&e))
|
||||
}
|
||||
|
||||
/// Extract text from first matching element using a pre-compiled selector, or return default.
|
||||
pub fn text_from_or(element: &ElementRef, selector: &Selector, default: &str) -> String {
|
||||
text_from(element, selector).unwrap_or_else(|| default.to_string())
|
||||
}
|
||||
|
||||
/// Extract attribute from first matching element.
|
||||
pub fn select_attr(element: &ElementRef, css: &str, attr: &str) -> Option<String> {
|
||||
let sel = selector(css)?;
|
||||
element
|
||||
.select(&sel)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr(attr))
|
||||
.map(String::from)
|
||||
}
|
||||
|
||||
/// Extract attribute from first matching element using a pre-compiled selector.
|
||||
pub fn attr_from(element: &ElementRef, selector: &Selector, attr: &str) -> Option<String> {
|
||||
element
|
||||
.select(selector)
|
||||
.next()
|
||||
.and_then(|e| e.value().attr(attr))
|
||||
.map(String::from)
|
||||
}
|
||||
|
||||
/// Extract attribute from first matching element using a pre-compiled selector, or return default.
|
||||
pub fn attr_from_or(
|
||||
element: &ElementRef,
|
||||
selector: &Selector,
|
||||
attr: &str,
|
||||
default: &str,
|
||||
) -> String {
|
||||
attr_from(element, selector, attr).unwrap_or_else(|| default.to_string())
|
||||
}
|
||||
|
||||
/// Extract attribute from element.
|
||||
pub fn attr(element: &ElementRef, name: &str) -> Option<String> {
|
||||
element.value().attr(name).map(String::from)
|
||||
}
|
||||
|
||||
/// Select all matching elements.
|
||||
pub fn select_all<'a>(document: &'a Html, css: &str) -> Vec<ElementRef<'a>> {
|
||||
selector(css)
|
||||
.map(|s| document.select(&s).collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Extract slug from URL (last path segment).
|
||||
pub fn extract_slug(url: &str) -> String {
|
||||
static SLUG_REGEX: LazyLock<Result<Regex, regex::Error>> =
|
||||
LazyLock::new(|| Regex::new(r"/([^/]+)/?$"));
|
||||
|
||||
SLUG_REGEX
|
||||
.as_ref()
|
||||
.ok()
|
||||
.and_then(|r| r.captures(url))
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str().to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Remove HTML tags from string.
|
||||
pub fn strip_tags(html: &str) -> String {
|
||||
static TAG_REGEX: LazyLock<Result<Regex, regex::Error>> =
|
||||
LazyLock::new(|| Regex::new(r"<[^>]+>"));
|
||||
TAG_REGEX
|
||||
.as_ref()
|
||||
.map(|r| r.replace_all(html, "").trim().to_string())
|
||||
.unwrap_or_else(|_| html.to_string())
|
||||
}
|
||||
|
||||
/// Extract number from text.
|
||||
pub fn extract_number(text: &str) -> Option<i64> {
|
||||
static NUM_REGEX: LazyLock<Result<Regex, regex::Error>> = LazyLock::new(|| Regex::new(r"\d+"));
|
||||
NUM_REGEX
|
||||
.as_ref()
|
||||
.ok()
|
||||
.and_then(|r| r.find(text))
|
||||
.and_then(|m| m.as_str().parse().ok())
|
||||
}
|
||||
|
||||
/// Extract text inside parentheses.
|
||||
pub fn extract_parentheses(text: &str) -> Option<String> {
|
||||
static PAREN_REGEX: LazyLock<Result<Regex, regex::Error>> =
|
||||
LazyLock::new(|| Regex::new(r"\(([^)]+)\)"));
|
||||
PAREN_REGEX
|
||||
.as_ref()
|
||||
.ok()
|
||||
.and_then(|r| r.captures(text))
|
||||
.and_then(|cap| cap.get(1))
|
||||
.map(|m| m.as_str().to_string())
|
||||
}
|
||||
|
||||
/// Builder for scraping elements.
|
||||
pub struct Scraper<'a> {
|
||||
element: ElementRef<'a>,
|
||||
}
|
||||
|
||||
impl<'a> Scraper<'a> {
|
||||
pub fn new(element: ElementRef<'a>) -> Self {
|
||||
Self { element }
|
||||
}
|
||||
|
||||
/// Get text from selector.
|
||||
pub fn text(&self, css: &str) -> Option<String> {
|
||||
select_text(&self.element, css)
|
||||
}
|
||||
|
||||
/// Get text or default.
|
||||
pub fn text_or(&self, css: &str, default: &str) -> String {
|
||||
self.text(css).unwrap_or_else(|| default.to_string())
|
||||
}
|
||||
|
||||
/// Get attribute from selector.
|
||||
pub fn attr(&self, css: &str, name: &str) -> Option<String> {
|
||||
select_attr(&self.element, css, name)
|
||||
}
|
||||
|
||||
/// Get attribute or default.
|
||||
pub fn attr_or(&self, css: &str, name: &str, default: &str) -> String {
|
||||
self.attr(css, name).unwrap_or_else(|| default.to_string())
|
||||
}
|
||||
|
||||
/// Get href from first link.
|
||||
pub fn href(&self, css: &str) -> Option<String> {
|
||||
self.attr(css, "href")
|
||||
}
|
||||
|
||||
/// Get src from first image.
|
||||
pub fn src(&self, css: &str) -> Option<String> {
|
||||
self.attr(css, "src")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
// Proxy fetch logic with Redis cache AND Request Coalescing (SingleFlight)
|
||||
// Updated for sync Redis API, reqwest API changes, and concurrency optimization.
|
||||
|
||||
use dashmap::DashMap;
|
||||
use redis::AsyncCommands;
|
||||
use std::sync::LazyLock;
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::infrastructure::cache::redis_pool::get_redis_conn;
|
||||
use crate::infrastructure::utils::cache_ttl::CACHE_TTL_VERY_SHORT;
|
||||
use crate::infrastructure::utils::http::common_headers;
|
||||
use crate::infrastructure::utils::http::is_internet_baik_block_page;
|
||||
use crate::infrastructure::utils::http_client::http_client;
|
||||
use crate::presentation::error::AppError;
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct FetchResult {
|
||||
pub data: String,
|
||||
pub content_type: Option<String>,
|
||||
}
|
||||
|
||||
// Implement Display to allow .to_string()
|
||||
impl std::fmt::Display for FetchResult {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"FetchResult {{ data: {}, content_type: {} }}",
|
||||
self.data,
|
||||
self.content_type.as_deref().unwrap_or("None")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Global In-Flight Request Map for Request Coalescing
|
||||
// Maps URL slug -> Broadcast Sender
|
||||
static IN_FLIGHT: LazyLock<DashMap<String, broadcast::Sender<Result<FetchResult, String>>>> =
|
||||
LazyLock::new(DashMap::new);
|
||||
|
||||
// Global Blacklist for domains that consistently fail direct fetch (Timeouts, SSL, Cloudflare blocks)
|
||||
static FAILED_DOMAINS: LazyLock<dashmap::DashSet<String>> = LazyLock::new(dashmap::DashSet::new);
|
||||
|
||||
const RELAY_ENDPOINTS: &[&str] = &[
|
||||
"https://opennext-app.superaseph.workers.dev",
|
||||
"https://proxy-bun.vercel.app",
|
||||
"https://proxy-bun-mytheclipse8647-orfq73fe.apn.leapcell.dev",
|
||||
];
|
||||
|
||||
// --- REDIS CACHE WRAPPER START ---
|
||||
fn get_fetch_cache_key(slug: &str) -> String {
|
||||
format!("fetch:proxy:{slug}")
|
||||
}
|
||||
|
||||
async fn get_cached_fetch(slug: &str) -> Result<Option<FetchResult>, AppError> {
|
||||
let mut conn = get_redis_conn().await?;
|
||||
let key = get_fetch_cache_key(slug);
|
||||
|
||||
let cached: Option<String> = conn.get(&key).await?;
|
||||
|
||||
if let Some(cached_str) = cached {
|
||||
match serde_json::from_str::<FetchResult>(&cached_str) {
|
||||
Ok(parsed) => {
|
||||
debug!("[fetchWithProxy] Returning cached response for {}", slug);
|
||||
Ok(Some(parsed))
|
||||
}
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
async fn set_cached_fetch(slug: &str, value: &FetchResult) -> Result<(), AppError> {
|
||||
let mut conn = get_redis_conn().await?;
|
||||
let key = get_fetch_cache_key(slug);
|
||||
let json_string = serde_json::to_string(value)?;
|
||||
|
||||
// Use standardized TTL
|
||||
conn.set_ex::<_, _, ()>(&key, &json_string, CACHE_TTL_VERY_SHORT)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
// --- REDIS CACHE WRAPPER END ---
|
||||
|
||||
/// Main entry point: Fetches with proxy, using Cache and Request Coalescing
|
||||
pub async fn fetch_with_proxy(slug: &str) -> Result<FetchResult, AppError> {
|
||||
// 1. Try Cache First
|
||||
if let Ok(Some(cached)) = get_cached_fetch(slug).await {
|
||||
return Ok(cached);
|
||||
}
|
||||
|
||||
// 2. Request Coalescing (SingleFlight)
|
||||
// Check if there is already an in-flight request for this slug
|
||||
let tx = {
|
||||
if let Some(in_flight) = IN_FLIGHT.get(slug) {
|
||||
debug!("[Coalesce] Joining in-flight request for {}", slug);
|
||||
in_flight.value().clone()
|
||||
} else {
|
||||
// No in-flight request, create a new channel
|
||||
let (tx, _) = broadcast::channel(1); // Capacity 1 is enough for single result
|
||||
IN_FLIGHT.insert(slug.to_string(), tx.clone());
|
||||
debug!("[Coalesce] Starting leader request for {}", slug);
|
||||
|
||||
// We are the leader, we must execute the fetch
|
||||
// Spawn the fetch task so we don't block holding the map lock (though insert is fast)
|
||||
// But actually we are not holding the lock here anymore.
|
||||
|
||||
// Clone for the async block
|
||||
let slug_clone = slug.to_string();
|
||||
let tx_clone = tx.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
// RAII Guard: Guarantee slug eviction exactly once the task finishes or panics!
|
||||
struct DropGuard(String);
|
||||
impl Drop for DropGuard {
|
||||
fn drop(&mut self) {
|
||||
IN_FLIGHT.remove(&self.0);
|
||||
}
|
||||
}
|
||||
let _guard = DropGuard(slug_clone.clone());
|
||||
|
||||
let result = perform_fetch(&slug_clone).await;
|
||||
|
||||
// Map AppError to String for broadcast (since AppError might not be Clone)
|
||||
// FetchResult is Clone.
|
||||
let broadcast_result = match &result {
|
||||
Ok(res) => Ok(res.clone()),
|
||||
Err(e) => Err(e.to_string()),
|
||||
};
|
||||
|
||||
// Broadcast result to all waiting subscribers
|
||||
let _ = tx_clone.send(broadcast_result);
|
||||
});
|
||||
|
||||
tx
|
||||
}
|
||||
};
|
||||
|
||||
// 3. Wait for result (Leader or Follower)
|
||||
let mut rx = tx.subscribe();
|
||||
match rx.recv().await {
|
||||
Ok(Ok(res)) => Ok(res),
|
||||
Ok(Err(e_str)) => Err(AppError::Internal(e_str)),
|
||||
Err(e) => {
|
||||
warn!("[Coalesce] Receive mismatch for {}: {:?}", slug, e);
|
||||
Err(AppError::Internal("Request coalescing error".to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The actual fetch logic (Direct -> Retry -> Proxy)
|
||||
async fn perform_fetch(slug: &str) -> Result<FetchResult, AppError> {
|
||||
// 1. Extract domain for Circuit Breaker logic
|
||||
let domain = reqwest::Url::parse(slug)
|
||||
.ok()
|
||||
.and_then(|u| u.host_str().map(|s| s.to_string()))
|
||||
.unwrap_or_default();
|
||||
|
||||
// 2. Immediate proxy fallback if domain has a known history of brutal timeouts/blocks
|
||||
if !domain.is_empty() && FAILED_DOMAINS.contains(&domain) {
|
||||
warn!(
|
||||
"[Circuit Breaker] Domain {} is blacklisted from direct-fetch. Routing via Relays.",
|
||||
domain
|
||||
);
|
||||
return perform_proxy_chain(slug).await;
|
||||
}
|
||||
|
||||
// Use shared global HTTP client
|
||||
let client = http_client().client();
|
||||
let headers = common_headers();
|
||||
|
||||
match client
|
||||
.get(slug)
|
||||
.headers(headers)
|
||||
.send() // Timeout handled by client
|
||||
.await
|
||||
{
|
||||
Ok(res) => {
|
||||
debug!(
|
||||
"[fetchWithProxy] Direct fetch response: url={}, status={}",
|
||||
slug,
|
||||
res.status()
|
||||
);
|
||||
if res.status().is_success() {
|
||||
let content_type = res
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|h| h.to_str().ok())
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let bytes = res.bytes().await?;
|
||||
|
||||
// Check if response is Gzip compressed (magic header 1f 8b)
|
||||
let text_data = if bytes.len() > 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
|
||||
// Gzip compressed, offload decompression to blocking thread
|
||||
let decompressed = tokio::task::spawn_blocking(move || {
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::Read;
|
||||
let decoder = GzDecoder::new(&bytes[..]);
|
||||
let mut decompressed = Vec::new();
|
||||
// 10MB absolute decompression bounds to prevent GZIP Bombs (OOM vulnerability)
|
||||
decoder
|
||||
.take(10_000_000)
|
||||
.read_to_end(&mut decompressed)
|
||||
.map(|_| decompressed)
|
||||
.map_err(|e| {
|
||||
AppError::Internal(format!(
|
||||
"Decompression failed or exceeded limits: {:?}",
|
||||
e
|
||||
))
|
||||
})
|
||||
})
|
||||
.await??;
|
||||
|
||||
match std::str::from_utf8(&decompressed) {
|
||||
Ok(s) => s.to_string(),
|
||||
Err(_) => String::from_utf8_lossy(&decompressed).to_string(),
|
||||
}
|
||||
} else {
|
||||
match std::str::from_utf8(&bytes) {
|
||||
Ok(s) => s.to_string(),
|
||||
Err(_) => {
|
||||
warn!("Response bytes are not valid UTF-8, using lossy conversion");
|
||||
String::from_utf8_lossy(&bytes).to_string()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if is_internet_baik_block_page(&text_data) {
|
||||
warn!("Blocked by internetbaik (direct fetch) for {}", slug);
|
||||
return perform_proxy_chain(slug).await;
|
||||
} else {
|
||||
let result = FetchResult {
|
||||
data: text_data,
|
||||
content_type,
|
||||
};
|
||||
// Cache the success result
|
||||
if let Err(e) = set_cached_fetch(slug, &result).await {
|
||||
warn!("Failed to cache result for {}: {:?}", slug, e);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
} else {
|
||||
let error_msg = format!(
|
||||
"Direct fetch failed with status {} for {}",
|
||||
res.status(),
|
||||
slug
|
||||
);
|
||||
|
||||
// Penalize domain if it throws aggressive Anti-Bot or Gateway Timeout codes
|
||||
if res.status().is_server_error()
|
||||
|| res.status() == reqwest::StatusCode::FORBIDDEN
|
||||
|| res.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE
|
||||
{
|
||||
if !domain.is_empty() {
|
||||
warn!("[Circuit Breaker] Blacklisting domain {} due to hostile HTTP status {}", domain, res.status());
|
||||
FAILED_DOMAINS.insert(domain.clone());
|
||||
}
|
||||
error!("{}", error_msg);
|
||||
return perform_proxy_chain(slug).await;
|
||||
} else {
|
||||
warn!("{}", error_msg);
|
||||
}
|
||||
Err(AppError::Internal(error_msg))
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Direct fetch failed for {}: {:?}", slug, e);
|
||||
warn!("{}", error_msg);
|
||||
|
||||
// Hard panic mapping: If reqwest core network/TLS fails, instantly blacklist the domain
|
||||
if !domain.is_empty() {
|
||||
warn!("[Circuit Breaker] Blacklisting domain {} due to core network/SSL trace failure", domain);
|
||||
FAILED_DOMAINS.insert(domain.clone());
|
||||
}
|
||||
|
||||
// Fall back seamlessly to Relay network proxy instead of throwing fatal transient backoff
|
||||
perform_proxy_chain(slug).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn fetch_with_proxy_only(slug: &str) -> Result<FetchResult, AppError> {
|
||||
if let Ok(Some(cached)) = get_cached_fetch(slug).await {
|
||||
return Ok(cached);
|
||||
}
|
||||
|
||||
perform_proxy_chain(slug).await
|
||||
}
|
||||
|
||||
async fn perform_proxy_chain(slug: &str) -> Result<FetchResult, AppError> {
|
||||
// 1. Try Relays
|
||||
match fetch_via_relays(slug).await {
|
||||
Ok(res) => return Ok(res),
|
||||
Err(e) => warn!("[ProxyChain] All relays failed for {}: {:?}", slug, e),
|
||||
}
|
||||
|
||||
// 2. Try Browserless as absolute last resort
|
||||
match fetch_via_browserless(slug).await {
|
||||
Ok(res) => Ok(res),
|
||||
Err(e) => {
|
||||
error!(
|
||||
"[ProxyChain] Browserless fallback failed for {}: {:?}",
|
||||
slug, e
|
||||
);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_via_relays(slug: &str) -> Result<FetchResult, AppError> {
|
||||
// Use shared client
|
||||
let client = http_client().client();
|
||||
|
||||
for relay in RELAY_ENDPOINTS {
|
||||
debug!(
|
||||
"[fetch_via_relays] Attempting to fetch {} via relay {}",
|
||||
slug, relay
|
||||
);
|
||||
|
||||
match client
|
||||
.get(*relay)
|
||||
.header("x-relay-target", slug)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(res) if res.status().is_success() => {
|
||||
let content_type = res
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|h| h.to_str().ok())
|
||||
.map(|s| s.to_string());
|
||||
let data = res.text().await?;
|
||||
|
||||
let result = FetchResult { data, content_type };
|
||||
debug!(
|
||||
"[fetch_via_relays] Successfully fetched {} via relay {}",
|
||||
slug, relay
|
||||
);
|
||||
|
||||
if let Err(e) = set_cached_fetch(slug, &result).await {
|
||||
warn!("Failed to cache relay result for {}: {:?}", slug, e);
|
||||
}
|
||||
|
||||
return Ok(result);
|
||||
}
|
||||
Ok(res) => {
|
||||
warn!(
|
||||
"[fetch_via_relays] Relay {} returned status {} for {}",
|
||||
relay,
|
||||
res.status(),
|
||||
slug
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"[fetch_via_relays] Relay {} failed for {}: {:?}",
|
||||
relay, slug, e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(AppError::Internal("All relay endpoints failed".to_string()))
|
||||
}
|
||||
|
||||
async fn fetch_via_browserless(slug: &str) -> Result<FetchResult, AppError> {
|
||||
use crate::infrastructure::browser::pool::get_browser_pool;
|
||||
|
||||
warn!("[Browserless] Falling back to remote browser for {}", slug);
|
||||
|
||||
let pool = get_browser_pool()
|
||||
.ok_or_else(|| AppError::Internal("Browser pool not initialized".to_string()))?;
|
||||
|
||||
let tab = pool
|
||||
.get_tab()
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(format!("Failed to get browser tab: {:?}", e)))?;
|
||||
|
||||
tab.goto(slug).await.map_err(|e| {
|
||||
AppError::Internal(format!("Browser navigation failed for {}: {:?}", slug, e))
|
||||
})?;
|
||||
|
||||
let data = tab
|
||||
.content()
|
||||
.await
|
||||
.map_err(|e| AppError::Internal(format!("Failed to get browser content: {:?}", e)))?;
|
||||
|
||||
let result = FetchResult {
|
||||
data,
|
||||
content_type: Some("text/html".to_string()),
|
||||
};
|
||||
|
||||
if let Err(e) = set_cached_fetch(slug, &result).await {
|
||||
warn!("Failed to cache browserless result for {}: {:?}", slug, e);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//! HTTP retry utilities with exponential backoff.
|
||||
|
||||
use backoff::ExponentialBackoff;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Default retry configuration for HTTP requests.
|
||||
pub fn default_backoff() -> ExponentialBackoff {
|
||||
ExponentialBackoff {
|
||||
initial_interval: Duration::from_millis(500),
|
||||
max_interval: Duration::from_secs(10),
|
||||
multiplier: 2.0,
|
||||
max_elapsed_time: Some(Duration::from_secs(30)),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a custom exponential backoff.
|
||||
pub fn custom_backoff(
|
||||
initial_ms: u64,
|
||||
max_secs: u64,
|
||||
multiplier: f64,
|
||||
max_elapsed_secs: u64,
|
||||
) -> ExponentialBackoff {
|
||||
ExponentialBackoff {
|
||||
initial_interval: Duration::from_millis(initial_ms),
|
||||
max_interval: Duration::from_secs(max_secs),
|
||||
multiplier,
|
||||
max_elapsed_time: Some(Duration::from_secs(max_elapsed_secs)),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Quick backoff for fast retries (3 attempts, 100ms initial).
|
||||
pub fn quick_backoff() -> ExponentialBackoff {
|
||||
ExponentialBackoff {
|
||||
initial_interval: Duration::from_millis(100),
|
||||
max_interval: Duration::from_secs(1),
|
||||
multiplier: 2.0,
|
||||
max_elapsed_time: Some(Duration::from_secs(5)),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Slow backoff for long operations (10 attempts, 1s initial).
|
||||
pub fn slow_backoff() -> ExponentialBackoff {
|
||||
ExponentialBackoff {
|
||||
initial_interval: Duration::from_secs(1),
|
||||
max_interval: Duration::from_secs(30),
|
||||
multiplier: 2.0,
|
||||
max_elapsed_time: Some(Duration::from_secs(120)),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Make an error transient (will be retried).
|
||||
pub fn transient<E>(err: E) -> backoff::Error<E> {
|
||||
backoff::Error::transient(err)
|
||||
}
|
||||
|
||||
/// Make an error permanent (will NOT be retried).
|
||||
pub fn permanent<E>(err: E) -> backoff::Error<E> {
|
||||
backoff::Error::permanent(err)
|
||||
}
|
||||
|
||||
// Re-export retry function for convenience
|
||||
pub use backoff::future::retry;
|
||||
@@ -0,0 +1,30 @@
|
||||
//! URL constants and dynamic environment-based URLs.
|
||||
//!
|
||||
//! Note: These URLs are kept as dynamic env lookups because they may vary
|
||||
//! between deployments and are not critical startup dependencies.
|
||||
|
||||
use crate::config::CONFIG;
|
||||
use std::env;
|
||||
|
||||
pub const BASE_URL: &str = "http://127.0.0.1:4090";
|
||||
pub const OTAKUDESU_BASE_URL: &str = "https://otakudesu.blog/";
|
||||
|
||||
/// Get Komik URL from environment config.
|
||||
pub fn get_komik_url() -> String {
|
||||
env::var("KOMIK2_BASE_URL").unwrap_or_else(|_| "https://komiku.org".to_string())
|
||||
}
|
||||
|
||||
/// Get production URL from environment config.
|
||||
pub fn get_production_url() -> String {
|
||||
env::var("NEXT_PUBLIC_PROD").unwrap_or_else(|_| CONFIG.urls.site_url.clone())
|
||||
}
|
||||
|
||||
/// Get Komik API URL from environment config.
|
||||
pub fn get_komik_api_url() -> String {
|
||||
env::var("KOMIK2_API_URL").unwrap_or_else(|_| "https://api.komiku.org".to_string())
|
||||
}
|
||||
|
||||
/// Get Otakudesu URL from environment config.
|
||||
pub fn get_otakudesu_url() -> String {
|
||||
env::var("OTAKUDESU_BASE_URL").unwrap_or_else(|_| OTAKUDESU_BASE_URL.to_string())
|
||||
}
|
||||
Reference in New Issue
Block a user