diff --git a/README.md b/README.md index 9151224..16d9f4b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ Optional yang sering dipakai: ```env RUST_LOG=info -EXTERNAL_BROWSERLESS_WS= MINIO_ENDPOINT= MINIO_BUCKET_NAME= MINIO_ACCESS_KEY= diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs index a4ff4a7..98fc9a0 100644 --- a/src/bootstrap/mod.rs +++ b/src/bootstrap/mod.rs @@ -46,14 +46,6 @@ impl Application { // Redis let _ = get_redis_conn().await; - // Browser Pool - tracing::info!("Initializing browser pool..."); - let browser_config = crate::infrastructure::browser::BrowserPoolConfig::default(); - match crate::infrastructure::browser::pool::init_browser_pool(browser_config).await { - Ok(_) => tracing::info!("✓ Browser pool initialized"), - Err(e) => tracing::error!("⚠️ Failed to initialize browser pool: {}", e), - } - // Database let mut opt = sea_orm::ConnectOptions::new(CONFIG.database_url.clone()); opt.max_connections(20) diff --git a/src/infrastructure/browser/mod.rs b/src/infrastructure/browser/mod.rs deleted file mode 100644 index 52bd506..0000000 --- a/src/infrastructure/browser/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod pool; - -pub use pool::{init_browser_pool, BrowserPoolConfig}; diff --git a/src/infrastructure/browser/pool.rs b/src/infrastructure/browser/pool.rs deleted file mode 100644 index e6157f4..0000000 --- a/src/infrastructure/browser/pool.rs +++ /dev/null @@ -1,483 +0,0 @@ -//! Browser pool for managing a single remote browser with multiple tabs. -//! -//! This pool maintains a connection to a remote Chrome DevTools Protocol (CDP) -//! endpoint and provides tabs on-demand for scraping. Tabs are returned to the -//! pool after use. Requires EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS to be set. - -use reqwest::StatusCode; -use serde_json::json; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use tokio::sync::{Mutex, Semaphore}; -use tracing::{debug, info, warn}; - -fn build_http_endpoint(remote_url: &str, path: &str) -> anyhow::Result { - let mut url = reqwest::Url::parse(remote_url) - .map_err(|e| anyhow::anyhow!("Invalid remote browser URL '{}': {}", remote_url, e))?; - - match url.scheme() { - "ws" => { - url.set_scheme("http") - .map_err(|_| anyhow::anyhow!("Failed to convert scheme from ws to http"))?; - } - "wss" => { - url.set_scheme("https") - .map_err(|_| anyhow::anyhow!("Failed to convert scheme from wss to https"))?; - } - "http" | "https" => {} - scheme => { - return Err(anyhow::anyhow!( - "Unsupported browser URL scheme '{}'. Expected ws:// or wss://", - scheme - )); - } - } - - let normalized = path.trim_start_matches('/'); - url.set_path(normalized); - - Ok(url.to_string()) -} - -/// Configuration for the browser pool. -#[derive(Debug, Clone)] -pub struct BrowserPoolConfig { - /// Remote Chrome DevTools Protocol WebSocket URL (required). - /// Must be set via `EXTERNAL_BROWSERLESS_WS` or `CHROME_REMOTE_WS` environment variable. - /// Takes precedence from: EXTERNAL_BROWSERLESS_WS → CHROME_REMOTE_WS → None - pub remote_websocket_url: String, - /// Maximum number of concurrent tabs - pub max_tabs: usize, -} - -impl Default for BrowserPoolConfig { - fn default() -> Self { - // Priority (highest → lowest): - // 1. EXTERNAL_BROWSERLESS_WS — explicit operator override, always wins - // 2. CHROME_REMOTE_WS — may be injected by Coolify/Docker networking; - // rejected when it resolves to the unroutable Docker alias - let external = std::env::var("EXTERNAL_BROWSERLESS_WS").ok(); - let chrome_remote = std::env::var("CHROME_REMOTE_WS").ok(); - - let remote_websocket_url = if let Some(ext) = external { - tracing::info!(" Browser: using EXTERNAL_BROWSERLESS_WS"); - ext - } else if let Some(ref cr) = chrome_remote { - if cr == "ws://browserless:3000" { - eprintln!( - "CHROME_REMOTE_WS is set to the unroutable Docker alias \"ws://browserless:3000\" \ - and EXTERNAL_BROWSERLESS_WS is unset. Remote browser connectivity is required." - ); - std::process::exit(1); - } else { - tracing::info!(" Browser: using CHROME_REMOTE_WS"); - cr.clone() - } - } else { - eprintln!( - "Remote browser URL not configured. Set EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS environment variable." - ); - std::process::exit(1); - }; - - Self { - remote_websocket_url, - max_tabs: 10, - } - } -} - -/// A pool of browser tabs backed by a single remote browser instance. -/// -/// # Example -/// -/// ```ignore -/// use scraper_service::browser::{BrowserPool, BrowserPoolConfig}; -/// -/// let pool = BrowserPool::new(BrowserPoolConfig::default()).await?; -/// let tab = pool.get_tab().await?; -/// tab.goto("https://example.com").await?; -/// let html = tab.content().await?; -/// ``` -pub struct BrowserPool { - /// Available (idle) tab IDs - available_tabs: Mutex>, - /// Semaphore to limit concurrent tabs - semaphore: Arc, - /// Configuration - config: BrowserPoolConfig, - /// Counter for generating unique tab IDs - tab_counter: AtomicU64, -} - -impl BrowserPool { - /// Create a new browser pool connecting to a remote CDP endpoint. - /// - /// The pool will attempt to connect to the remote browser URL specified in - /// BrowserPoolConfig. If the connection fails, an error is returned. - pub async fn new(config: BrowserPoolConfig) -> anyhow::Result> { - info!( - "Initializing browser pool (remote: {}) with max {} tabs", - config.remote_websocket_url, config.max_tabs - ); - - // Verify connection to remote browser with a simple health check - Self::verify_remote_connection(&config).await?; - - let pool = Arc::new(Self { - available_tabs: Mutex::new(Vec::new()), - semaphore: Arc::new(Semaphore::new(config.max_tabs)), - config: config.clone(), - tab_counter: AtomicU64::new(0), - }); - - info!("Browser pool initialized (remote-only)"); - Ok(pool) - } - - /// Verify connection to the remote browser. - async fn verify_remote_connection(config: &BrowserPoolConfig) -> anyhow::Result<()> { - let client = reqwest::Client::new(); - - // Browserless/CDP deployments vary by exposed endpoint. - // Try multiple known paths and succeed on any valid HTTP response from the host. - let check_paths = ["json/version", "health", ""]; - let mut last_non_success: Option<(String, StatusCode)> = None; - - for path in check_paths { - let endpoint = build_http_endpoint(&config.remote_websocket_url, path)?; - let response = tokio::time::timeout( - std::time::Duration::from_secs(5), - client.get(&endpoint).send(), - ) - .await; - - match response { - Ok(Ok(r)) if r.status().is_success() => { - info!("Remote browser health check passed via {}", endpoint); - return Ok(()); - } - Ok(Ok(r)) - if r.status() == StatusCode::UNAUTHORIZED - || r.status() == StatusCode::FORBIDDEN => - { - return Err(anyhow::anyhow!( - "Remote browser reachable but authentication failed at {}: {}", - endpoint, - r.status() - )); - } - Ok(Ok(r)) => { - last_non_success = Some((endpoint, r.status())); - } - Ok(Err(e)) => { - return Err(anyhow::anyhow!( - "Failed to connect to remote browser: {}", - e - )); - } - Err(_) => { - return Err(anyhow::anyhow!("Remote browser connection timeout")); - } - } - } - - if let Some((endpoint, status)) = last_non_success { - warn!( - "Remote browser reachable but no known health endpoint succeeded (last: {} -> {})", - endpoint, status - ); - } - - Ok(()) - } - - /// Get a tab from the pool. - /// - /// This will reuse an existing idle tab or create a new one. - /// The returned `PooledTab` automatically returns to the pool when dropped. - pub async fn get_tab(self: &Arc) -> anyhow::Result { - // Acquire semaphore permit (limits concurrent tabs) - let permit = self.semaphore.clone().acquire_owned().await?; - - // Try to reuse an existing tab ID, or generate a new one - let tab_id = { - let mut tabs = self.available_tabs.lock().await; - tabs.pop().unwrap_or_else(|| { - let id = self.tab_counter.fetch_add(1, Ordering::SeqCst); - format!("tab-{}", id) - }) - }; - - debug!("Allocated tab: {}", tab_id); - - Ok(PooledTab { - tab_id, - cdp_url: self.config.remote_websocket_url.clone(), - pool: Arc::clone(self), - _permit: permit, - }) - } - - /// Get the number of available (idle) tabs in the pool. - pub async fn available_count(&self) -> usize { - self.available_tabs.lock().await.len() - } - - /// Close the browser pool. - pub async fn close(&self) -> anyhow::Result<()> { - info!("Closing browser pool"); - self.available_tabs.lock().await.clear(); - Ok(()) - } -} - -/// A tab borrowed from the pool (remote CDP-based). -/// -/// This is a lightweight wrapper that communicates with a remote Chrome instance -/// via HTTP calls to the browser service. The tab is returned to the pool when dropped. -pub struct PooledTab { - /// Unique identifier for this tab - pub tab_id: String, - /// Remote CDP/browser service URL - cdp_url: String, - /// Pool reference for returning on drop - pool: Arc, - /// Semaphore permit (released on drop) - _permit: tokio::sync::OwnedSemaphorePermit, -} - -impl PooledTab { - /// Navigate to a URL. - pub async fn goto(&self, url: &str) -> anyhow::Result<()> { - let client = reqwest::Client::new(); - let timeout = std::time::Duration::from_secs(15); - let endpoint = build_http_endpoint(&self.cdp_url, "goto")?; - - let response = tokio::time::timeout( - timeout, - client.post(&endpoint).json(&json!({ "url": url })).send(), - ) - .await - .map_err(|_| anyhow::anyhow!("Timeout navigating to {}", url))? - .map_err(|e| anyhow::anyhow!("Failed to navigate to {}: {}", url, e))?; - - if response.status().is_success() { - Ok(()) - } else { - Err(anyhow::anyhow!( - "Failed to navigate to {}: {}", - url, - response.status() - )) - } - } - - /// Get the page content (HTML). - pub async fn content(&self) -> anyhow::Result { - let client = reqwest::Client::new(); - let timeout = std::time::Duration::from_secs(15); - let endpoint = build_http_endpoint(&self.cdp_url, "content")?; - - let response = tokio::time::timeout(timeout, client.post(&endpoint).send()) - .await - .map_err(|_| anyhow::anyhow!("Timeout getting page content"))? - .map_err(|e| anyhow::anyhow!("Failed to get page content: {}", e))?; - - response - .text() - .await - .map_err(|e| anyhow::anyhow!("Failed to read response body: {}", e)) - } - - /// Execute JavaScript and return the result. - pub async fn evaluate( - &self, - expression: &str, - ) -> anyhow::Result { - let client = reqwest::Client::new(); - let timeout = std::time::Duration::from_secs(15); - let endpoint = build_http_endpoint(&self.cdp_url, "evaluate")?; - - let response = tokio::time::timeout( - timeout, - client - .post(&endpoint) - .json(&json!({ "expression": expression })) - .send(), - ) - .await - .map_err(|_| anyhow::anyhow!("Timeout evaluating JS"))? - .map_err(|e| anyhow::anyhow!("Failed to evaluate JS: {}", e))?; - - let data: serde_json::Value = response - .json() - .await - .map_err(|e| anyhow::anyhow!("Failed to parse JS result: {}", e))?; - - serde_json::from_value(data).map_err(|e| anyhow::anyhow!("Invalid JS result: {}", e)) - } - - /// Wait for a selector to appear. - pub async fn wait_for_selector(&self, selector: &str) -> anyhow::Result<()> { - let client = reqwest::Client::new(); - let timeout = std::time::Duration::from_secs(15); - let endpoint = build_http_endpoint(&self.cdp_url, "waitForSelector")?; - - let response = tokio::time::timeout( - timeout, - client - .post(&endpoint) - .json(&json!({ "selector": selector })) - .send(), - ) - .await - .map_err(|_| anyhow::anyhow!("Timeout waiting for selector '{}'", selector))? - .map_err(|e| anyhow::anyhow!("Selector '{}' error: {}", selector, e))?; - - if response.status().is_success() { - Ok(()) - } else { - Err(anyhow::anyhow!( - "Selector '{}' not found: {}", - selector, - response.status() - )) - } - } - - /// Click an element by selector. - pub async fn click(&self, selector: &str) -> anyhow::Result<()> { - let client = reqwest::Client::new(); - let timeout = std::time::Duration::from_secs(15); - let endpoint = build_http_endpoint(&self.cdp_url, "click")?; - - let response = tokio::time::timeout( - timeout, - client - .post(&endpoint) - .json(&json!({ "selector": selector })) - .send(), - ) - .await - .map_err(|_| anyhow::anyhow!("Timeout clicking element"))? - .map_err(|e| anyhow::anyhow!("Click error: {}", e))?; - - if response.status().is_success() { - Ok(()) - } else { - Err(anyhow::anyhow!( - "Failed to click '{}': {}", - selector, - response.status() - )) - } - } - - /// Type text into an element. - pub async fn type_text(&self, selector: &str, text: &str) -> anyhow::Result<()> { - let client = reqwest::Client::new(); - let timeout = std::time::Duration::from_secs(15); - let endpoint = build_http_endpoint(&self.cdp_url, "type")?; - - let response = tokio::time::timeout( - timeout, - client - .post(&endpoint) - .json(&json!({ "selector": selector, "text": text })) - .send(), - ) - .await - .map_err(|_| anyhow::anyhow!("Timeout typing text"))? - .map_err(|e| anyhow::anyhow!("Type error: {}", e))?; - - if response.status().is_success() { - Ok(()) - } else { - Err(anyhow::anyhow!( - "Failed to type into '{}': {}", - selector, - response.status() - )) - } - } - - /// Take a screenshot as PNG bytes. - pub async fn screenshot(&self) -> anyhow::Result> { - let client = reqwest::Client::new(); - let timeout = std::time::Duration::from_secs(15); - let endpoint = build_http_endpoint(&self.cdp_url, "screenshot")?; - - let response = tokio::time::timeout( - timeout, - client - .post(&endpoint) - .json(&json!({ "fullPage": true })) - .send(), - ) - .await - .map_err(|_| anyhow::anyhow!("Timeout taking screenshot"))? - .map_err(|e| anyhow::anyhow!("Failed to take screenshot: {}", e))?; - - response - .bytes() - .await - .map(|b| b.to_vec()) - .map_err(|e| anyhow::anyhow!("Failed to read screenshot bytes: {}", e)) - } - - /// Get the current URL. - pub async fn url(&self) -> anyhow::Result { - let client = reqwest::Client::new(); - let timeout = std::time::Duration::from_secs(10); - let endpoint = build_http_endpoint(&self.cdp_url, "url")?; - - let response = tokio::time::timeout(timeout, client.post(&endpoint).send()) - .await - .map_err(|_| anyhow::anyhow!("Timeout getting URL"))? - .map_err(|e| anyhow::anyhow!("Failed to get URL: {}", e))?; - - response - .text() - .await - .map_err(|e| anyhow::anyhow!("Failed to read URL: {}", e)) - } -} - -impl Drop for PooledTab { - fn drop(&mut self) { - let pool = Arc::clone(&self.pool); - let tab_id = self.tab_id.clone(); - - let rt = tokio::runtime::Handle::try_current(); - if let Ok(handle) = rt { - handle.spawn(async move { - // Return tab ID to available pool for reuse - pool.available_tabs.lock().await.push(tab_id); - }); - } else { - warn!("Cannot return tab: no tokio runtime available"); - } - } -} - -// Global browser pool instance -use std::sync::OnceLock; - -static BROWSER_POOL: OnceLock> = OnceLock::new(); - -/// Initialize the global browser pool. -/// Call this once at application startup. -pub async fn init_browser_pool(config: BrowserPoolConfig) -> anyhow::Result<()> { - let pool = BrowserPool::new(config).await?; - BROWSER_POOL - .set(pool) - .map_err(|_| anyhow::anyhow!("Browser pool already initialized"))?; - Ok(()) -} - -/// Get the global browser pool. -/// Returns None if not initialized. -pub fn get_browser_pool() -> Option> { - BROWSER_POOL.get().cloned() -} diff --git a/src/infrastructure/mod.rs b/src/infrastructure/mod.rs index 18f2310..fb4ed2b 100644 --- a/src/infrastructure/mod.rs +++ b/src/infrastructure/mod.rs @@ -1,4 +1,3 @@ -pub mod browser; pub mod cache; pub mod persistence; pub mod repository; diff --git a/src/infrastructure/repository/alqanime.rs b/src/infrastructure/repository/alqanime.rs index 514e328..030cd95 100644 --- a/src/infrastructure/repository/alqanime.rs +++ b/src/infrastructure/repository/alqanime.rs @@ -94,7 +94,7 @@ impl ScrapingRepository for AlqanimeRepository { .await .map_err(|e| ScrapingError::Http(format!("Alqanime fetch failed: {}", e)))?; if response.data.trim().is_empty() { - warn!("Alqanime browserless fetch returned empty body for {}", url); + warn!("Alqanime fetch returned empty body for {}", url); } Ok(response.data) } diff --git a/src/infrastructure/scraping/proxy_fetch.rs b/src/infrastructure/scraping/proxy_fetch.rs index 96ee3fb..a668e47 100644 --- a/src/infrastructure/scraping/proxy_fetch.rs +++ b/src/infrastructure/scraping/proxy_fetch.rs @@ -289,20 +289,10 @@ pub async fn fetch_with_proxy_only(slug: &str) -> Result } async fn perform_proxy_chain(slug: &str) -> Result { - // 1. Try Relays match fetch_via_relays(slug).await { - Ok(res) => return Ok(res), - Err(e) => warn!("[ProxyChain] All relays failed for {}: {:?}", slug, e), - } - - // 2. Try Browserless as absolute last resort - match fetch_via_browserless(slug).await { Ok(res) => Ok(res), Err(e) => { - error!( - "[ProxyChain] Browserless fallback failed for {}: {:?}", - slug, e - ); + error!("[ProxyChain] All relays failed for {}: {:?}", slug, e); Err(e) } } @@ -363,37 +353,3 @@ async fn fetch_via_relays(slug: &str) -> Result { Err(AppError::Internal("All relay endpoints failed".to_string())) } - -async fn fetch_via_browserless(slug: &str) -> Result { - use crate::infrastructure::browser::pool::get_browser_pool; - - warn!("[Browserless] Falling back to remote browser for {}", slug); - - let pool = get_browser_pool() - .ok_or_else(|| AppError::Internal("Browser pool not initialized".to_string()))?; - - let tab = pool - .get_tab() - .await - .map_err(|e| AppError::Internal(format!("Failed to get browser tab: {:?}", e)))?; - - tab.goto(slug).await.map_err(|e| { - AppError::Internal(format!("Browser navigation failed for {}: {:?}", slug, e)) - })?; - - let data = tab - .content() - .await - .map_err(|e| AppError::Internal(format!("Failed to get browser content: {:?}", e)))?; - - let result = FetchResult { - data, - content_type: Some("text/html".to_string()), - }; - - if let Err(e) = set_cached_fetch(slug, &result).await { - warn!("Failed to cache browserless result for {}: {:?}", slug, e); - } - - Ok(result) -}