refactor: remove EXTERNAL_BROWSERLESS_WS / browserless fallback

- Delete infrastructure/browser/ directory entirely (browser pool is gone) - Remove browser pool initialization from bootstrap - Remove fetch_via_browserless fallback from proxy_fetch - perform_proxy_chain now only uses relay endpoints - Clean up log messages, README, and compose env var The proxy-bun service at https://git.imrnes.team/MythEclipse/proxy-bun handles all proxy needs now.
2026-07-22 12:55:21 +07:00
parent 1ddcd15540
commit 1b195ac3f6
7 changed files with 2 additions and 542 deletions
@@ -29,7 +29,6 @@ Optional yang sering dipakai:
 ```env
 RUST_LOG=info
 EXTERNAL_BROWSERLESS_WS=
 MINIO_ENDPOINT=
 MINIO_BUCKET_NAME=
 MINIO_ACCESS_KEY=
@@ -46,14 +46,6 @@ impl Application {
        // Redis
        let _ = get_redis_conn().await;
        // Browser Pool
        tracing::info!("Initializing browser pool...");
        let browser_config = crate::infrastructure::browser::BrowserPoolConfig::default();
        match crate::infrastructure::browser::pool::init_browser_pool(browser_config).await {
            Ok(_) => tracing::info!("✓ Browser pool initialized"),
            Err(e) => tracing::error!("⚠️ Failed to initialize browser pool: {}", e),
        }
        // Database
        let mut opt = sea_orm::ConnectOptions::new(CONFIG.database_url.clone());
        opt.max_connections(20)
@@ -1,3 +0,0 @@
 pub mod pool;
 pub use pool::{init_browser_pool, BrowserPoolConfig};
@@ -1,483 +0,0 @@
 //! Browser pool for managing a single remote browser with multiple tabs.
 //!
 //! This pool maintains a connection to a remote Chrome DevTools Protocol (CDP)
 //! endpoint and provides tabs on-demand for scraping. Tabs are returned to the
 //! pool after use. Requires EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS to be set.
 use reqwest::StatusCode;
 use serde_json::json;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 use tokio::sync::{Mutex, Semaphore};
 use tracing::{debug, info, warn};
 fn build_http_endpoint(remote_url: &str, path: &str) -> anyhow::Result<String> {
    let mut url = reqwest::Url::parse(remote_url)
        .map_err(|e| anyhow::anyhow!("Invalid remote browser URL '{}': {}", remote_url, e))?;
    match url.scheme() {
        "ws" => {
            url.set_scheme("http")
                .map_err(|_| anyhow::anyhow!("Failed to convert scheme from ws to http"))?;
        }
        "wss" => {
            url.set_scheme("https")
                .map_err(|_| anyhow::anyhow!("Failed to convert scheme from wss to https"))?;
        }
        "http" | "https" => {}
        scheme => {
            return Err(anyhow::anyhow!(
                "Unsupported browser URL scheme '{}'. Expected ws:// or wss://",
                scheme
            ));
        }
    }
    let normalized = path.trim_start_matches('/');
    url.set_path(normalized);
    Ok(url.to_string())
 }
 /// Configuration for the browser pool.
 #[derive(Debug, Clone)]
 pub struct BrowserPoolConfig {
    /// Remote Chrome DevTools Protocol WebSocket URL (required).
    /// Must be set via `EXTERNAL_BROWSERLESS_WS` or `CHROME_REMOTE_WS` environment variable.
    /// Takes precedence from: EXTERNAL_BROWSERLESS_WS → CHROME_REMOTE_WS → None
    pub remote_websocket_url: String,
    /// Maximum number of concurrent tabs
    pub max_tabs: usize,
 }
 impl Default for BrowserPoolConfig {
    fn default() -> Self {
        // Priority (highest → lowest):
        //   1. EXTERNAL_BROWSERLESS_WS  — explicit operator override, always wins
        //   2. CHROME_REMOTE_WS         — may be injected by Coolify/Docker networking;
        //      rejected when it resolves to the unroutable Docker alias
        let external = std::env::var("EXTERNAL_BROWSERLESS_WS").ok();
        let chrome_remote = std::env::var("CHROME_REMOTE_WS").ok();
        let remote_websocket_url = if let Some(ext) = external {
            tracing::info!(" Browser: using EXTERNAL_BROWSERLESS_WS");
            ext
        } else if let Some(ref cr) = chrome_remote {
            if cr == "ws://browserless:3000" {
                eprintln!(
                    "CHROME_REMOTE_WS is set to the unroutable Docker alias \"ws://browserless:3000\" \
                     and EXTERNAL_BROWSERLESS_WS is unset. Remote browser connectivity is required."
                );
                std::process::exit(1);
            } else {
                tracing::info!(" Browser: using CHROME_REMOTE_WS");
                cr.clone()
            }
        } else {
            eprintln!(
                "Remote browser URL not configured. Set EXTERNAL_BROWSERLESS_WS or CHROME_REMOTE_WS environment variable."
            );
            std::process::exit(1);
        };
        Self {
            remote_websocket_url,
            max_tabs: 10,
        }
    }
 }
 /// A pool of browser tabs backed by a single remote browser instance.
 ///
 /// # Example
 ///
 /// ```ignore
 /// use scraper_service::browser::{BrowserPool, BrowserPoolConfig};
 ///
 /// let pool = BrowserPool::new(BrowserPoolConfig::default()).await?;
 /// let tab = pool.get_tab().await?;
 /// tab.goto("https://example.com").await?;
 /// let html = tab.content().await?;
 /// ```
 pub struct BrowserPool {
    /// Available (idle) tab IDs
    available_tabs: Mutex<Vec<String>>,
    /// Semaphore to limit concurrent tabs
    semaphore: Arc<Semaphore>,
    /// Configuration
    config: BrowserPoolConfig,
    /// Counter for generating unique tab IDs
    tab_counter: AtomicU64,
 }
 impl BrowserPool {
    /// Create a new browser pool connecting to a remote CDP endpoint.
    ///
    /// The pool will attempt to connect to the remote browser URL specified in
    /// BrowserPoolConfig. If the connection fails, an error is returned.
    pub async fn new(config: BrowserPoolConfig) -> anyhow::Result<Arc<Self>> {
        info!(
            "Initializing browser pool (remote: {}) with max {} tabs",
            config.remote_websocket_url, config.max_tabs
        );
        // Verify connection to remote browser with a simple health check
        Self::verify_remote_connection(&config).await?;
        let pool = Arc::new(Self {
            available_tabs: Mutex::new(Vec::new()),
            semaphore: Arc::new(Semaphore::new(config.max_tabs)),
            config: config.clone(),
            tab_counter: AtomicU64::new(0),
        });
        info!("Browser pool initialized (remote-only)");
        Ok(pool)
    }
    /// Verify connection to the remote browser.
    async fn verify_remote_connection(config: &BrowserPoolConfig) -> anyhow::Result<()> {
        let client = reqwest::Client::new();
        // Browserless/CDP deployments vary by exposed endpoint.
        // Try multiple known paths and succeed on any valid HTTP response from the host.
        let check_paths = ["json/version", "health", ""];
        let mut last_non_success: Option<(String, StatusCode)> = None;
        for path in check_paths {
            let endpoint = build_http_endpoint(&config.remote_websocket_url, path)?;
            let response = tokio::time::timeout(
                std::time::Duration::from_secs(5),
                client.get(&endpoint).send(),
            )
            .await;
            match response {
                Ok(Ok(r)) if r.status().is_success() => {
                    info!("Remote browser health check passed via {}", endpoint);
                    return Ok(());
                }
                Ok(Ok(r))
                    if r.status() == StatusCode::UNAUTHORIZED
                        || r.status() == StatusCode::FORBIDDEN =>
                {
                    return Err(anyhow::anyhow!(
                        "Remote browser reachable but authentication failed at {}: {}",
                        endpoint,
                        r.status()
                    ));
                }
                Ok(Ok(r)) => {
                    last_non_success = Some((endpoint, r.status()));
                }
                Ok(Err(e)) => {
                    return Err(anyhow::anyhow!(
                        "Failed to connect to remote browser: {}",
                        e
                    ));
                }
                Err(_) => {
                    return Err(anyhow::anyhow!("Remote browser connection timeout"));
                }
            }
        }
        if let Some((endpoint, status)) = last_non_success {
            warn!(
                "Remote browser reachable but no known health endpoint succeeded (last: {} -> {})",
                endpoint, status
            );
        }
        Ok(())
    }
    /// Get a tab from the pool.
    ///
    /// This will reuse an existing idle tab or create a new one.
    /// The returned `PooledTab` automatically returns to the pool when dropped.
    pub async fn get_tab(self: &Arc<Self>) -> anyhow::Result<PooledTab> {
        // Acquire semaphore permit (limits concurrent tabs)
        let permit = self.semaphore.clone().acquire_owned().await?;
        // Try to reuse an existing tab ID, or generate a new one
        let tab_id = {
            let mut tabs = self.available_tabs.lock().await;
            tabs.pop().unwrap_or_else(|| {
                let id = self.tab_counter.fetch_add(1, Ordering::SeqCst);
                format!("tab-{}", id)
            })
        };
        debug!("Allocated tab: {}", tab_id);
        Ok(PooledTab {
            tab_id,
            cdp_url: self.config.remote_websocket_url.clone(),
            pool: Arc::clone(self),
            _permit: permit,
        })
    }
    /// Get the number of available (idle) tabs in the pool.
    pub async fn available_count(&self) -> usize {
        self.available_tabs.lock().await.len()
    }
    /// Close the browser pool.
    pub async fn close(&self) -> anyhow::Result<()> {
        info!("Closing browser pool");
        self.available_tabs.lock().await.clear();
        Ok(())
    }
 }
 /// A tab borrowed from the pool (remote CDP-based).
 ///
 /// This is a lightweight wrapper that communicates with a remote Chrome instance
 /// via HTTP calls to the browser service. The tab is returned to the pool when dropped.
 pub struct PooledTab {
    /// Unique identifier for this tab
    pub tab_id: String,
    /// Remote CDP/browser service URL
    cdp_url: String,
    /// Pool reference for returning on drop
    pool: Arc<BrowserPool>,
    /// Semaphore permit (released on drop)
    _permit: tokio::sync::OwnedSemaphorePermit,
 }
 impl PooledTab {
    /// Navigate to a URL.
    pub async fn goto(&self, url: &str) -> anyhow::Result<()> {
        let client = reqwest::Client::new();
        let timeout = std::time::Duration::from_secs(15);
        let endpoint = build_http_endpoint(&self.cdp_url, "goto")?;
        let response = tokio::time::timeout(
            timeout,
            client.post(&endpoint).json(&json!({ "url": url })).send(),
        )
        .await
        .map_err(|_| anyhow::anyhow!("Timeout navigating to {}", url))?
        .map_err(|e| anyhow::anyhow!("Failed to navigate to {}: {}", url, e))?;
        if response.status().is_success() {
            Ok(())
        } else {
            Err(anyhow::anyhow!(
                "Failed to navigate to {}: {}",
                url,
                response.status()
            ))
        }
    }
    /// Get the page content (HTML).
    pub async fn content(&self) -> anyhow::Result<String> {
        let client = reqwest::Client::new();
        let timeout = std::time::Duration::from_secs(15);
        let endpoint = build_http_endpoint(&self.cdp_url, "content")?;
        let response = tokio::time::timeout(timeout, client.post(&endpoint).send())
            .await
            .map_err(|_| anyhow::anyhow!("Timeout getting page content"))?
            .map_err(|e| anyhow::anyhow!("Failed to get page content: {}", e))?;
        response
            .text()
            .await
            .map_err(|e| anyhow::anyhow!("Failed to read response body: {}", e))
    }
    /// Execute JavaScript and return the result.
    pub async fn evaluate<T: serde::de::DeserializeOwned>(
        &self,
        expression: &str,
    ) -> anyhow::Result<T> {
        let client = reqwest::Client::new();
        let timeout = std::time::Duration::from_secs(15);
        let endpoint = build_http_endpoint(&self.cdp_url, "evaluate")?;
        let response = tokio::time::timeout(
            timeout,
            client
                .post(&endpoint)
                .json(&json!({ "expression": expression }))
                .send(),
        )
        .await
        .map_err(|_| anyhow::anyhow!("Timeout evaluating JS"))?
        .map_err(|e| anyhow::anyhow!("Failed to evaluate JS: {}", e))?;
        let data: serde_json::Value = response
            .json()
            .await
            .map_err(|e| anyhow::anyhow!("Failed to parse JS result: {}", e))?;
        serde_json::from_value(data).map_err(|e| anyhow::anyhow!("Invalid JS result: {}", e))
    }
    /// Wait for a selector to appear.
    pub async fn wait_for_selector(&self, selector: &str) -> anyhow::Result<()> {
        let client = reqwest::Client::new();
        let timeout = std::time::Duration::from_secs(15);
        let endpoint = build_http_endpoint(&self.cdp_url, "waitForSelector")?;
        let response = tokio::time::timeout(
            timeout,
            client
                .post(&endpoint)
                .json(&json!({ "selector": selector }))
                .send(),
        )
        .await
        .map_err(|_| anyhow::anyhow!("Timeout waiting for selector '{}'", selector))?
        .map_err(|e| anyhow::anyhow!("Selector '{}' error: {}", selector, e))?;
        if response.status().is_success() {
            Ok(())
        } else {
            Err(anyhow::anyhow!(
                "Selector '{}' not found: {}",
                selector,
                response.status()
            ))
        }
    }
    /// Click an element by selector.
    pub async fn click(&self, selector: &str) -> anyhow::Result<()> {
        let client = reqwest::Client::new();
        let timeout = std::time::Duration::from_secs(15);
        let endpoint = build_http_endpoint(&self.cdp_url, "click")?;
        let response = tokio::time::timeout(
            timeout,
            client
                .post(&endpoint)
                .json(&json!({ "selector": selector }))
                .send(),
        )
        .await
        .map_err(|_| anyhow::anyhow!("Timeout clicking element"))?
        .map_err(|e| anyhow::anyhow!("Click error: {}", e))?;
        if response.status().is_success() {
            Ok(())
        } else {
            Err(anyhow::anyhow!(
                "Failed to click '{}': {}",
                selector,
                response.status()
            ))
        }
    }
    /// Type text into an element.
    pub async fn type_text(&self, selector: &str, text: &str) -> anyhow::Result<()> {
        let client = reqwest::Client::new();
        let timeout = std::time::Duration::from_secs(15);
        let endpoint = build_http_endpoint(&self.cdp_url, "type")?;
        let response = tokio::time::timeout(
            timeout,
            client
                .post(&endpoint)
                .json(&json!({ "selector": selector, "text": text }))
                .send(),
        )
        .await
        .map_err(|_| anyhow::anyhow!("Timeout typing text"))?
        .map_err(|e| anyhow::anyhow!("Type error: {}", e))?;
        if response.status().is_success() {
            Ok(())
        } else {
            Err(anyhow::anyhow!(
                "Failed to type into '{}': {}",
                selector,
                response.status()
            ))
        }
    }
    /// Take a screenshot as PNG bytes.
    pub async fn screenshot(&self) -> anyhow::Result<Vec<u8>> {
        let client = reqwest::Client::new();
        let timeout = std::time::Duration::from_secs(15);
        let endpoint = build_http_endpoint(&self.cdp_url, "screenshot")?;
        let response = tokio::time::timeout(
            timeout,
            client
                .post(&endpoint)
                .json(&json!({ "fullPage": true }))
                .send(),
        )
        .await
        .map_err(|_| anyhow::anyhow!("Timeout taking screenshot"))?
        .map_err(|e| anyhow::anyhow!("Failed to take screenshot: {}", e))?;
        response
            .bytes()
            .await
            .map(|b| b.to_vec())
            .map_err(|e| anyhow::anyhow!("Failed to read screenshot bytes: {}", e))
    }
    /// Get the current URL.
    pub async fn url(&self) -> anyhow::Result<String> {
        let client = reqwest::Client::new();
        let timeout = std::time::Duration::from_secs(10);
        let endpoint = build_http_endpoint(&self.cdp_url, "url")?;
        let response = tokio::time::timeout(timeout, client.post(&endpoint).send())
            .await
            .map_err(|_| anyhow::anyhow!("Timeout getting URL"))?
            .map_err(|e| anyhow::anyhow!("Failed to get URL: {}", e))?;
        response
            .text()
            .await
            .map_err(|e| anyhow::anyhow!("Failed to read URL: {}", e))
    }
 }
 impl Drop for PooledTab {
    fn drop(&mut self) {
        let pool = Arc::clone(&self.pool);
        let tab_id = self.tab_id.clone();
        let rt = tokio::runtime::Handle::try_current();
        if let Ok(handle) = rt {
            handle.spawn(async move {
                // Return tab ID to available pool for reuse
                pool.available_tabs.lock().await.push(tab_id);
            });
        } else {
            warn!("Cannot return tab: no tokio runtime available");
        }
    }
 }
 // Global browser pool instance
 use std::sync::OnceLock;
 static BROWSER_POOL: OnceLock<Arc<BrowserPool>> = OnceLock::new();
 /// Initialize the global browser pool.
 /// Call this once at application startup.
 pub async fn init_browser_pool(config: BrowserPoolConfig) -> anyhow::Result<()> {
    let pool = BrowserPool::new(config).await?;
    BROWSER_POOL
        .set(pool)
        .map_err(|_| anyhow::anyhow!("Browser pool already initialized"))?;
    Ok(())
 }
 /// Get the global browser pool.
 /// Returns None if not initialized.
 pub fn get_browser_pool() -> Option<Arc<BrowserPool>> {
    BROWSER_POOL.get().cloned()
 }
@@ -1,4 +1,3 @@
 pub mod browser;
 pub mod cache;
 pub mod persistence;
 pub mod repository;
@@ -94,7 +94,7 @@ impl ScrapingRepository for AlqanimeRepository {
            .await
            .map_err(|e| ScrapingError::Http(format!("Alqanime fetch failed: {}", e)))?;
        if response.data.trim().is_empty() {
-            warn!("Alqanime browserless fetch returned empty body for {}", url);
+            warn!("Alqanime fetch returned empty body for {}", url);
        }
        Ok(response.data)
    }
@@ -289,20 +289,10 @@ pub async fn fetch_with_proxy_only(slug: &str) -> Result<FetchResult, AppError>
 }
 async fn perform_proxy_chain(slug: &str) -> Result<FetchResult, AppError> {
    // 1. Try Relays
    match fetch_via_relays(slug).await {
        Ok(res) => return Ok(res),
        Err(e) => warn!("[ProxyChain] All relays failed for {}: {:?}", slug, e),
    }
    // 2. Try Browserless as absolute last resort
    match fetch_via_browserless(slug).await {
        Ok(res) => Ok(res),
        Err(e) => {
-            error!(
+            error!("[ProxyChain] All relays failed for {}: {:?}", slug, e);
                "[ProxyChain] Browserless fallback failed for {}: {:?}",
                slug, e
            );
            Err(e)
        }
    }
@@ -363,37 +353,3 @@ async fn fetch_via_relays(slug: &str) -> Result<FetchResult, AppError> {
    Err(AppError::Internal("All relay endpoints failed".to_string()))
 }
 async fn fetch_via_browserless(slug: &str) -> Result<FetchResult, AppError> {
    use crate::infrastructure::browser::pool::get_browser_pool;
    warn!("[Browserless] Falling back to remote browser for {}", slug);
    let pool = get_browser_pool()
        .ok_or_else(|| AppError::Internal("Browser pool not initialized".to_string()))?;
    let tab = pool
        .get_tab()
        .await
        .map_err(|e| AppError::Internal(format!("Failed to get browser tab: {:?}", e)))?;
    tab.goto(slug).await.map_err(|e| {
        AppError::Internal(format!("Browser navigation failed for {}: {:?}", slug, e))
    })?;
    let data = tab
        .content()
        .await
        .map_err(|e| AppError::Internal(format!("Failed to get browser content: {:?}", e)))?;
    let result = FetchResult {
        data,
        content_type: Some("text/html".to_string()),
    };
    if let Err(e) = set_cached_fetch(slug, &result).await {
        warn!("Failed to cache browserless result for {}: {:?}", slug, e);
    }
    Ok(result)
 }
		`@@ -1,3 +0,0 @@`
			`pub mod pool;`

			`pub use pool::{init_browser_pool, BrowserPoolConfig};`