fix: exhaust all proxies then direct fallback before giving up

fetchWithRetry:
- Dynamically calculate maxAttempts = pool.size + 1 (direct fallback)
- Try every proxy in pool via rotation, then direct as last resort
- Only classify as error when even direct produces no response

fetchWithSessionRetry:
- Default maxRetries = sessionPool.size + 1 instead of hardcoded 3
- Try pool.size proxy attempts (each on a different proxy via rotateNow),
  then 1 direct attempt (no proxy) before giving up
- Only classify as error if direct also returns no response

ProxyPool.rotate():
- Skip proxies that have exceeded the failure threshold (isFailed)

Co-Authored-By: Claude <noreply@anthropic.com>

src/lib/fetch-utils.ts

@@ -137,9 +137,13 @@ export async function fetchWithRetry(
   let lastError: unknown;
   let usedProxy = false;
 
-  for (let attempt = 0; attempt < 3; attempt++) {
-    if (proxyPool && proxyPool.size > 0) {
-      // Proactive proxy usage on every attempt, rotating each time
+  // Calculate max attempts: pool.size proxies + 1 direct fallback, min 3
+  const poolSize = proxyPool?.size ?? 0;
+  const maxAttempts = poolSize > 0 ? poolSize + 1 : 3;
+
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    if (proxyPool && proxyPool.size > 0 && attempt < poolSize) {
+      // Use proxy from pool — first attempt gets current, rest rotate
       if (attempt === 0) {
         init.proxy = proxyPool.getProxyUrl()!;
         usedProxy = true;
@@ -150,13 +154,13 @@ export async function fetchWithRetry(
         usedProxy = true;
       }
     } else {
-      // No proxy pool — direct only
+      // Last resort — direct (no proxy)
       init.proxy = undefined;
       usedProxy = false;
     }
 
     const proxyShort = init.proxy ? init.proxy.replace(/https?:\/\//, "").replace(/@.*/, "@***") : "direct";
-    logProxy("fetchWithRetry", `attempt=${attempt + 1}/3 proxy=${proxyShort}`, { context });
+    logProxy("fetchWithRetry", `attempt=${attempt + 1}/${maxAttempts} proxy=${proxyShort}`, { context });
 
     try {
       response = await fetch(url, init);
@@ -186,11 +190,11 @@ export async function fetchWithRetry(
       const ctx = context ? `[${context}] ` : "";
       const errMsg = err instanceof Error ? err.message : String(err);
       logProxy("fetchWithRetry", `failed attempt=${attempt + 1} err=${errMsg}`, { context });
-      console.warn(`${ctx}fetch attempt ${attempt + 1}/3 failed: ${errMsg}`);
+      console.warn(`${ctx}fetch attempt ${attempt + 1}/${maxAttempts} failed: ${errMsg}`);
     }
   }
 
-  // All attempts exhausted
+  // All attempts exhausted — tried every proxy + direct
   logProxy("fetchWithRetry", "exhausted all retries", { context, hadResponse: !!response });
 
   // If we got at least one HTTP response, return it as-is so the caller
@@ -257,7 +261,7 @@ export async function fetchWithSessionRetry(
   sessionPool: SessionProxyPool | undefined,
   sessionId: string,
   context?: string,
-  maxRetries = 3,
+  maxRetries?: number,
 ): Promise<FetchWithRetryResult> {
   // Fallback when no session pool is available
   if (!sessionPool) {
@@ -270,24 +274,36 @@ export async function fetchWithSessionRetry(
     }
   }
 
+  // Exhaust all proxies + 1 direct fallback
+  const totalAttempts = maxRetries ?? sessionPool.size + 1;
   let lastError: unknown;
   let lastResponse: Response | undefined;
+  let triedDirect = false;
 
-  for (let attempt = 0; attempt < maxRetries; attempt++) {
-    // First attempt: acquire() assigns a proxy to this session (least-loaded
-    // distribution so many concurrent users spread across different IPs).
-    // Subsequent attempts: getProxyUrl() reads the existing (or rotated) proxy.
-    const proxyUrl = attempt === 0
-      ? sessionPool.acquire(sessionId)
-      : sessionPool.getProxyUrl(sessionId);
-    if (proxyUrl) {
-      init.proxy = proxyUrl;
+  for (let attempt = 0; attempt < totalAttempts; attempt++) {
+    // Determine proxy for this attempt
+    if (attempt < sessionPool.size) {
+      // Use session-sticky proxy
+      const proxyUrl = attempt === 0
+        ? sessionPool.acquire(sessionId)
+        : sessionPool.getProxyUrl(sessionId);
+      if (proxyUrl) {
+        init.proxy = proxyUrl;
+      }
+      triedDirect = false;
+    } else if (!triedDirect) {
+      // Last resort — direct (no proxy)
+      init.proxy = undefined;
+      triedDirect = true;
+    } else {
+      // Already tried direct, exhausted everything
+      break;
     }
 
     const proxyShort = init.proxy
       ? init.proxy.replace(/https?:\/\//, "").replace(/@.*/, "@***")
       : "direct";
-    logProxy("fetchWithSessionRetry", `attempt=${attempt + 1}/${maxRetries} proxy=${proxyShort}`, {
+    logProxy("fetchWithSessionRetry", `attempt=${attempt + 1}/${totalAttempts} proxy=${proxyShort}`, {
       context,
       sessionId: sessionId.slice(0, 8),
     });
@@ -317,7 +333,7 @@ export async function fetchWithSessionRetry(
 
       const ctx = context ? `[${context}] ` : "";
       console.warn(
-        `${ctx}fetchWithSessionRetry attempt ${attempt + 1}/${maxRetries} ` +
+        `${ctx}fetchWithSessionRetry attempt ${attempt + 1}/${totalAttempts} ` +
           `failed with ${response.status}${rotated ? " (rotated proxy)" : ""}`,
       );
     } catch (err) {
@@ -332,14 +348,14 @@ export async function fetchWithSessionRetry(
 
       const ctx = context ? `[${context}] ` : "";
       console.warn(
-        `${ctx}fetchWithSessionRetry attempt ${attempt + 1}/${maxRetries} ` +
+        `${ctx}fetchWithSessionRetry attempt ${attempt + 1}/${totalAttempts} ` +
           `failed: ${errMsg}${rotated ? " (rotated proxy)" : ""}`,
       );
     }
   }
 
-  // All attempts exhausted
-  logProxy("fetchWithSessionRetry", "exhausted all retries", {
+  // All attempts exhausted — tried every proxy + direct
+  logProxy("fetchWithSessionRetry", "exhausted all retries — tried all proxies + direct", {
     context,
     sessionId: sessionId.slice(0, 8),
     hadResponse: !!lastResponse,

src/lib/proxy-pool.ts

@@ -138,15 +138,30 @@ export class ProxyPool {
 
 	/**
 	 * Advance to the next proxy (round-robin, wraps around).
-	 * Returns the new current proxy or `null` if the pool is empty.
+	 * Skips proxies that have exceeded the failure threshold.
+	 * Returns the new current proxy or `null` if the pool is empty or
+	 * all proxies are failed.
 	 */
 	rotate(): ProxyEntry | null {
 		if (this.proxies.length === 0) return null;
 		const oldIndex = this.currentIndex;
-		this.currentIndex = (this.currentIndex + 1) % this.proxies.length;
-		const entry = this.proxies[this.currentIndex] ?? null;
-		logPool(`rotate ${oldIndex} -> ${this.currentIndex}`, { host: entry?.host });
-		return entry;
+		const startIndex = this.currentIndex;
+
+		// Keep advancing until we find a non-failed proxy or loop back
+		let checked = 0;
+		do {
+			this.currentIndex = (this.currentIndex + 1) % this.proxies.length;
+			checked++;
+			if (!this.isFailed()) {
+				const entry = this.proxies[this.currentIndex] ?? null;
+				logPool(`rotate ${oldIndex} -> ${this.currentIndex} (skipped ${checked - 1} failed)`);
+				return entry;
+			}
+		} while (this.currentIndex !== startIndex && checked <= this.proxies.length);
+
+		// All proxies failed — stay on current but log it
+		logPool(`rotate ${oldIndex} -> ${this.currentIndex} (all proxies failed)`);
+		return this.proxies[this.currentIndex] ?? null;
 	}
 
 	/**