From 283ed26231cfd96c9c5c714bbf0b10d863ec9a43 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Wed, 17 Jun 2026 04:45:30 +0700 Subject: [PATCH] fix: exhaust all proxies then direct fallback before giving up fetchWithRetry: - Dynamically calculate maxAttempts = pool.size + 1 (direct fallback) - Try every proxy in pool via rotation, then direct as last resort - Only classify as error when even direct produces no response fetchWithSessionRetry: - Default maxRetries = sessionPool.size + 1 instead of hardcoded 3 - Try pool.size proxy attempts (each on a different proxy via rotateNow), then 1 direct attempt (no proxy) before giving up - Only classify as error if direct also returns no response ProxyPool.rotate(): - Skip proxies that have exceeded the failure threshold (isFailed) Co-Authored-By: Claude --- src/lib/fetch-utils.ts | 60 ++++++++++++++++++++++++++---------------- src/lib/proxy-pool.ts | 25 ++++++++++++++---- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/lib/fetch-utils.ts b/src/lib/fetch-utils.ts index f5772db..543dd83 100644 --- a/src/lib/fetch-utils.ts +++ b/src/lib/fetch-utils.ts @@ -137,9 +137,13 @@ export async function fetchWithRetry( let lastError: unknown; let usedProxy = false; - for (let attempt = 0; attempt < 3; attempt++) { - if (proxyPool && proxyPool.size > 0) { - // Proactive proxy usage on every attempt, rotating each time + // Calculate max attempts: pool.size proxies + 1 direct fallback, min 3 + const poolSize = proxyPool?.size ?? 0; + const maxAttempts = poolSize > 0 ? poolSize + 1 : 3; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + if (proxyPool && proxyPool.size > 0 && attempt < poolSize) { + // Use proxy from pool — first attempt gets current, rest rotate if (attempt === 0) { init.proxy = proxyPool.getProxyUrl()!; usedProxy = true; @@ -150,13 +154,13 @@ export async function fetchWithRetry( usedProxy = true; } } else { - // No proxy pool — direct only + // Last resort — direct (no proxy) init.proxy = undefined; usedProxy = false; } const proxyShort = init.proxy ? init.proxy.replace(/https?:\/\//, "").replace(/@.*/, "@***") : "direct"; - logProxy("fetchWithRetry", `attempt=${attempt + 1}/3 proxy=${proxyShort}`, { context }); + logProxy("fetchWithRetry", `attempt=${attempt + 1}/${maxAttempts} proxy=${proxyShort}`, { context }); try { response = await fetch(url, init); @@ -186,11 +190,11 @@ export async function fetchWithRetry( const ctx = context ? `[${context}] ` : ""; const errMsg = err instanceof Error ? err.message : String(err); logProxy("fetchWithRetry", `failed attempt=${attempt + 1} err=${errMsg}`, { context }); - console.warn(`${ctx}fetch attempt ${attempt + 1}/3 failed: ${errMsg}`); + console.warn(`${ctx}fetch attempt ${attempt + 1}/${maxAttempts} failed: ${errMsg}`); } } - // All attempts exhausted + // All attempts exhausted — tried every proxy + direct logProxy("fetchWithRetry", "exhausted all retries", { context, hadResponse: !!response }); // If we got at least one HTTP response, return it as-is so the caller @@ -257,7 +261,7 @@ export async function fetchWithSessionRetry( sessionPool: SessionProxyPool | undefined, sessionId: string, context?: string, - maxRetries = 3, + maxRetries?: number, ): Promise { // Fallback when no session pool is available if (!sessionPool) { @@ -270,24 +274,36 @@ export async function fetchWithSessionRetry( } } + // Exhaust all proxies + 1 direct fallback + const totalAttempts = maxRetries ?? sessionPool.size + 1; let lastError: unknown; let lastResponse: Response | undefined; + let triedDirect = false; - for (let attempt = 0; attempt < maxRetries; attempt++) { - // First attempt: acquire() assigns a proxy to this session (least-loaded - // distribution so many concurrent users spread across different IPs). - // Subsequent attempts: getProxyUrl() reads the existing (or rotated) proxy. - const proxyUrl = attempt === 0 - ? sessionPool.acquire(sessionId) - : sessionPool.getProxyUrl(sessionId); - if (proxyUrl) { - init.proxy = proxyUrl; + for (let attempt = 0; attempt < totalAttempts; attempt++) { + // Determine proxy for this attempt + if (attempt < sessionPool.size) { + // Use session-sticky proxy + const proxyUrl = attempt === 0 + ? sessionPool.acquire(sessionId) + : sessionPool.getProxyUrl(sessionId); + if (proxyUrl) { + init.proxy = proxyUrl; + } + triedDirect = false; + } else if (!triedDirect) { + // Last resort — direct (no proxy) + init.proxy = undefined; + triedDirect = true; + } else { + // Already tried direct, exhausted everything + break; } const proxyShort = init.proxy ? init.proxy.replace(/https?:\/\//, "").replace(/@.*/, "@***") : "direct"; - logProxy("fetchWithSessionRetry", `attempt=${attempt + 1}/${maxRetries} proxy=${proxyShort}`, { + logProxy("fetchWithSessionRetry", `attempt=${attempt + 1}/${totalAttempts} proxy=${proxyShort}`, { context, sessionId: sessionId.slice(0, 8), }); @@ -317,7 +333,7 @@ export async function fetchWithSessionRetry( const ctx = context ? `[${context}] ` : ""; console.warn( - `${ctx}fetchWithSessionRetry attempt ${attempt + 1}/${maxRetries} ` + + `${ctx}fetchWithSessionRetry attempt ${attempt + 1}/${totalAttempts} ` + `failed with ${response.status}${rotated ? " (rotated proxy)" : ""}`, ); } catch (err) { @@ -332,14 +348,14 @@ export async function fetchWithSessionRetry( const ctx = context ? `[${context}] ` : ""; console.warn( - `${ctx}fetchWithSessionRetry attempt ${attempt + 1}/${maxRetries} ` + + `${ctx}fetchWithSessionRetry attempt ${attempt + 1}/${totalAttempts} ` + `failed: ${errMsg}${rotated ? " (rotated proxy)" : ""}`, ); } } - // All attempts exhausted - logProxy("fetchWithSessionRetry", "exhausted all retries", { + // All attempts exhausted — tried every proxy + direct + logProxy("fetchWithSessionRetry", "exhausted all retries — tried all proxies + direct", { context, sessionId: sessionId.slice(0, 8), hadResponse: !!lastResponse, diff --git a/src/lib/proxy-pool.ts b/src/lib/proxy-pool.ts index 3e764a9..0d8df6c 100644 --- a/src/lib/proxy-pool.ts +++ b/src/lib/proxy-pool.ts @@ -138,15 +138,30 @@ export class ProxyPool { /** * Advance to the next proxy (round-robin, wraps around). - * Returns the new current proxy or `null` if the pool is empty. + * Skips proxies that have exceeded the failure threshold. + * Returns the new current proxy or `null` if the pool is empty or + * all proxies are failed. */ rotate(): ProxyEntry | null { if (this.proxies.length === 0) return null; const oldIndex = this.currentIndex; - this.currentIndex = (this.currentIndex + 1) % this.proxies.length; - const entry = this.proxies[this.currentIndex] ?? null; - logPool(`rotate ${oldIndex} -> ${this.currentIndex}`, { host: entry?.host }); - return entry; + const startIndex = this.currentIndex; + + // Keep advancing until we find a non-failed proxy or loop back + let checked = 0; + do { + this.currentIndex = (this.currentIndex + 1) % this.proxies.length; + checked++; + if (!this.isFailed()) { + const entry = this.proxies[this.currentIndex] ?? null; + logPool(`rotate ${oldIndex} -> ${this.currentIndex} (skipped ${checked - 1} failed)`); + return entry; + } + } while (this.currentIndex !== startIndex && checked <= this.proxies.length); + + // All proxies failed — stay on current but log it + logPool(`rotate ${oldIndex} -> ${this.currentIndex} (all proxies failed)`); + return this.proxies[this.currentIndex] ?? null; } /**