feat(core): implement robust fetch utilities and stream management

Introduce a centralized `fetch-utils.ts` to handle retry logic with proxy fallback, SSE line buffering to prevent chunk-boundary corruption, and graceful shutdown via active reader tracking.

Key changes:
- Add `fetchWithRetry` for automatic direct-to-proxy failover.
- Implement `SSELineBuffer` to ensure reliable parsing of split SSE chunks.
- Add `createStreamBodyLimiter` to enforce payload limits on streaming requests.
- Refactor `ProxyPool` to decouple failure marking from rotation.
- Standardize CORS handling and environment variable configuration.
- Clean up documentation and remove obsolete skill files.
This commit is contained in:
MythEclipse
2026-06-11 03:56:26 +07:00
parent db065f788d
commit b09bca8def
13 changed files with 1022 additions and 822 deletions
+23 -16
View File
@@ -8,7 +8,7 @@
import { readFileSync, existsSync } from "node:fs";
// ─── Types ───────────────────────────────────────────────────────────────────────
// --- Types -------------------------------------------------------------------
export interface ProxyEntry {
host: string;
@@ -17,21 +17,21 @@ export interface ProxyEntry {
password: string;
}
// ─── ProxyPool ───────────────────────────────────────────────────────────────────
// --- ProxyPool ---------------------------------------------------------------
export class ProxyPool {
private proxies: ProxyEntry[] = [];
private currentIndex = 0;
private failureThreshold = 3;
/** host:port consecutive failure count */
/** host:port -> consecutive failure count */
private failures = new Map<string, number>();
// ── Load ──────────────────────────────────────────────────────────
// -- Load --------------------------------------------------------------------
/**
* Load proxies from a file at `filePath`.
*
* Expected format one proxy per line:
* Expected format -- one proxy per line:
* host:port:username:password
*
* Lines starting with `#` are ignored as comments.
@@ -79,7 +79,7 @@ export class ProxyPool {
}
/**
* Convenience load from a path or skip.
* Convenience -- load from a path or skip.
* Returns `true` if proxies were loaded.
*/
tryLoad(filePath?: string): boolean {
@@ -87,7 +87,7 @@ export class ProxyPool {
return this.proxies.length > 0;
}
// ── Access ────────────────────────────────────────────────────────
// -- Access ------------------------------------------------------------------
/** Total number of proxies in the pool. */
get size(): number {
@@ -123,7 +123,7 @@ export class ProxyPool {
return `http://${auth}${entry.host}:${entry.port}`;
}
// ── Rotation ──────────────────────────────────────────────────────
// -- Rotation ----------------------------------------------------------------
/**
* Advance to the next proxy (round-robin, wraps around).
@@ -136,15 +136,16 @@ export class ProxyPool {
}
/**
* Mark the **current** proxy as failed.
* Mark the **current** proxy as failed. Increments the failure count;
* when the count reaches the threshold a warning is logged.
*
* If the failure count exceeds `threshold`, the proxy is skipped and
* rotation continues until a healthy proxy is found (or we've tried
* all of them).
* NOTE: This no longer calls `rotate()` automatically -- the caller is
* responsible for deciding when to rotate. Previously this was conflated
* and caused double-rotation bugs in retry loops.
*/
markFailed(threshold?: number): ProxyEntry | null {
markFailed(threshold?: number): void {
const entry = this.getCurrent();
if (!entry) return null;
if (!entry) return;
const key = `${entry.host}:${entry.port}`;
const count = (this.failures.get(key) ?? 0) + 1;
@@ -153,11 +154,17 @@ export class ProxyPool {
const th = threshold ?? this.failureThreshold;
if (count >= th) {
console.warn(
`[proxy-pool] Proxy ${key} failed ${count}/${th} times skipping`,
`[proxy-pool] Proxy ${key} failed ${count}/${th} times -- skipping`,
);
}
}
return this.rotate();
/** Check if the current proxy has exceeded the failure threshold. */
isFailed(threshold?: number): boolean {
const entry = this.getCurrent();
if (!entry) return true;
const key = `${entry.host}:${entry.port}`;
return (this.failures.get(key) ?? 0) >= (threshold ?? this.failureThreshold);
}
/** Reset the failure counter for the current proxy. */