feat: optimasi boros bandwidth dan CPU (#4)

- Cache layer: LRU cache + TTL untuk non-streaming LLM responses
  (CACHE_TTL, env: CACHE_TTL, CACHE_MAX_SIZE)
- Retries: turunkan default dari pool.size+1 ke 2 (env: MAX_RETRIES)
- Generic stream passthrough: trust content-type, bukan provider name
  (env: STREAM_PASSTHROUGH)
- DSML detection toggle: matikan parsing hot-path kalo gak perlu
  (env: DSML_DETECTION)

All 274 tests pass.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Asep Haryana Saputra
2026-06-27 15:53:42 +07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent f78f1f5bbf
commit 820ac3b56c
4 changed files with 321 additions and 11 deletions
+6 -4
View File
@@ -238,9 +238,10 @@ export async function fetchWithRetry(
let usedProxy = false;
// Strategy: direct first, then proxies as fallback.
// maxAttempts = 1 direct + pool.size proxies, min 3 (all direct if no pool).
// Default: 2 attempts (1 direct + 1 proxy fallback). Override via MAX_RETRIES env.
const MAX_RETRIES = Number(process.env.MAX_RETRIES ?? 2);
const poolSize = proxyPool?.size ?? 0;
const maxAttempts = poolSize > 0 ? poolSize + 1 : 3;
const maxAttempts = Math.min(MAX_RETRIES, poolSize > 0 ? poolSize + 1 : MAX_RETRIES);
for (let attempt = 0; attempt < maxAttempts; attempt++) {
// Exponential backoff between attempts (skip on first attempt)
@@ -407,8 +408,9 @@ export async function fetchWithSessionRetry(
}
// Strategy: direct first, then session-sticky proxies as fallback.
// totalAttempts = 1 direct + sessionPool.size proxies, min 3.
const totalAttempts = maxRetries ?? Math.max(3, sessionPool.size + 1);
// Default: 2 attempts (1 direct + 1 proxy fallback). Override via MAX_RETRIES env.
const MAX_RETRIES = Number(process.env.MAX_RETRIES ?? 2);
const totalAttempts = maxRetries ?? Math.min(MAX_RETRIES, Math.max(2, sessionPool.size + 1));
let lastError: unknown;
let lastResponse: Response | undefined;
for (let attempt = 0; attempt < totalAttempts; attempt++) {