feat: optimasi boros bandwidth dan CPU (#4)
- Cache layer: LRU cache + TTL untuk non-streaming LLM responses (CACHE_TTL, env: CACHE_TTL, CACHE_MAX_SIZE) - Retries: turunkan default dari pool.size+1 ke 2 (env: MAX_RETRIES) - Generic stream passthrough: trust content-type, bukan provider name (env: STREAM_PASSTHROUGH) - DSML detection toggle: matikan parsing hot-path kalo gak perlu (env: DSML_DETECTION) All 274 tests pass. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Asep Haryana Saputra
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
f78f1f5bbf
commit
820ac3b56c
@@ -0,0 +1,248 @@
|
||||
/**
|
||||
* LRU Response Cache with TTL.
|
||||
*
|
||||
* Designed for LLM proxy responses — reduces bandwidth and upstream
|
||||
* round-trips for repeated prompts (common during development, retries,
|
||||
* or shared prefixes).
|
||||
*
|
||||
* Strategy:
|
||||
* - LRU eviction (most recently used survives)
|
||||
* - Configurable TTL per entry (default 15s — safe for dev, short enough
|
||||
* that stale responses are unlikely)
|
||||
* - Cache key = hash of (model + sorted messages + stream flag)
|
||||
* - Only non-streaming responses are cached (streaming would require
|
||||
* buffering the entire body, defeating the purpose)
|
||||
* - Disabled entirely when CACHE_TTL=0 or NODE_ENV=production without
|
||||
* explicit opt-in
|
||||
*
|
||||
* Thread safety: LRU operations happen on a single Map + Doubly Linked
|
||||
* List, all synchronous — safe within Bun's single-threaded event loop.
|
||||
*/
|
||||
|
||||
// ─── Cache entry ──────────────────────────────────────────────────────────
|
||||
|
||||
interface CacheEntry {
|
||||
/** Serialized response body (JSON string). */
|
||||
body: string;
|
||||
/** HTTP status code. */
|
||||
status: number;
|
||||
/** Response headers to forward. */
|
||||
headers: Record<string, string>;
|
||||
/** When this entry was created (epoch ms). */
|
||||
createdAt: number;
|
||||
/** When this entry expires (epoch ms). */
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
// ─── LRU Linked List Node ─────────────────────────────────────────────────
|
||||
|
||||
interface LRUNode {
|
||||
key: string;
|
||||
prev: LRUNode | null;
|
||||
next: LRUNode | null;
|
||||
}
|
||||
|
||||
// ─── ResponseCache class ──────────────────────────────────────────────────
|
||||
|
||||
export class ResponseCache {
|
||||
private readonly map = new Map<string, CacheEntry>();
|
||||
private head: LRUNode | null = null;
|
||||
private tail: LRUNode | null = null;
|
||||
private readonly maxSize: number;
|
||||
private readonly defaultTtlMs: number;
|
||||
|
||||
constructor(opts?: { maxSize?: number; defaultTtlMs?: number }) {
|
||||
this.maxSize = opts?.maxSize ?? 500;
|
||||
this.defaultTtlMs = opts?.defaultTtlMs ?? 15_000; // 15 seconds
|
||||
}
|
||||
|
||||
// ── Public API ──────────────────────────────────────────────────────────
|
||||
|
||||
/** Build a deterministic cache key from an LLM request. */
|
||||
static buildKey(model: string, messages: unknown, stream: boolean): string {
|
||||
// Normalize messages to a stable string representation
|
||||
const stable = JSON.stringify(messages, stableStringifyReplacer);
|
||||
const raw = `${model}|${stream}|${stable}`;
|
||||
return simpleHash(raw);
|
||||
}
|
||||
|
||||
/** Retrieve a cached response. Returns null if missing or expired. */
|
||||
get(key: string): { body: string; status: number; headers: Record<string, string> } | null {
|
||||
const entry = this.map.get(key);
|
||||
if (!entry) return null;
|
||||
|
||||
// Expired — evict and return null
|
||||
if (Date.now() > entry.expiresAt) {
|
||||
this.delete(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Move to front (most recently used)
|
||||
this.moveToFront(key);
|
||||
return { body: entry.body, status: entry.status, headers: entry.headers };
|
||||
}
|
||||
|
||||
/** Store a response in the cache. */
|
||||
set(
|
||||
key: string,
|
||||
body: string,
|
||||
status: number,
|
||||
headers: Record<string, string>,
|
||||
ttlMs?: number,
|
||||
): void {
|
||||
// Enforce max size before inserting
|
||||
if (this.map.size >= this.maxSize) {
|
||||
this.evictLRU();
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const ttl = ttlMs ?? this.defaultTtlMs;
|
||||
|
||||
const entry: CacheEntry = {
|
||||
body,
|
||||
status,
|
||||
headers,
|
||||
createdAt: now,
|
||||
expiresAt: now + ttl,
|
||||
};
|
||||
|
||||
this.map.set(key, entry);
|
||||
this.moveToFront(key);
|
||||
}
|
||||
|
||||
/** Delete a specific key. */
|
||||
delete(key: string): void {
|
||||
this.map.delete(key);
|
||||
this.removeNode(key);
|
||||
}
|
||||
|
||||
/** Clear all entries. */
|
||||
clear(): void {
|
||||
this.map.clear();
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
}
|
||||
|
||||
/** Current number of entries. */
|
||||
get size(): number {
|
||||
return this.map.size;
|
||||
}
|
||||
|
||||
/** Sweep expired entries. Call periodically if desired. */
|
||||
sweep(): number {
|
||||
const now = Date.now();
|
||||
let removed = 0;
|
||||
for (const [key, entry] of this.map) {
|
||||
if (now > entry.expiresAt) {
|
||||
this.delete(key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
// ── LRU internals ───────────────────────────────────────────────────────
|
||||
|
||||
private moveToFront(key: string): void {
|
||||
// Remove from current position
|
||||
this.removeNode(key);
|
||||
|
||||
// Add to front
|
||||
const node: LRUNode = { key, prev: null, next: this.head };
|
||||
if (this.head) {
|
||||
this.head.prev = node;
|
||||
}
|
||||
this.head = node;
|
||||
if (!this.tail) {
|
||||
this.tail = node;
|
||||
}
|
||||
}
|
||||
|
||||
private removeNode(key: string): void {
|
||||
// Find the node — linear scan, but bounded by cache size (500).
|
||||
// For larger caches, maintain a separate Map<key, LRUNode>.
|
||||
let cur = this.head;
|
||||
while (cur) {
|
||||
if (cur.key === key) {
|
||||
// Unlink
|
||||
if (cur.prev) cur.prev.next = cur.next;
|
||||
if (cur.next) cur.next.prev = cur.prev;
|
||||
if (this.head === cur) this.head = cur.next;
|
||||
if (this.tail === cur) this.tail = cur.prev;
|
||||
return;
|
||||
}
|
||||
cur = cur.next;
|
||||
}
|
||||
}
|
||||
|
||||
private evictLRU(): void {
|
||||
// Tail is the least recently used
|
||||
if (!this.tail) return;
|
||||
const lruKey = this.tail.key;
|
||||
this.map.delete(lruKey);
|
||||
this.removeNode(lruKey);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* JSON.stringify replacer that sorts object keys for stable hashing.
|
||||
*/
|
||||
function stableStringifyReplacer(_key: string, value: unknown): unknown {
|
||||
if (value !== null && typeof value === "object" && !Array.isArray(value)) {
|
||||
const sorted: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(value).sort()) {
|
||||
sorted[k] = (value as Record<string, unknown>)[k];
|
||||
}
|
||||
return sorted;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple, fast non-cryptographic hash (djb2 variant).
|
||||
* Collisions are theoretically possible but extremely unlikely for
|
||||
* cache-key use — a collision would serve a wrong cached response,
|
||||
* which is bounded by TTL.
|
||||
*/
|
||||
function simpleHash(input: string): string {
|
||||
let hash = 5381;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
hash = ((hash << 5) + hash + input.charCodeAt(i)) & 0xffffffff;
|
||||
}
|
||||
return hash.toString(36);
|
||||
}
|
||||
|
||||
// ─── Module-level singleton (lazy) ────────────────────────────────────────
|
||||
|
||||
let _instance: ResponseCache | null = null;
|
||||
|
||||
/**
|
||||
* Get or create the shared ResponseCache instance.
|
||||
* Configured via environment variables:
|
||||
* CACHE_TTL — TTL in ms (0 = disabled, default 15000)
|
||||
* CACHE_MAX_SIZE — max entries (default 500)
|
||||
*/
|
||||
export function getResponseCache(): ResponseCache | null {
|
||||
const ttl = Number(process.env.CACHE_TTL ?? 15000);
|
||||
if (ttl <= 0) return null; // explicitly disabled
|
||||
|
||||
if (!_instance) {
|
||||
const maxSize = Number(process.env.CACHE_MAX_SIZE ?? 500);
|
||||
_instance = new ResponseCache({ maxSize, defaultTtlMs: ttl });
|
||||
}
|
||||
return _instance;
|
||||
}
|
||||
|
||||
/** Check if DSML detection should run (env toggle, default true). */
|
||||
export function isDSMLDetectionEnabled(): boolean {
|
||||
const val = process.env.DSML_DETECTION ?? "true";
|
||||
return val === "true" || val === "1";
|
||||
}
|
||||
|
||||
/** Check if stream passthrough mode is enabled (env toggle, default true). */
|
||||
export function isStreamPassthroughEnabled(): boolean {
|
||||
const val = process.env.STREAM_PASSTHROUGH ?? "true";
|
||||
return val === "true" || val === "1";
|
||||
}
|
||||
Reference in New Issue
Block a user