From 0eee7b9390b2fdcbb67b59af1b68d199dcf70c5e Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Thu, 14 May 2026 04:48:20 +0700 Subject: [PATCH] fix: cap AI batch size and split failed batches Reduce effective AI batch size so streaming requests finish before timeout. Keep token-based batching but cap each request to 80 messages or about 9k content tokens, and recursively split failed batches instead of marking the whole batch failed. Co-Authored-By: Claude Opus 4.7 --- src/moderation/aiAnalyzer.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/moderation/aiAnalyzer.ts b/src/moderation/aiAnalyzer.ts index dc580a2..b96d953 100644 --- a/src/moderation/aiAnalyzer.ts +++ b/src/moderation/aiAnalyzer.ts @@ -10,8 +10,9 @@ const queuedMessageIds = new Set(); let isProcessing = false; let activeRequests = 0; const MAX_CONCURRENT_REQUESTS = 1; -const MAX_AI_REQUEST_TOKENS = 80_000; -const AI_PROMPT_TOKEN_RESERVE = 6_000; +const MAX_AI_REQUEST_TOKENS = 12_000; +const AI_PROMPT_TOKEN_RESERVE = 3_000; +const MAX_AI_BATCH_MESSAGES = 80; interface ChatCompletionResponse { choices?: Array<{ @@ -238,6 +239,17 @@ async function analyzeAndStoreBatch(db: SqliteDatabase, messages: MessageRecord[ if (row) (globalThis as any).broadcastMessageAnalyzed?.(row); } } catch (error) { + if (analyzableMessages.length > 1) { + const midpoint = Math.ceil(analyzableMessages.length / 2); + logger.warn( + { count: analyzableMessages.length, nextBatchSizes: [midpoint, analyzableMessages.length - midpoint], error }, + "AI batch failed, splitting into smaller batches", + ); + await analyzeAndStoreBatch(db, analyzableMessages.slice(0, midpoint)); + await analyzeAndStoreBatch(db, analyzableMessages.slice(midpoint)); + return; + } + const errorMsg = error instanceof Error ? error.message : String(error); for (const message of analyzableMessages) { const row = updateMessageAIAnalysis(db, message.id, { @@ -276,7 +288,7 @@ async function drainQueue(db: SqliteDatabase): Promise { if (!message) continue; const messageTokens = estimateMessageTokens(message); - if (batch.length > 0 && tokenEstimate + messageTokens > batchTokenLimit) { + if (batch.length > 0 && (batch.length >= MAX_AI_BATCH_MESSAGES || tokenEstimate + messageTokens > batchTokenLimit)) { queuedMessageIds.add(messageId); break; }