MythEclipse/dc-recorder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: cap AI batch size and split failed batches

Browse Source 
Reduce effective AI batch size so streaming requests finish before timeout. Keep token-based batching but cap each request to 80 messages or about 9k content tokens, and recursively split failed batches instead of marking the whole batch failed.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-05-14 04:48:20 +07:00
co-authored by Claude Opus 4.7
parent 81bb9cc6ab
commit 0eee7b9390
1 changed files with 15 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/moderation/aiAnalyzer.ts
+15 -3
View File
@@ -10,8 +10,9 @@ const queuedMessageIds = new Set<string>();
let isProcessing = false; let isProcessing = false;
let activeRequests = 0; let activeRequests = 0;
const MAX_CONCURRENT_REQUESTS = 1; const MAX_CONCURRENT_REQUESTS = 1;
const MAX_AI_REQUEST_TOKENS = 80_000; const MAX_AI_REQUEST_TOKENS = 12_000;
const AI_PROMPT_TOKEN_RESERVE = 6_000; const AI_PROMPT_TOKEN_RESERVE = 3_000;
const MAX_AI_BATCH_MESSAGES = 80;
interface ChatCompletionResponse { interface ChatCompletionResponse {
choices?: Array<{ choices?: Array<{
@@ -238,6 +239,17 @@ async function analyzeAndStoreBatch(db: SqliteDatabase, messages: MessageRecord[
if (row) (globalThis as any).broadcastMessageAnalyzed?.(row); if (row) (globalThis as any).broadcastMessageAnalyzed?.(row);
} }
} catch (error) { } catch (error) {
if (analyzableMessages.length > 1) {
const midpoint = Math.ceil(analyzableMessages.length / 2);
logger.warn(
{ count: analyzableMessages.length, nextBatchSizes: [midpoint, analyzableMessages.length - midpoint], error },
"AI batch failed, splitting into smaller batches",
);
await analyzeAndStoreBatch(db, analyzableMessages.slice(0, midpoint));
await analyzeAndStoreBatch(db, analyzableMessages.slice(midpoint));
return;
}
const errorMsg = error instanceof Error ? error.message : String(error); const errorMsg = error instanceof Error ? error.message : String(error);
for (const message of analyzableMessages) { for (const message of analyzableMessages) {
const row = updateMessageAIAnalysis(db, message.id, { const row = updateMessageAIAnalysis(db, message.id, {
@@ -276,7 +288,7 @@ async function drainQueue(db: SqliteDatabase): Promise<void> {
if (!message) continue; if (!message) continue;
const messageTokens = estimateMessageTokens(message); const messageTokens = estimateMessageTokens(message);
if (batch.length > 0 && tokenEstimate + messageTokens > batchTokenLimit) { if (batch.length > 0 && (batch.length >= MAX_AI_BATCH_MESSAGES || tokenEstimate + messageTokens > batchTokenLimit)) {
queuedMessageIds.add(messageId); queuedMessageIds.add(messageId);
break; break;
} }