From 4cf5b87f2b8d1dec7d8a1a13d724b259b8dd8ff3 Mon Sep 17 00:00:00 2001 From: asepharyana Date: Sun, 16 Aug 2026 19:00:29 +0700 Subject: [PATCH] perf(ai-moderation): pack more messages per LLM request (fewer API calls when busy) User insight: rather than many small per-batch API requests, pack many messages into ONE request so a burst is analyzed with far fewer calls. - AI_LLM_TEXT_BATCH_SIZE 20 -> 60 (one request now carries ~3x more messages). - AI_ANALYSIS_MAX_TARGET_TOKENS 4000 -> 14000 (the scheduler's token-budget gate was trimming pending messages to ~20 before they reached the sub-batch splitter; raising it lets ~60 messages through to a single LLM call). - AI_LLM_TEXT_ANALYSIS_TIMEOUT_MS 30000 -> 45000 (one larger call needs more headroom; gemini-flash-lite has a 1M-token context so 14k+8k is trivial). Net effect when ramai: a 60-message burst = 1-2 API calls instead of 3+, less semaphore contention, faster throughput. --- services/discord-gateway/src/shared/config/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/discord-gateway/src/shared/config/index.ts b/services/discord-gateway/src/shared/config/index.ts index b692e0b..c2d47df 100644 --- a/services/discord-gateway/src/shared/config/index.ts +++ b/services/discord-gateway/src/shared/config/index.ts @@ -183,7 +183,7 @@ export const configSchema = z .int() .positive() .default(1024), - AI_LLM_TEXT_BATCH_SIZE: z.coerce.number().int().positive().default(20), + AI_LLM_TEXT_BATCH_SIZE: z.coerce.number().int().positive().default(60), AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS: z.coerce .number() .int() @@ -205,7 +205,7 @@ export const configSchema = z .number() .int() .positive() - .default(30000), + .default(45000), // Term glossary — per-word Wikipedia lookups (via SearXNG) for words the // LLM may not know (slang, jargon, regional language, foreign terms). // Definitions are cached (in-memory + Redis) so repeat lookups are fast. @@ -236,7 +236,7 @@ export const configSchema = z // ── AI Analysis Batch ─────────────────────────────────────────────── AI_ANALYSIS_MAX_BATCH_SIZE: z.coerce.number().int().positive().default(200), AI_ANALYSIS_MAX_CONTEXT_TOKENS: z.coerce.number().positive().default(8000), - AI_ANALYSIS_MAX_TARGET_TOKENS: z.coerce.number().positive().default(4000), + AI_ANALYSIS_MAX_TARGET_TOKENS: z.coerce.number().positive().default(14000), AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT: z.coerce .number() .int()