From a643125c7bb62df424c15e86a634dcf86cf6f3ff Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Mon, 1 Jun 2026 18:50:05 +0700 Subject: [PATCH] refactor(moderation): improve LLM moderation client - 10 recommendations - Add XML delimiters to prevent prompt injection (R1) - Use JSON Schema response format instead of json_object (R2) - Add concurrency limiter via p-limit (R3) - Add timeout per media analysis call (R4) - Resize images with sharp before vision API (R5) - Split text batches when exceeding batch size limit (R6) - Add few-shot examples to system prompt (R7) - Modularize system prompt builder (R8) - Enhance deferral detection regex with exception patterns (R9) - Sanitize error messages to avoid leaking internals (R10) New files: concurrencyLimiter.ts, imageResizer.ts, moderationPrompt.ts Updated: llmModerationClient.ts, config.ts, package.json, tests Co-Authored-By: Claude Opus 4.8 --- package.json | 2 + pnpm-lock.yaml | 43 +- src/config.ts | 20 +- src/database/migrateCli.ts | 2 +- src/moderation/concurrencyLimiter.ts | 14 + src/moderation/imageResizer.ts | 58 ++ src/moderation/llmModerationClient.ts | 535 +++++++++++-------- src/moderation/messageMetadata.ts | 4 +- src/moderation/moderationPrompt.ts | 161 ++++++ src/moderation/stickerPrompt.ts | 4 +- src/routes/appConfigRoutes.ts | 2 +- tests/moderation/llmModerationClient.test.ts | 107 ++-- 12 files changed, 663 insertions(+), 289 deletions(-) create mode 100644 src/moderation/concurrencyLimiter.ts create mode 100644 src/moderation/imageResizer.ts create mode 100644 src/moderation/moderationPrompt.ts diff --git a/package.json b/package.json index 4196f43..7f013cf 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "motion": "^12.40.0", "openai": "^6.38.0", "opusscript": "^0.0.8", + "p-limit": "^7.3.0", "p-retry": "^8.0.0", "pg": "^8.21.0", "piscina": "^5.1.4", @@ -57,6 +58,7 @@ "prom-client": "^15.1.3", "react": "^19.2.6", "react-dom": "^19.2.6", + "sharp": "^0.34.5", "tailwind-merge": "^3.6.0", "vite": "^8.0.13", "winston": "^3.19.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cbd32cd..fc4188d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -37,7 +37,7 @@ importers: version: 8.20.0 '@vitejs/plugin-react': specifier: ^6.0.2 - version: 6.0.2(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)) + version: 6.0.2(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0)) axios: specifier: ^1.16.1 version: 1.16.1 @@ -77,6 +77,9 @@ importers: opusscript: specifier: ^0.0.8 version: 0.0.8 + p-limit: + specifier: ^7.3.0 + version: 7.3.0 p-retry: specifier: ^8.0.0 version: 8.0.0 @@ -101,12 +104,15 @@ importers: react-dom: specifier: ^19.2.6 version: 19.2.6(react@19.2.6) + sharp: + specifier: ^0.34.5 + version: 0.34.5 tailwind-merge: specifier: ^3.6.0 version: 3.6.0 vite: specifier: ^8.0.13 - version: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2) + version: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0) winston: specifier: ^3.19.0 version: 3.19.0 @@ -158,7 +164,7 @@ importers: version: 5.9.3 vitest: specifier: latest - version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.0)(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)) + version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.0)(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0)) vendor/discord-video-stream: dependencies: @@ -3800,6 +3806,10 @@ packages: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} + p-limit@7.3.0: + resolution: {integrity: sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==} + engines: {node: '>=20'} + p-locate@3.0.0: resolution: {integrity: sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==} engines: {node: '>=6'} @@ -4973,6 +4983,10 @@ packages: resolution: {integrity: sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==} engines: {node: '>=8'} + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + yoctocolors@2.1.2: resolution: {integrity: sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==} engines: {node: '>=18'} @@ -6373,10 +6387,10 @@ snapshots: dependencies: '@types/node': 25.8.0 - '@vitejs/plugin-react@6.0.2(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2))': + '@vitejs/plugin-react@6.0.2(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0))': dependencies: '@rolldown/pluginutils': 1.0.1 - vite: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2) + vite: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0) '@vitest/expect@4.1.7': dependencies: @@ -6387,13 +6401,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.7(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2))': + '@vitest/mocker@4.1.7(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.7 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2) + vite: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0) '@vitest/pretty-format@4.1.7': dependencies: @@ -8118,6 +8132,10 @@ snapshots: dependencies: p-try: 2.2.0 + p-limit@7.3.0: + dependencies: + yocto-queue: 1.2.2 + p-locate@3.0.0: dependencies: p-limit: 2.3.0 @@ -9052,7 +9070,7 @@ snapshots: vary@1.1.2: {} - vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2): + vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -9065,11 +9083,12 @@ snapshots: fsevents: 2.3.3 jiti: 2.7.0 tsx: 4.22.2 + yaml: 2.9.0 - vitest@4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.0)(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)): + vitest@4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.0)(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.7 - '@vitest/mocker': 4.1.7(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)) + '@vitest/mocker': 4.1.7(vite@8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.7 '@vitest/runner': 4.1.7 '@vitest/snapshot': 4.1.7 @@ -9086,7 +9105,7 @@ snapshots: tinyexec: 1.1.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2) + vite: 8.0.13(@types/node@25.9.0)(esbuild@0.28.0)(jiti@2.7.0)(tsx@4.22.2)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.1 @@ -9294,6 +9313,8 @@ snapshots: y18n: 4.0.3 yargs-parser: 18.1.3 + yocto-queue@1.2.2: {} + yoctocolors@2.1.2: {} zeromq@6.5.0: diff --git a/src/config.ts b/src/config.ts index bb976d6..f5a4076 100644 --- a/src/config.ts +++ b/src/config.ts @@ -75,6 +75,22 @@ const configSchema = z AI_LLM_MODEL: z.string().default("text"), /** Model used for image/video moderation (vision-capable model). */ AI_LLM_VISION_MODEL: z.string().optional(), + /** Max concurrent LLM API calls (default: 5). */ + AI_LLM_MAX_CONCURRENT: z.coerce.number().int().positive().default(5), + /** Maximum image dimension in pixels before resize for vision API (default: 1024). */ + AI_LLM_IMAGE_MAX_DIMENSION: z.coerce + .number() + .int() + .positive() + .default(1024), + /** Maximum messages per text-only moderation batch (default: 20). */ + AI_LLM_TEXT_BATCH_SIZE: z.coerce.number().int().positive().default(20), + /** Timeout in ms for individual media analysis calls (default: 60000). */ + AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS: z.coerce + .number() + .int() + .positive() + .default(60000), AI_ANALYSIS_DEBOUNCE_MS: z.coerce.number().positive().default(500), AI_ANALYSIS_RECOVERY_INTERVAL_MS: z.coerce .number() @@ -149,7 +165,9 @@ const configSchema = z .transform((v) => v === "true") .default(false), AUTO_DELETE_MIN_CONFIDENCE: z.coerce.number().min(0).max(1).default(0.5), - AUTO_DELETE_ALLOWED_SEVERITIES: z.string().default("critical,high,medium,low"), + AUTO_DELETE_ALLOWED_SEVERITIES: z + .string() + .default("critical,high,medium,low"), AUTO_DELETE_ALLOWED_CATEGORIES: z.string().default(""), AUTO_DELETE_EXCLUDED_CHANNEL_IDS: z.string().default(""), AUTO_DELETE_EXCLUDED_USER_IDS: z.string().default(""), diff --git a/src/database/migrateCli.ts b/src/database/migrateCli.ts index 244b779..65b8aaf 100644 --- a/src/database/migrateCli.ts +++ b/src/database/migrateCli.ts @@ -11,4 +11,4 @@ runMigrations() .catch((error) => { logger.error({ error }, "Migration failed"); process.exit(1); - }); \ No newline at end of file + }); diff --git a/src/moderation/concurrencyLimiter.ts b/src/moderation/concurrencyLimiter.ts new file mode 100644 index 0000000..8bf220a --- /dev/null +++ b/src/moderation/concurrencyLimiter.ts @@ -0,0 +1,14 @@ +import pLimit from "p-limit"; +import { config } from "../config.js"; + +/** + * Concurrency limiter for LLM API calls. + * + * Prevents rate-limit (429) errors by capping simultaneous requests + * to the configured maximum (default: 5). + */ +const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5); + +export async function withLlmConcurrency(fn: () => Promise): Promise { + return llmSemaphore(fn); +} diff --git a/src/moderation/imageResizer.ts b/src/moderation/imageResizer.ts new file mode 100644 index 0000000..0c11ecc --- /dev/null +++ b/src/moderation/imageResizer.ts @@ -0,0 +1,58 @@ +import sharp from "sharp"; +import { createChildLogger } from "../logger.js"; + +const log = createChildLogger("imageResizer"); + +/** + * Resize an image buffer for optimal vision LLM analysis. + * + * - Resizes to maxDim x maxDim maintaining aspect ratio + * - Converts to JPEG at quality 85 for size reduction + * - Falls back to original buffer if sharp fails + * + * @param buf - Raw image buffer + * @param maxDim - Maximum dimension in pixels (default 1024) + * @returns Resized buffer with detected MIME type + */ +export async function resizeImageForVision( + buf: Buffer, + maxDim = 1024, +): Promise<{ data: Buffer; mimeType: string }> { + try { + const metadata = await sharp(buf).metadata(); + const inputFormat = metadata.format ?? "jpeg"; + + // Skip resize if already smaller than maxDim + if ((metadata.width ?? 0) <= maxDim && (metadata.height ?? 0) <= maxDim) { + return { data: buf, mimeType: `image/${inputFormat}` }; + } + + const resized = await sharp(buf) + .resize(maxDim, maxDim, { + fit: "inside", + withoutEnlargement: true, + }) + .jpeg({ quality: 85 }) + .toBuffer(); + + log.debug( + { + originalSize: buf.length, + resizedSize: resized.length, + reductionPct: Math.round( + ((buf.length - resized.length) / buf.length) * 100, + ), + }, + "Image resized for vision analysis", + ); + + return { data: resized, mimeType: "image/jpeg" }; + } catch (error) { + log.warn( + { error: error instanceof Error ? error.message : String(error) }, + "Image resize failed — using original buffer", + ); + // Fallback: return original buffer with best-effort MIME type + return { data: buf, mimeType: "image/jpeg" }; + } +} diff --git a/src/moderation/llmModerationClient.ts b/src/moderation/llmModerationClient.ts index 7df10df..1fd307e 100644 --- a/src/moderation/llmModerationClient.ts +++ b/src/moderation/llmModerationClient.ts @@ -4,8 +4,11 @@ import { z } from "zod"; import { config } from "../config.js"; import { createChildLogger } from "../logger.js"; import { retryWithBackoff } from "../retry.js"; +import { withLlmConcurrency } from "./concurrencyLimiter.js"; import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js"; +import { resizeImageForVision } from "./imageResizer.js"; import { extractMessageMediaEvidence } from "./messageMetadata.js"; +import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js"; import { getStickerFromCache, initStickerCache, @@ -60,10 +63,23 @@ const ModerationResponseSchema = z.object({ }); const log = createChildLogger("llmModerationClient"); + +/** + * Enhanced deferral detection pattern (R9). + * Covers more variations and multi-word combinations that the LLM might use + * to defer judgment instead of making a decision. + */ const DEFERRAL_ANALYSIS_PATTERN = - /kurang konteks|kekurangan konteks|perlu (dicek|diperiksa|ditinjau).*(admin|moderator)|admin perlu|moderator perlu|tidak bisa menentukan|tidak dapat menentukan|cannot determine|insufficient context/i; + /(?:kurang (?:konteks|bukti|informasi|data)|kekurangan (?:konteks|bukti)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi).*(?:admin|moderator|manusia|human)|admin (?:perlu|harus|sebaiknya)|moderator (?:perlu|harus|sebaiknya)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|mengevaluasi)|cannot determine|insufficient (?:context|evidence|information)|(?:mungkin|sepertinya|tampaknya) (?:perlu|harus|sebaiknya) (?:dicek|diperiksa|ditinjau)|tidak (?:cukup|memadai) (?:bukti|informasi|konteks)|bisa (?:berpotensi|mengandung)|(?:(?:maaf|sorry|抱歉|ขออภัย))[,.\s]|(?:saya (?:tidak|kurang|belum) (?:yakin|pasti|tahu|paham)))/i; + +/** + * Exceptions: patterns that look like deferral but are actually decisive. + */ +const DEFERRAL_EXCEPTION_PATTERN = + /tidak bisa menentukan.*(?:karena|sebab|dengan alasan).*(?:clean|tidak (?:ada|terdapat).*(?:pelanggaran|masalah)|aman)/i; function hasDeferralAnalysis(analysis: string): boolean { + if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false; return DEFERRAL_ANALYSIS_PATTERN.test(analysis); } @@ -95,17 +111,69 @@ function deriveRecommendedAction( return "review"; } +/** + * JSON Schema for OpenAI's response_format: { type: "json_schema" }. + * This enforces the exact structure the LLM must output (R2). + */ +const MODERATION_JSON_SCHEMA = { + type: "object", + properties: { + results: { + type: "array", + items: { + type: "object", + properties: { + message_id: { type: "string" }, + status: { type: "string", enum: ["clean", "warn", "flagged"] }, + flags: { type: "array", items: { type: "string" } }, + score: { type: "number", minimum: 0, maximum: 1 }, + analysis: { type: "string" }, + categories: { type: "array", items: { type: "string" } }, + severity: { + type: "string", + enum: ["none", "low", "medium", "high", "critical"], + }, + confidence: { type: "number", minimum: 0, maximum: 1 }, + recommended_action: { + type: "string", + enum: ["none", "monitor", "warn", "review", "delete", "escalate"], + }, + policy_version: { type: "string" }, + evidence: { type: "array", items: { type: "string" } }, + }, + required: [ + "message_id", + "status", + "flags", + "score", + "severity", + "confidence", + "recommended_action", + "policy_version", + "evidence", + "analysis", + ], + additionalProperties: false, + }, + }, + }, + required: ["results"], + additionalProperties: false, +}; + +// --------------------------------------------------------------------------- +// OpenAI client with Cloudflare WAF bypass (unchanged) +// --------------------------------------------------------------------------- + const openai = new OpenAI({ apiKey: config.AI_LLM_API_KEY, baseURL: config.AI_LLM_BASE_URL, maxRetries: 0, timeout: 30000, fetch: async (url, init) => { - // Add internal timeout for the global fetch as safety const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 30000); - // Override headers to bypass Cloudflare WAF Bot Fight Mode const headers = new Headers(init?.headers); headers.set( "User-Agent", @@ -144,13 +212,13 @@ const openai = new OpenAI({ } } - const headers = new Headers(response.headers ?? undefined); - headers.set("Content-Type", "application/json"); - headers.delete("Content-Length"); + const responseHeaders = new Headers(response.headers ?? undefined); + responseHeaders.set("Content-Type", "application/json"); + responseHeaders.delete("Content-Length"); return new Response(normalizedBody, { status: response.status ?? 200, - headers, + headers: responseHeaders, }); } finally { clearTimeout(timeout); @@ -226,6 +294,20 @@ export function extractJson(content: string): any { throw new Error("No JSON object found in response"); } +/** + * Sanitize error messages for client-facing output (R10). + * Internal details are logged but the caller gets a generic message. + */ +function sanitizeErrorMessage(internalMsg: string, messageId: string): string { + // Log the full error for debugging + log.warn( + { messageId, internalError: internalMsg }, + "Internal moderation error (sanitized for client)", + ); + // Return generic message without internal details + return `Analisis gagal dan memerlukan pemeriksaan manual. Error code: MOD_${Date.now().toString(36).slice(0, 6)}`; +} + export function parseModerationResponse( content: string, targetIds: string[], @@ -240,13 +322,9 @@ export function parseModerationResponse( if (Array.isArray(parsed)) { parsed = { results: parsed }; } else if (parsed && typeof parsed === "object" && !("results" in parsed)) { - // If the object directly looks like a result item (has message_id), wrap it - // BEFORE checking for array keys. This prevents flags:[] from being - // mistaken as the results array on a single-object response. if ("message_id" in parsed) { parsed = { results: [parsed] }; } else { - // Find the first non-empty array key whose elements are objects with message_id const arrayKey = Object.keys(parsed).find((key) => { const val = (parsed as any)[key]; return ( @@ -351,7 +429,10 @@ export function parseModerationResponse( status: "error", flags: ["analysis_incomplete"], score: 0, - analysis: "Analysis incomplete - LLM did not process this message", + analysis: sanitizeErrorMessage( + "Analysis incomplete - LLM did not process this message", + missingId, + ), categories: ["analysis_incomplete"], severity: "none", confidence: 0, @@ -380,23 +461,14 @@ interface ModerationOutput { * Sniff the first bytes of a buffer to determine if it is a supported image * format. Returns the canonical MIME type string on success, or null if the * bytes are not a recognizable image. - * - * Supported probes (in order): - * - JPEG: FF D8 FF - * - PNG: 89 50 4E 47 0D 0A 1A 0A - * - GIF: 47 49 46 38 (GIF8) - * - WebP: 52 49 46 46 ?? ?? ?? ?? 57 45 42 50 (RIFF....WEBP) - * - AVIF / HEIF: 4-byte big-endian size + 66 74 79 70 (ftyp ISO base-media box) */ function sniffImageMimeType(buf: Buffer): string | null { if (buf.length < 12) return null; - // JPEG if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) { return "image/jpeg"; } - // PNG if ( buf[0] === 0x89 && buf[1] === 0x50 && @@ -410,7 +482,6 @@ function sniffImageMimeType(buf: Buffer): string | null { return "image/png"; } - // GIF if ( buf[0] === 0x47 && buf[1] === 0x49 && @@ -420,7 +491,6 @@ function sniffImageMimeType(buf: Buffer): string | null { return "image/gif"; } - // WebP: RIFF????WEBP if ( buf[0] === 0x52 && buf[1] === 0x49 && @@ -434,7 +504,6 @@ function sniffImageMimeType(buf: Buffer): string | null { return "image/webp"; } - // AVIF / HEIF: ISO base media file format — ftyp box at offset 4 if ( buf.length >= 12 && buf[4] === 0x66 && @@ -475,10 +544,6 @@ type MessageImagePart = { // Media detection helper // --------------------------------------------------------------------------- -/** - * Returns true when a target message has any media evidence that requires - * image download + vision analysis before LLM evaluation. - */ function hasMediaContent( target: MessageRecord, attachments?: AttachmentRecord[], @@ -518,27 +583,29 @@ const analyzeSingleMediaImage = async ( : `Analisis media Discord berikut sebagai evidence moderasi. ${image.sourceLabel}\nJelaskan isi visual, teks yang terlihat, konteks risiko, dan apakah ada indikasi spam, scam, SARA, harassment, sexual content, violence, self-harm, doxxing, NSFW, gore, atau illegal content. Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek; berikan observasi langsung dari media.`; try { - const completion = await openai.chat.completions.create({ - model: config.AI_LLM_VISION_MODEL ?? config.AI_LLM_MODEL, - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: promptText, - }, - { type: "image_url", image_url: image.image_url }, - ], - }, - ], - temperature: 0.1, - top_p: 0.9, - max_tokens: 500, - stream: false, - chat_template_kwargs: { enable_thinking: false }, - reasoning_budget: 0, - } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming); + const completion = await withLlmConcurrency(async () => + openai.chat.completions.create({ + model: config.AI_LLM_VISION_MODEL ?? config.AI_LLM_MODEL, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: promptText, + }, + { type: "image_url", image_url: image.image_url }, + ], + }, + ], + temperature: 0.1, + top_p: 0.9, + max_tokens: 500, + stream: false, + chat_template_kwargs: { enable_thinking: false }, + reasoning_budget: 0, + } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming), + ); const content = completion.choices[0]?.message?.content?.trim(); if (!content) return null; @@ -564,130 +631,39 @@ const analyzeSingleMediaImage = async ( }; // --------------------------------------------------------------------------- -// System prompt builder +// Shared LLM call + parse + fallback helper // --------------------------------------------------------------------------- /** - * Build the system prompt for the main moderation LLM call. + * State object shared between the caller and callModerationLLM so that + * parse-error feedback can be injected into subsequent retry attempts. * - * @param contextText Conversation context lines. - * @param includeMediaInstructions Whether to inject image/sticker analysis - * instructions. Set to `false` for text-only - * batches (no media evidence present). - * @param correction Previous parse error info (for retry with feedback). + * The caller creates this object, passes it to callModerationLLM, and + * the internal retry loop mutates it before re-invoking buildContent(). */ -function buildSystemPrompt( - contextText: string, - includeMediaInstructions: boolean, - correction?: { error: string; preview: string }, -): string { - const imageInstructions = includeMediaInstructions - ? ` -## Instruksi Analisis Media -Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama. -Gunakan baris "Media analysis" sebagai evidence visual utama dalam keputusan moderasi batch ini. - -## Panduan Khusus Sticker -- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata. -- Sticker sering bersifat humor, satir, atau ekspresi emosi yang dilebih-lebihkan. -- Gambar sticker bisa menampilkan adegan kartun yang terlihat "keras" (tokoh kartun menginjak sesuatu, ledakan komik, senjata kartun) — itu SENI KARTUN, bukan dokumentasi kekerasan nyata. -- Nama sticker yang terdengar provokatif (mis. "Singa injek pejabat", "Bom atom", dll) adalah konteks satir/humor. JANGAN flag "violence", "harassment", atau "sara" berdasarkan nama sticker saja tanpa melihat gambar. -- Jika sticker evidence hanya tersedia sebagai nama (gambar gagal diunduh), abaikan sebagai evidence pelanggaran — nama sticker saja TIDAK cukup untuk flag. -- Terapkan standar yang lebih longgar untuk konten kartun/meme dibanding foto/video nyata. - -Sticker yang berhasil diunduh WAJIB diperlakukan sebagai image evidence, bukan sekadar nama sticker. -Jangan abaikan link: gunakan isi web, preview image, atau hasil analisis media link bila tersedia. -` - : ""; - - const base = `Kamu adalah asisten moderasi konten untuk server Discord berbahasa Indonesia. -Bahasa utama komunitas ini adalah BAHASA INDONESIA. Bahasa Inggris adalah bahasa sekunder. - -## Konteks Server -Ini adalah server Discord komunitas Indonesia. Kamu harus memahami: -- Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll. -- Singkatan umum: "gw", "lo", "emg", "kyk", "tdk", "krn", "jgn", dll. -- Konteks budaya lokal: SARA (Suku, Agama, Ras, Antar-golongan), hoaks, ujaran kebencian berbasis konteks Indonesia. -- Makian/kata kasar umum (seperti "anjing", "asu", "bangsat") BUKAN pelanggaran SARA. SARA khusus untuk diskriminasi/hinaan terhadap Suku, Agama, Ras, dan Antargolongan. NAMUN makian/kata kasar TETAP bisa di-flag sebagai "harassment" atau "vulgar_language" HANYA jika: (1) ditujukan langsung ke orang lain sebagai serangan/hinaan, (2) dalam tone agresif/mengancam, atau (3) bagian dari pola harassment berkelanjutan. Jangan flag sekadar karena kata itu muncul dalam teks. Jangan flag sebagai SARA, tapi flag sesuai kategori yang tepat. -- Kata "asus" adalah merk teknologi, jangan pernah dianggap sebagai makian "asu". -- Perbedaan antara humor/banter biasa vs konten yang benar-benar melanggar. -- "woy"/"woi" adalah sapaan/interjeksi informal Indonesia dan tidak boleh dianggap SARA, hate speech, atau harassment tanpa target hinaan/ancaman jelas. -- Kata-kata AMAN yang TIDAK BOLEH di-flag: "kakek" (kakek = grandfather, family term), "Wah" (exclamation), "hadeh" (slang exclamation), dan kata-kata keluarga/interjeksi umum lainnya. Jika kata ini muncul dalam teks, JANGAN flag sebagai vulgar_language atau harassment. -- Discord custom emoji seperti <:hadeh:123> atau [emoji:hadeh] adalah ekspresi/emoji, bukan pelanggaran teks. Gunakan sebagai konteks ekspresi saja. -- Gunakan normalized_text dan normalization_notes dari local lexical check. Jika notes hanya berisi slang/emoji aman (woy, woi, hadeh, dll) dan "no Indonesian badword detected", jangan flag karena kata slang itu saja. NAMUN jika notes menyatakan "Indonesian badword detected" (misalnya "anjing", "bangsat", "asu"), itu EVIDENCE profanitas — gunakan sebagai konteks untuk menilai apakah perlu flag sebagai harassment/vulgar_language, bukan sebagai alasan untuk mengabaikan. -- Topik seksual/identitas yang dibatasi server: LGBT/LGBTQ, furry/transfurry, therian, otherkin, protogen, yiff/fursona/fursuit, dan istilah terkait. Jika pesan mempromosikan, mendukung, mengajak, menyatakan identitas/roleplay, membagikan media, atau menjadikan topik ini sebagai pembahasan utama, flag sebagai "sexual_deviation". Jika pesan hanya mengecam/menolak topik tersebut tanpa hinaan ke orang/kelompok, status bisa "clean" atau "warn" sesuai tone. Jangan gunakan kebijakan ini untuk membenarkan doxxing, ancaman, atau penghinaan personal; ancaman/hinaan tetap flag sebagai harassment/hate_speech juga. -- Kalimat ambigu dalam bahasa Indonesia harus diberi keputusan final: "clean" bila bukti pelanggaran tidak jelas, "flagged" bila bukti pelanggaran jelas. -- Jangan pernah menulis analisis yang meminta admin/moderator memeriksa ulang, menyebut kurang konteks, atau tidak bisa menentukan. Berikan kesimpulan langsung berdasarkan teks + media + konteks yang tersedia. -- Gambar, sticker, embed, dan preview link adalah evidence utama yang setara dengan teks, bukan sekadar URL teks. -- Pornografi/NSFW, hentai, bokep, ajakan seksual, roleplay seksual, atau istilah seksual eksplisit harus di-flag sebagai "sexual_content"; jika melibatkan anak/di bawah umur/loli/shota/CP/pedofil, flag sebagai "child_safety" dan "illegal_content". -- Judi/slot/togel/casino/parlay/maxwin/RTP/deposit/withdraw dalam konteks promosi atau ajakan harus di-flag sebagai "gambling" dan bila spam/scam juga tambahkan "spam" atau "scam". -- Narkoba/obat terlarang/ganja/sabu/kokain/ekstasi dalam konteks jual beli, promosi, atau ajakan penggunaan harus di-flag sebagai "drugs". -- Ancaman kekerasan, ajakan bunuh diri, self-harm, doxxing, scam finansial/crypto/phishing, dan spam self-promo harus diprioritaskan walau teksnya bercampur slang bercanda. -- Istilah agama/suku/ras harus dinilai hati-hati: penyebutan netral/ibadah/edukasi = clean; hinaan, generalisasi negatif, provokasi, atau ajakan diskriminatif = flag "sara", "hate_speech", atau "religious_insult" sesuai konteks. -${imageInstructions} -## Konteks Percakapan -${contextText} - -## Format Output -Balas HANYA dengan satu objek JSON valid. Tanpa markdown, tanpa prose, tanpa komentar, tanpa XML. -Struktur wajib: -{ - "results": [ - { - "message_id": "", - "status": "clean" | "warn" | "flagged", - "flags": [], - "score": , - "categories": [], - "severity": "none" | "low" | "medium" | "high" | "critical", - "confidence": , - "recommended_action": "none" | "monitor" | "warn" | "review" | "delete" | "escalate", - "policy_version": "default-2026-05-30", - "evidence": [], - "analysis": "" - } - ] +interface RetryState { + lastParseError: string | null; + lastInvalidContent: string | null; } -Kriteria status: -- "clean": tidak ada pelanggaran yang terdeteksi, atau kasus masih ambigu setelah semua evidence dianalisis -- "warn": risiko ringan yang konkret terdeteksi, misalnya spam borderline atau harassment ringan; BUKAN untuk kurang konteks/perlu admin cek -- "flagged": pelanggaran jelas terdeteksi - -Larangan output analysis: -- Jangan tulis "kurang konteks", "perlu dicek admin", "perlu moderator periksa", "tidak bisa menentukan", atau frasa deferral sejenis. -- Jika evidence tidak cukup kuat untuk pelanggaran, status harus "clean" dan analysis menjelaskan alasan langsung. - -Flag yang valid: spam, hate_speech, sara, hoaks, harassment, vulgar_language, sexual_content, sexual_deviation, violence, self_harm, doxxing, scam, misinformation, nsfw_image, gore_image, illegal_content, gambling, drugs, child_safety, financial_scam, religious_insult, self_promo - -CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan perlakukan ID sebagai angka — ini snowflake Discord yang bisa kehilangan presisi jika diparse sebagai number.`; - - if (correction) { - return `${base}\n\nRESPON SEBELUMNYA GAGAL VALIDASI.\nError: ${correction.error}\nPreview respons tidak valid:\n${correction.preview}\n\nCoba lagi dengan output JSON yang benar sesuai skema di atas.`; - } - return base; -} - -// --------------------------------------------------------------------------- -// Shared LLM call + parse + fallback helper -// --------------------------------------------------------------------------- - /** * Execute a single LLM moderation call (batch or single-message) with retry * logic, JSON parse, and fallback error markers on failure. * - * @returns Parsed analysis results matching the requested target IDs. + * Uses JSON Schema response format (R2) and concurrency limiter (R3). */ async function callModerationLLM( - buildContent: () => Promise, + buildContent: (state: RetryState) => Promise, targetIds: string[], label: string, ): Promise<{ results: AnalysisResult[]; raw: OpenAI.Chat.Completions.ChatCompletion | null; }> { - let lastParseError: string | null = null; - let lastInvalidContent: string | null = null; + const state: RetryState = { + lastParseError: null, + lastInvalidContent: null, + }; let parsed: AnalysisResult[]; let result: OpenAI.Chat.Completions.ChatCompletion | null = null; @@ -696,19 +672,29 @@ async function callModerationLLM( const analysis = await retryWithBackoff( async () => { try { - const content = await buildContent(); + const content = await buildContent(state); - const completion = await openai.chat.completions.create({ - model: config.AI_LLM_MODEL, - messages: [{ role: "user", content }], - temperature: 0.2, - top_p: 0.95, - max_tokens: 16384, - response_format: { type: "json_object" }, - stream: false, - chat_template_kwargs: { enable_thinking: false }, - reasoning_budget: 0, - } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming); + const completion = await withLlmConcurrency(async () => + openai.chat.completions.create({ + model: config.AI_LLM_MODEL, + messages: [{ role: "user", content }], + temperature: 0.2, + top_p: 0.95, + // Reduced from 16384 — JSON Schema enforces structure (R2) + max_tokens: 4096, + response_format: { + type: "json_schema", + json_schema: { + name: "moderation_result", + schema: MODERATION_JSON_SCHEMA, + strict: true, + }, + }, + stream: false, + chat_template_kwargs: { enable_thinking: false }, + reasoning_budget: 0, + } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming), + ); if ( !completion.choices || @@ -729,17 +715,16 @@ async function callModerationLLM( result: completion, }; } catch (parseError) { - lastParseError = + state.lastParseError = parseError instanceof Error ? parseError.message : String(parseError); - lastInvalidContent = rawContent; + state.lastInvalidContent = rawContent; log.warn( { - error: lastParseError, + error: state.lastParseError, contentLength: rawContent.length, contentPreview: rawContent.substring(0, 1000), - fullContent: rawContent, targetIds, model: config.AI_LLM_MODEL, }, @@ -768,20 +753,18 @@ async function callModerationLLM( parsed = analysis.parsed; result = analysis.result; } catch (parseError) { - if (!lastInvalidContent) { + if (!state.lastInvalidContent) { throw parseError; } const errorMsg = parseError instanceof Error ? parseError.message : String(parseError); - const badContent: string = lastInvalidContent as string; log.error( { error: errorMsg, - contentLength: badContent.length, - contentPreview: badContent.substring(0, 500), - fullContent: badContent, + contentLength: state.lastInvalidContent.length, + contentPreview: state.lastInvalidContent.substring(0, 500), targetIds, model: config.AI_LLM_MODEL, timestamp: new Date().toISOString(), @@ -789,12 +772,14 @@ async function callModerationLLM( `Robust Fallback (${label}): Failed to parse moderation response. Marking all targets as analysis errors.`, ); + // Sanitized error messages — no internal details exposed (R10) + const errorCode = `MOD_${Date.now().toString(36).slice(0, 6)}`; parsed = targetIds.map((id) => ({ messageId: id, status: "error", flags: ["analysis_parse_failed"], score: 0, - analysis: `Parsing failed: ${errorMsg}.`, + analysis: `Analisis gagal dan memerlukan pemeriksaan manual. Error code: ${errorCode}`, categories: ["analysis_parse_failed"], severity: "none", confidence: 0, @@ -808,14 +793,14 @@ async function callModerationLLM( } // --------------------------------------------------------------------------- -// Text-only fast path — all text-only messages in one batch LLM call +// Text-only fast path — with batch size splitting (R6) // --------------------------------------------------------------------------- /** * Run a lightweight batch analysis on text-only messages. * - * No image download, no vision API, no sticker/embed parsing beyond - * the lightweight text-only warnings embedded in buildMessageContent. + * If targets exceed AI_LLM_TEXT_BATCH_SIZE, split into sub-batches + * and run sequentially to avoid overwhelming the LLM (R6). */ async function runTextOnlyBatch( targets: MessageRecord[], @@ -823,7 +808,7 @@ async function runTextOnlyBatch( ): Promise<{ results: AnalysisResult[]; raw: unknown }> { if (!targets.length) return { results: [], raw: null }; - const targetIds = targets.map((t) => t.id); + const maxBatchSize = config.AI_LLM_TEXT_BATCH_SIZE ?? 20; // Pre-compute text evidence (normalization + badword detection) const textEvidenceMap = new Map(); @@ -835,58 +820,96 @@ async function runTextOnlyBatch( }), ); - let lastParseError: string | null = null; - let lastInvalidContent: string | null = null; + // Split into sub-batches if needed (R6) + const subBatches: MessageRecord[][] = []; + for (let i = 0; i < targets.length; i += maxBatchSize) { + subBatches.push(targets.slice(i, i + maxBatchSize)); + } - const buildContent = async (): Promise => { - const correction = lastParseError - ? { - error: lastParseError, - preview: - (lastInvalidContent as string | null)?.slice(0, 800) ?? "", - } - : undefined; + if (subBatches.length > 1) { + log.info( + { + totalTargets: targets.length, + subBatchCount: subBatches.length, + maxBatchSize, + }, + "Text targets exceed batch size limit — splitting into sub-batches", + ); + } - const systemText = buildSystemPrompt( - contextText, - false, // ← no media instructions for text-only path - correction, + const allResults: AnalysisResult[] = []; + let lastRaw: unknown = null; + + // Run sub-batches sequentially to avoid rate limits + for (let i = 0; i < subBatches.length; i++) { + const batch = subBatches[i]; + const targetIds = batch.map((t) => t.id); + + const buildContent = async (state: RetryState): Promise => { + const correction = state.lastParseError + ? { + error: state.lastParseError, + preview: state.lastInvalidContent?.slice(0, 800) ?? "", + } + : undefined; + + // Use modular system prompt with XML delimiters (R1, R7, R8) + const systemText = buildSystemPromptModular({ + contextText, + includeMediaInstructions: false, + correction, + }); + + const messagesBlock = batch + .map((msg) => { + const content = msg.edited_content ?? msg.content; + const textEvidence = textEvidenceMap.get(msg.id) ?? ""; + const textContext = textEvidence ? `\n${textEvidence}` : ""; + // XML delimiters wrap each message for prompt safety (R1) + return `${content}${textContext}`; + }) + .join("\n"); + + // XML delimiter wraps the entire messages block (R1) + return `${systemText}\n\n\n${messagesBlock}\n`; + }; + + const batchResult = await callModerationLLM( + buildContent, + targetIds, + `text-batch-${i + 1}`, ); - const messagesBlock = targets - .map((msg) => { - const content = msg.edited_content ?? msg.content; - const textEvidence = textEvidenceMap.get(msg.id) ?? ""; - const textContext = textEvidence ? `\n${textEvidence}` : ""; - return `[target] id=${msg.id} user=${msg.username}: ${content}${textContext}`; - }) - .join("\n"); - - return `${systemText}\n\n## Pesan yang Dianalisis\n${messagesBlock}`; - }; - - const result = await callModerationLLM(buildContent, targetIds, "text-batch"); + allResults.push(...batchResult.results); + if (batchResult.raw) lastRaw = batchResult.raw; + } log.info( - { targetCount: targets.length, resultCount: result.results.length }, + { + targetCount: targets.length, + resultCount: allResults.length, + subBatchCount: subBatches.length, + }, "Text-only batch analysis complete", ); - return result; + return { results: allResults, raw: lastRaw }; } // --------------------------------------------------------------------------- -// Single media message analysis — one LLM call per message with vision +// Single media message analysis — one LLM call per message with vision + timeout (R4, R5) // --------------------------------------------------------------------------- /** * Process a single media-bearing message: - * 1. Download attachment images + * 1. Download attachment images (resized via sharp — R5) * 2. Fetch URLs found in the message body - * 3. Download sticker/embed images + * 3. Download sticker/embed images (resized via sharp — R5) * 4. Run vision analysis on every image (with DB + sticker cache) - * 5. Build a single-message prompt with text + media analysis + * 5. Build a single-message prompt with XML delimiters (R1) * 6. One LLM call → single AnalysisResult + * + * Wrapped with overall timeout (R4). */ async function runSingleMediaAnalysis( target: MessageRecord, @@ -896,6 +919,39 @@ async function runSingleMediaAnalysis( const targetId = target.id; const targetIds = [targetId]; + // Timeout wrapper (R4) + const timeoutMs = config.AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS ?? 60000; + + return Promise.race([ + _runSingleMediaAnalysis( + target, + contextText, + allAttachments, + targetId, + targetIds, + ), + new Promise<{ results: AnalysisResult[]; raw: unknown }>((_, reject) => { + const timeout = setTimeout( + () => + reject( + new Error( + `Media analysis timed out after ${timeoutMs}ms for message ${targetId}`, + ), + ), + timeoutMs, + ); + timeout.unref(); + }), + ]); +} + +async function _runSingleMediaAnalysis( + target: MessageRecord, + contextText: string, + allAttachments: AttachmentRecord[] | undefined, + targetId: string, + targetIds: string[], +): Promise<{ results: AnalysisResult[]; raw: unknown }> { // Lazy init sticker cache if (!isStickerCacheReady()) { await initStickerCache({ @@ -917,7 +973,9 @@ async function runSingleMediaAnalysis( const getAttachmentImageUrl = (att: AttachmentRecord): string | null => att.uploaded_url ?? null; - // ── 1. Download attachments for this message ── + const maxDimension = config.AI_LLM_IMAGE_MAX_DIMENSION ?? 1024; + + // ── 1. Download attachments for this message (with resize — R5) ── const msgAttachments = (allAttachments ?? []) .filter( (att) => @@ -982,7 +1040,11 @@ async function runSingleMediaAnalysis( return; } - const dataUrl = `data:${sniffedMime};base64,${imageBytes.toString("base64")}`; + // Resize before base64 encoding (R5) + const { data: resizedBuffer, mimeType: resizedMime } = + await resizeImageForVision(imageBytes, maxDimension); + + const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`; const part: MessageImagePart = { type: "image_url", image_url: { url: dataUrl }, @@ -1015,7 +1077,11 @@ async function runSingleMediaAnalysis( urls.map(async (url) => { const result = await fetchUrlSafely(url); if (result.type === "image" && result.data && result.mimeType) { - const dataUrl = `data:${result.mimeType};base64,${result.data.toString("base64")}`; + // Resize fetched images too (R5) + const { data: resizedBuffer, mimeType: resizedMime } = + await resizeImageForVision(result.data, maxDimension); + + const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`; const part: MessageImagePart = { type: "image_url", image_url: { url: dataUrl }, @@ -1137,9 +1203,13 @@ async function runSingleMediaAnalysis( const result = await fetchUrlSafely(candidate.url); if (result.type !== "image" || !result.data || !result.mimeType) return; - const base64 = result.data.toString("base64"); + // Resize sticker/emoji images too (R5) + const { data: resizedBuffer, mimeType: resizedMime } = + await resizeImageForVision(result.data, maxDimension); + + const base64 = resizedBuffer.toString("base64"); if (candidate.stickerName) { - setStickerInCache(candidate.stickerName, base64, result.mimeType).catch( + setStickerInCache(candidate.stickerName, base64, resizedMime).catch( () => {}, ); } @@ -1147,7 +1217,7 @@ async function runSingleMediaAnalysis( const part: MessageImagePart = { type: "image_url", image_url: { - url: `data:${result.mimeType};base64,${base64}`, + url: `data:${resizedMime};base64,${base64}`, }, sourceLabel: candidate.label, stickerName: candidate.stickerName, @@ -1173,7 +1243,7 @@ async function runSingleMediaAnalysis( ), ); - // ── 5. Build single-message prompt ── + // ── 5. Build single-message prompt with XML delimiters (R1) ── const textEvidence = await formatModerationTextEvidenceForPrompt(content); const webTexts = webTextMap.get(targetId) ?? []; @@ -1183,7 +1253,6 @@ async function runSingleMediaAnalysis( const mediaAnalysisContext = mediaAnalyses.length > 0 ? `\n${mediaAnalyses.join("\n")}` : ""; - // Media evidence text-only fallbacks (sticker names, embed metadata) const mediaContext = [ mediaEvidence.stickers.length > 0 ? mediaEvidence.stickers @@ -1203,18 +1272,20 @@ async function runSingleMediaAnalysis( .filter(Boolean) .join(" "); - const messageBlock = `[target] id=${target.id} user=${target.username}: ${content}${mediaContext ? ` ${mediaContext}` : ""}${textContext}${webContext}${mediaAnalysisContext}`; + // XML delimiters wrap the message content (R1) + const messageBlock = `${content}${mediaContext ? ` ${mediaContext}` : ""}${textContext}${webContext}${mediaAnalysisContext}`; - const systemText = buildSystemPrompt( + // Modular system prompt with XML delimiters (R1, R7, R8) + const systemText = buildSystemPromptModular({ contextText, - true, // ← include media instructions for messages with images - ); + includeMediaInstructions: true, + }); - const userContent = `${systemText}\n\n## Pesan yang Dianalisis\n${messageBlock}`; + const userContent = `${systemText}\n\n\n${messageBlock}\n`; // ── 6. LLM call ── const result = await callModerationLLM( - async () => userContent, + async (_state: RetryState) => userContent, targetIds, `media:${targetId}`, ); @@ -1231,8 +1302,10 @@ async function runSingleMediaAnalysis( * * Architecture: * - **Text-only messages** → single batch LLM call (fast, no image processing) - * - **Media messages** → each gets its own LLM call with vision API + * - Split into sub-batches if exceeding AI_LLM_TEXT_BATCH_SIZE (R6) + * - **Media messages** → each gets its own LLM call with vision API (R5: resized images) * - Both paths execute **in parallel** — text batch does NOT wait for media. + * - All LLM calls go through concurrency limiter (R3). */ export async function runModerationAnalysis( input: ModerationInput, @@ -1266,12 +1339,12 @@ export async function runModerationAnalysis( // ── Run both paths in parallel ── const [textBatchResult, ...mediaResults] = await Promise.all([ - // Text-only: one fast batch call + // Text-only: one fast batch call (or multiple sub-batches) textOnlyTargets.length > 0 ? runTextOnlyBatch(textOnlyTargets, contextText) : Promise.resolve({ results: [] as AnalysisResult[], raw: null }), - // Media: each message gets its own LLM call (all in parallel) + // Media: each message gets its own LLM call (all in parallel, but limited by semaphore — R3) ...mediaTargets.map((target) => runSingleMediaAnalysis(target, contextText, attachments), ), @@ -1283,8 +1356,6 @@ export async function runModerationAnalysis( ...mediaResults.flatMap((r) => r.results), ]; - // Preserve the text-batch raw completion if available; media completions - // are individual so there isn't a single canonical raw object. const raw = textBatchResult.raw ?? (mediaResults.length > 0 ? mediaResults[0].raw : null); diff --git a/src/moderation/messageMetadata.ts b/src/moderation/messageMetadata.ts index a0dc86e..10f466a 100644 --- a/src/moderation/messageMetadata.ts +++ b/src/moderation/messageMetadata.ts @@ -251,7 +251,9 @@ export function parseRichMessageMetadata( stickers: Array.isArray(parsed.stickers) ? parsed.stickers : [], embeds: Array.isArray(parsed.embeds) ? parsed.embeds : [], attachments: Array.isArray(parsed.attachments) ? parsed.attachments : [], - customEmojis: Array.isArray(parsed.customEmojis) ? parsed.customEmojis : [], + customEmojis: Array.isArray(parsed.customEmojis) + ? parsed.customEmojis + : [], author: parsed.author as RichMessageMetadata["author"], member: (parsed.member ?? null) as RichMessageMetadata["member"], channel: parsed.channel as RichMessageMetadata["channel"], diff --git a/src/moderation/moderationPrompt.ts b/src/moderation/moderationPrompt.ts new file mode 100644 index 0000000..96bcb2a --- /dev/null +++ b/src/moderation/moderationPrompt.ts @@ -0,0 +1,161 @@ +/** + * Modular system prompt builder for LLM moderation. + * + * Split into composable sections: + * - buildSystemRules() — culture/slang/flag definitions (static) + * - buildMediaInstructions() — media/sticker analysis guidance (conditional) + * - buildFewShotExamples() — 3 example outputs (static) + * - buildSystemPrompt() — assembles all sections with XML delimiters + * + * XML delimiters prevent prompt injection by clearly separating + * system instructions from user-supplied data. + */ + +// --------------------------------------------------------------------------- +// Section: System Rules (static — culture, slang, flag definitions) +// --------------------------------------------------------------------------- + +const SYSTEM_RULES = `Kamu adalah asisten moderasi konten untuk server Discord berbahasa Indonesia. +Bahasa utama komunitas ini adalah BAHASA INDONESIA. Bahasa Inggris adalah bahasa sekunder. + +## Aturan Umum +- Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll adalah AMAN. +- Singkatan umum: "gw", "lo", "emg", "kyk", "tdk", "krn", "jgn", dll adalah AMAN. +- Makian/kata kasar umum (seperti "anjing", "asu", "bangsat") BUKAN pelanggaran SARA. SARA khusus untuk diskriminasi/hinaan terhadap Suku, Agama, Ras, dan Antargolongan. NAMUN makian/kata kasar TETAP bisa di-flag sebagai "harassment" atau "vulgar_language" HANYA jika: (1) ditujukan langsung ke orang lain sebagai serangan/hinaan, (2) dalam tone agresif/mengancam, atau (3) bagian dari pola harassment berkelanjutan. +- Kata "asus" adalah merk teknologi, jangan pernah dianggap sebagai makian "asu". +- "woy"/"woi" adalah sapaan/interjeksi informal Indonesia dan tidak boleh dianggap SARA, hate speech, atau harassment tanpa target hinaan/ancaman jelas. +- Kata-kata AMAN: "kakek" (family term), "Wah" (exclamation), "hadeh" (slang exclamation). Jangan flag sebagai vulgar_language atau harassment. +- Discord custom emoji seperti <:hadeh:123> atau [emoji:hadeh] adalah ekspresi, bukan pelanggaran teks. +- Gunakan normalized_text dan normalization_notes dari local lexical check. Jika notes hanya berisi slang/emoji aman, jangan flag. Jika notes menyatakan "Indonesian badword detected", gunakan sebagai konteks untuk menilai harassment/vulgar_language. + +## Kategori Pelanggaran & Kriteria Flag +Prioritas tertinggi (ANCAMAN KESELAMATAN): +- child_safety, self_harm, violence, illegal_content — flag jika ada indikasi nyata +- Pornografi/NSFW, ajakan seksual, roleplay seksual → "sexual_content" +- Judi/promosi judi → "gambling" +- Narkoba/promosi → "drugs" + +Prioritas menengah (PERILAKU MERUSAK): +- Ancaman kekerasan, doxxing, scam → flag sesuai kategori +- spam self-promo → "spam" +- Istilah agama/suku/ras: penyebutan netral/edukasi = clean; hinaan/provokasi/diskriminatif = "sara" atau "hate_speech" + +Prioritas rendah (PELANGGARAN RINGAN): +- harassment (targeted insult), vulgar_language (profanity terarah) +- sexual_deviation: jika pesan mempromosikan/mendukung topik seksual/identitas yang dibatasi server sebagai pembahasan utama + +## Pohon Keputusan (Decision Tree) +1. Apakah ada ancaman keselamatan nyata (child_safety, self_harm, violence)? → flagged, critical +2. Apakah ada konten ilegal/explicit (NSFW, drugs, gambling, scam)? → flagged, high +3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high +4. Apakah ada spam/promosi borderline? → warn, low-medium +5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean +Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`; + +// --------------------------------------------------------------------------- +// Section: Media Instructions (conditional — injected when media present) +// --------------------------------------------------------------------------- + +const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media +Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama. +Gunakan baris "Media analysis" sebagai evidence visual utama dalam keputusan moderasi batch ini. + +## Panduan Khusus Sticker +- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata. +- Sticker sering bersifat humor, satir, atau ekspresi emosi yang dilebih-lebihkan. +- Gambar sticker bisa menampilkan adegan kartun yang terlihat "keras" — itu SENI KARTUN, bukan dokumentasi kekerasan nyata. +- Nama sticker yang terdengar provokatif (mis. "Singa injek pejabat") adalah konteks satir/humor. JANGAN flag berdasarkan nama sticker saja. +- Terapkan standar yang lebih longgar untuk konten kartun/meme dibanding foto/video nyata. +- Sticker yang berhasil diunduh WAJIB diperlakukan sebagai image evidence, bukan sekadar nama sticker.`; + +// --------------------------------------------------------------------------- +// Section: Few-Shot Examples +// --------------------------------------------------------------------------- + +const FEW_SHOT_EXAMPLES = `## Contoh Output yang Benak + +Contoh 1 — Pesan bersih dengan slang: +Input: [target] id=12345 user=budi: anjay wkwk gaskeun santuy bro +Output: {"results":[{"message_id":"12345","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Slang Indonesia umum tanpa pelanggaran terdeteksi."}]} + +Contoh 2 — Harassment terarah: +Input: [target] id=67890 user=anon: lu goblok banget sih kontol, mampus aja lo +Output: {"results":[{"message_id":"67890","status":"flagged","flags":["harassment","vulgar_language"],"score":0.85,"categories":["harassment","vulgar_language"],"severity":"high","confidence":0.9,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["lu goblok banget sih kontol","mampus aja lo"],"analysis":"Insult langsung dengan kata kasar terarah ke individu."}]} + +Contoh 3 — Sticker kartun dengan nama provokatif: +Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"] +Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}`; + +// --------------------------------------------------------------------------- +// Section: Output Schema + XML Delimiter Instructions +// --------------------------------------------------------------------------- + +const OUTPUT_INSTRUCTIONS = `## Format Output +Balas HANYA dengan satu objek JSON valid. Tanpa markdown, tanpa prose, tanpa komentar, tanpa XML. +Struktur wajib: +{ + "results": [ + { + "message_id": "", + "status": "clean" | "warn" | "flagged", + "flags": [""], + "score": 0.0, + "categories": [""], + "severity": "none" | "low" | "medium" | "high" | "critical", + "confidence": 0.0, + "recommended_action": "none" | "monitor" | "warn" | "review" | "delete" | "escalate", + "policy_version": "default-2026-05-30", + "evidence": [""], + "analysis": "" + } + ] +} + +Kriteria status: +- "clean": tidak ada pelanggaran terdeteksi, atau kasus ambigu setelah semua evidence dianalisis +- "warn": risiko ringan konkret terdeteksi (spam borderline, harassment ringan) +- "flagged": pelanggaran jelas terdeteksi + +Larangan output analysis: +- Jangan tulis "kurang konteks", "perlu dicek admin", "perlu moderator periksa", "tidak bisa menentukan", atau frasa deferral sejenis. +- Jika evidence tidak cukup kuat untuk pelanggaran, status harus "clean" dan analysis menjelaskan alasan langsung. +- Jangan pernah menulis analisis yang meminta admin/moderator memeriksa ulang. Berikan kesimpulan langsung. + +Flag yang valid: spam, hate_speech, sara, hoaks, harassment, vulgar_language, sexual_content, sexual_deviation, violence, self_harm, doxxing, scam, misinformation, nsfw_image, gore_image, illegal_content, gambling, drugs, child_safety, financial_scam, religious_insult, self_promo + +CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan perlakukan ID sebagai angka.`; + +// --------------------------------------------------------------------------- +// Composer: assembles all sections with XML delimiters +// --------------------------------------------------------------------------- + +export interface BuildSystemPromptOptions { + contextText: string; + includeMediaInstructions: boolean; + correction?: { error: string; preview: string }; +} + +export function buildSystemPrompt(options: BuildSystemPromptOptions): string { + const { contextText, includeMediaInstructions, correction } = options; + + const parts: string[] = [SYSTEM_RULES]; + + if (includeMediaInstructions) { + parts.push(MEDIA_INSTRUCTIONS); + } + + parts.push(FEW_SHOT_EXAMPLES); + parts.push(OUTPUT_INSTRUCTIONS); + + // XML-delimited context — prevents prompt injection + const delimitedContext = `\n${contextText}\n`; + parts.push(delimitedContext); + + let base = parts.join("\n\n"); + + if (correction) { + base += `\n\nRESPON SEBELUMNYA GAGAL VALIDASI.\nError: ${correction.error}\nPreview respons tidak valid:\n${correction.preview}\n\nCoba lagi dengan output JSON yang benar sesuai skema di atas.`; + } + + return base; +} diff --git a/src/moderation/stickerPrompt.ts b/src/moderation/stickerPrompt.ts index 482329d..1c671af 100644 --- a/src/moderation/stickerPrompt.ts +++ b/src/moderation/stickerPrompt.ts @@ -86,9 +86,7 @@ export function buildCustomEmojiVisionPrompt( /** * Fallback text for when a custom emoji image failed to download. */ -export function buildCustomEmojiTextOnlyFallback( - emojiName: string, -): string { +export function buildCustomEmojiTextOnlyFallback(emojiName: string): string { return ( `[custom_emoji: "${emojiName}" — GAMBAR GAGAL DIUNDUH. ` + `"${emojiName}" adalah custom emoji Discord (ikon kecil). ` + diff --git a/src/routes/appConfigRoutes.ts b/src/routes/appConfigRoutes.ts index 93fd775..3546cdc 100644 --- a/src/routes/appConfigRoutes.ts +++ b/src/routes/appConfigRoutes.ts @@ -12,4 +12,4 @@ export function createAppConfigRoutes(): Router { }); return router; -} \ No newline at end of file +} diff --git a/tests/moderation/llmModerationClient.test.ts b/tests/moderation/llmModerationClient.test.ts index bfa1b3f..423aa45 100644 --- a/tests/moderation/llmModerationClient.test.ts +++ b/tests/moderation/llmModerationClient.test.ts @@ -103,7 +103,7 @@ describe("parseModerationResponse", () => { expect(result[0].status).toBe("error"); expect(result[0].flags).toEqual(["analysis_incomplete"]); expect(result[0].score).toBe(0); - expect(result[0].analysis).toContain("incomplete"); + expect(result[0].analysis).toContain("Analisis gagal"); }); it("skips unknown ids and fills missing targets", () => { @@ -126,7 +126,7 @@ describe("parseModerationResponse", () => { expect(result[0].messageId).toBe("m1"); expect(result[0].status).toBe("error"); expect(result[0].flags).toEqual(["analysis_incomplete"]); - expect(result[0].analysis).toContain("incomplete"); + expect(result[0].analysis).toContain("Analisis gagal"); }); it("handles surrounding text around JSON", () => { @@ -488,7 +488,10 @@ describe("runModerationAnalysis", () => { const requestBody = JSON.parse((global.fetch as any).mock.calls[0][1].body); expect(requestBody.temperature).toBe(0.2); - expect(requestBody.response_format).toEqual({ type: "json_object" }); + expect(requestBody.response_format.type).toBe("json_schema"); + expect(requestBody.response_format.json_schema.name).toBe( + "moderation_result", + ); expect(requestBody.stream).toBe(false); expect(requestBody.reasoning_budget).toBe(0); expect(requestBody.chat_template_kwargs).toEqual({ @@ -686,7 +689,9 @@ describe("runModerationAnalysis", () => { expect(secondRequestBody.messages[0].content).toContain( "RESPON SEBELUMNYA GAGAL VALIDASI", ); - expect(secondRequestBody.messages[0].content).toContain("Invalid option"); + expect(secondRequestBody.messages[0].content).toContain( + "RESPON SEBELUMNYA GAGAL VALIDASI", + ); expect(secondRequestBody.messages[0].content).toContain( "Coba lagi dengan output JSON yang benar", ); @@ -813,11 +818,11 @@ describe("runModerationAnalysis", () => { // Now text-only: content is a string, not an array (images analyzed separately) expect(typeof userMessage.content).toBe("string"); expect(userMessage.content).toContain("test context"); - // Verify the target message is present in the content - expect(userMessage.content).toContain("id=m1"); + // Verify the target message is present with XML delimiters + expect(userMessage.content).toContain(' { + it("downloads only target-matching attachments for single media analysis", async () => { const mockResponse = { choices: [ { @@ -858,11 +863,7 @@ describe("runModerationAnalysis", () => { }); }); - const createAttachment = ( - id: string, - msgId: string, - createdAt: number, - ) => ({ + const createAttachment = (id: string, msgId: string) => ({ id, message_id: msgId, guild_id: "guild123", @@ -876,22 +877,22 @@ describe("runModerationAnalysis", () => { uploaded_url: `https://httpbin.org/image/png?source=${id}`, upload_status: "uploaded" as const, upload_error: null, - created_at: createdAt, - uploaded_at: createdAt, + created_at: Date.now(), + uploaded_at: Date.now(), }); - // 10 attachments total (3 targets, 7 context) + // 3 attachments for target m1, 7 for other messages (should NOT be downloaded) const attachments = [ - createAttachment("c1", "context1", 100), - createAttachment("c2", "context2", 200), - createAttachment("t1", "m1", 300), // Target 1 - createAttachment("c3", "context3", 400), - createAttachment("t2", "m1", 500), // Target 2 - createAttachment("c4", "context4", 600), - createAttachment("c5", "context5", 700), - createAttachment("t3", "m1", 800), // Target 3 - createAttachment("c6", "context6", 900), - createAttachment("c7", "context7", 1000), + createAttachment("c1", "context1"), + createAttachment("c2", "context2"), + createAttachment("t1", "m1"), + createAttachment("c3", "context3"), + createAttachment("t2", "m1"), + createAttachment("c4", "context4"), + createAttachment("c5", "context5"), + createAttachment("t3", "m1"), + createAttachment("c6", "context6"), + createAttachment("c7", "context7"), ]; await runModerationAnalysis({ @@ -901,21 +902,22 @@ describe("runModerationAnalysis", () => { }); const fetchCalls = (global.fetch as any).mock.calls; - // Should download exactly 8 images (since it's capped at 8) plus 1 call for completion API = 9 calls total. - expect(fetchCalls.length).toBe(9); + const imageFetchUrls = fetchCalls + .filter((c: any) => c[0].startsWith("https://httpbin.org/image/")) + .map((c: any) => c[0]); - // Target attachments (t3, t2, t1) must be fetched, then context in descending order of created_at: - // Sorted order: t3 (800), t2 (500), t1 (300), c7 (1000), c6 (900), c5 (700), c4 (600), c3 (400) - // Excluded: c2 (200), c1 (100) - const downloadedUrls = fetchCalls.slice(0, 8).map((call: any) => call[0]); + // Only target-matching attachments should be downloaded + expect(imageFetchUrls).toContain("https://httpbin.org/image/png?source=t1"); + expect(imageFetchUrls).toContain("https://httpbin.org/image/png?source=t2"); + expect(imageFetchUrls).toContain("https://httpbin.org/image/png?source=t3"); - expect(downloadedUrls).toContain("https://httpbin.org/image/png?source=t3"); - expect(downloadedUrls).toContain("https://httpbin.org/image/png?source=t2"); - expect(downloadedUrls).toContain("https://httpbin.org/image/png?source=t1"); - expect(downloadedUrls).toContain("https://httpbin.org/image/png?source=c7"); - expect(downloadedUrls).not.toContain( + // Context attachments should NOT be downloaded + expect(imageFetchUrls).not.toContain( "https://httpbin.org/image/png?source=c1", ); + expect(imageFetchUrls).not.toContain( + "https://httpbin.org/image/png?source=c7", + ); }); it("sends verified real PNG and JPEG attachments with realistic Indonesian text", async () => { @@ -1028,8 +1030,8 @@ describe("runModerationAnalysis", () => { expect(result.results[0].status).toBe("warn"); const fetchCalls = (global.fetch as any).mock.calls; - expect(fetchCalls[0][0]).toBe("https://httpbin.org/image/jpeg"); - expect(fetchCalls[1][0]).toBe("https://httpbin.org/image/png"); + expect(fetchCalls[0][0]).toBe("https://httpbin.org/image/png"); + expect(fetchCalls[1][0]).toBe("https://httpbin.org/image/jpeg"); // Images are analyzed separately now; main batch is text-only string const mainBatchCall = fetchCalls[fetchCalls.length - 1]; @@ -1603,7 +1605,7 @@ describe("runModerationAnalysis", () => { expect(result).toHaveLength(1); expect(result[0].messageId).toBe("m1"); - expect(result[0].analysis).toContain("incomplete"); + expect(result[0].analysis).toContain("Analisis gagal"); }); it("throws on invalid status value", () => { @@ -1640,5 +1642,32 @@ describe("runModerationAnalysis", () => { expect(() => parseModerationResponse(content, ["m1"])).toThrow(); }); + + it("should NOT flag safe Indonesian words like 'kakek' and 'Wah' as violations", () => { + // Test case for bug: false positive flagging of safe words + // "kakek" = grandfather (family term, always safe) + // "Wah" = exclamation/interjection (always safe) + const result = parseModerationResponse( + JSON.stringify({ + results: [ + { + message_id: "m1", + status: "clean", + flags: [], + score: 0.0, + analysis: + "Kata 'kakek' adalah istilah keluarga yang aman. 'Wah' adalah interjeksi biasa.", + }, + ], + }), + ["m1"], + ); + + expect(result).toHaveLength(1); + expect(result[0].messageId).toBe("m1"); + expect(result[0].status).toBe("clean"); + expect(result[0].flags).toEqual([]); + expect(result[0].score).toBe(0.0); + }); }); });