diff --git a/src/moderation/conversationContext.ts b/src/moderation/conversationContext.ts index 394ea3d..5ed955c 100644 --- a/src/moderation/conversationContext.ts +++ b/src/moderation/conversationContext.ts @@ -1,3 +1,4 @@ +import { formatMediaEvidenceForPrompt } from "./messageMetadata.js"; import type { MessageRecord } from "./types.js"; export interface ConversationContextInput { @@ -29,7 +30,9 @@ export function formatMessageForPrompt( ): string { const content = msg.edited_content ?? msg.content; const timestamp = formatTimestamp(msg.created_at); - return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}`; + const mediaEvidence = formatMediaEvidenceForPrompt(msg.metadata); + const mediaSuffix = mediaEvidence ? ` ${mediaEvidence}` : ""; + return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}${mediaSuffix}`; } /** diff --git a/src/moderation/llmModerationClient.ts b/src/moderation/llmModerationClient.ts index 7854f5d..afe0ca4 100644 --- a/src/moderation/llmModerationClient.ts +++ b/src/moderation/llmModerationClient.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { config } from "../config.js"; import { createChildLogger } from "../logger.js"; import { retryWithBackoff } from "../retry.js"; +import { extractMessageMediaEvidence } from "./messageMetadata.js"; import type { AnalysisResult, AttachmentRecord, @@ -24,6 +25,13 @@ const ModerationResponseSchema = z.object({ }); const log = createChildLogger("llmModerationClient"); +const DEFERRAL_ANALYSIS_PATTERN = + /kurang konteks|kekurangan konteks|perlu (dicek|diperiksa|ditinjau).*(admin|moderator)|admin perlu|moderator perlu|tidak bisa menentukan|tidak dapat menentukan|cannot determine|insufficient context/i; + +function hasDeferralAnalysis(analysis: string): boolean { + return DEFERRAL_ANALYSIS_PATTERN.test(analysis); +} + const openai = new OpenAI({ apiKey: config.AI_LLM_API_KEY, baseURL: config.AI_LLM_BASE_URL, @@ -87,18 +95,6 @@ const openai = new OpenAI({ }, }); -interface RawModerationResult { - message_id: string; - status: string; - flags: unknown; - score: number; - analysis: string; -} - -interface RawModerationResponse { - results: RawModerationResult[]; -} - /** * Helper to extract JSON from a potentially conversational or markdown-wrapped string. */ @@ -214,6 +210,12 @@ export function parseModerationResponse( foundIds.add(finalId); + if (hasDeferralAnalysis(analysis)) { + throw new Error( + `Deferral analysis is not allowed for message ${finalId}; return a direct moderation decision`, + ); + } + return { messageId: finalId, status: status as "clean" | "warn" | "flagged", @@ -357,7 +359,8 @@ export async function runModerationAnalysis( // Build a lookup: message_id → list of resolved base64 image parts type RawImagePart = { type: "image_url"; image_url: { url: string } }; - type MessageImageMap = Map; + type MessageImagePart = RawImagePart & { sourceLabel: string }; + type MessageImageMap = Map; // Resolve and download image attachments, grouped by message_id. // Only images whose message_id appears in the full attachment list are kept; @@ -447,9 +450,10 @@ export async function runModerationAnalysis( } const dataUrl = `data:${sniffedMime};base64,${imageBytes.toString("base64")}`; - const part: RawImagePart = { + const part: MessageImagePart = { type: "image_url", image_url: { url: dataUrl }, + sourceLabel: `[gambar di atas adalah attachment ${att.filename} dari pesan id=${att.message_id}]`, }; const existing = messageImageMap.get(att.message_id) ?? []; @@ -488,9 +492,10 @@ export async function runModerationAnalysis( if (result.type === "image" && result.data && result.mimeType) { // Append as an image part const dataUrl = `data:${result.mimeType};base64,${result.data.toString("base64")}`; - const part: RawImagePart = { + const part: MessageImagePart = { type: "image_url", image_url: { url: dataUrl }, + sourceLabel: `[gambar di atas berasal dari link ${url} pada pesan id=${msg.id}]`, }; const existing = messageImageMap.get(msg.id) ?? []; existing.push(part); @@ -512,6 +517,63 @@ export async function runModerationAnalysis( }), ); + const mediaImageCandidates = targets.flatMap((msg) => { + const evidence = extractMessageMediaEvidence(msg.metadata); + return [ + ...evidence.stickers + .filter((sticker) => sticker.url) + .map((sticker) => ({ + messageId: msg.id, + url: sticker.url, + label: `[gambar di atas adalah sticker "${sticker.name}" dari pesan id=${msg.id}]`, + })), + ...evidence.embeds.flatMap((embed) => + [ + embed.image + ? { + messageId: msg.id, + url: embed.image, + label: `[gambar di atas berasal dari embed image pada pesan id=${msg.id}]`, + } + : null, + embed.thumbnail + ? { + messageId: msg.id, + url: embed.thumbnail, + label: `[gambar di atas berasal dari embed thumbnail pada pesan id=${msg.id}]`, + } + : null, + ].filter( + (candidate): candidate is { messageId: string; url: string; label: string } => + candidate !== null, + ), + ), + ]; + }); + + const remainingImageSlots = Math.max( + 0, + 8 - Array.from(messageImageMap.values()).reduce((sum, imgs) => sum + imgs.length, 0), + ); + + await Promise.all( + mediaImageCandidates.slice(0, remainingImageSlots).map(async (candidate) => { + const result = await fetchUrlSafely(candidate.url); + if (result.type !== "image" || !result.data || !result.mimeType) return; + + const part: MessageImagePart = { + type: "image_url", + image_url: { + url: `data:${result.mimeType};base64,${result.data.toString("base64")}`, + }, + sourceLabel: candidate.label, + }; + const existing = messageImageMap.get(candidate.messageId) ?? []; + existing.push(part); + messageImageMap.set(candidate.messageId, existing); + }), + ); + const hasImages = messageImageMap.size > 0; // ------------------------------------------------------------------------- @@ -533,11 +595,11 @@ export async function runModerationAnalysis( }): string => { const imageInstructions = hasImages ? ` -## Instruksi Analisis Gambar -Beberapa pesan menyertakan lampiran gambar. Setiap gambar muncul TEPAT SETELAH baris teks pesan yang memilikinya. -Gambar dan teks pesan harus dianalisis sebagai SATU KESATUAN — evaluasilah konten teks DAN gambar secara bersamaan untuk membentuk kesimpulan final. -Jangan pisahkan penilaian gambar dari konteks teks pesannya. -Jika gambar mengandung teks (meme, screenshot), baca dan pertimbangkan teks tersebut sebagai bagian dari konten pesan. +## Instruksi Analisis Gambar/Sticker/Embed +Beberapa pesan menyertakan gambar, sticker, preview link, atau embed. Setiap media muncul TEPAT SETELAH baris teks pesan yang memilikinya. +Media dan teks pesan harus dianalisis sebagai SATU KESATUAN — evaluasilah konten teks, gambar, sticker, dan preview link secara bersamaan untuk membentuk kesimpulan final. +Jangan pisahkan penilaian media dari konteks teks pesannya. +Jika gambar/sticker/preview mengandung teks (meme, screenshot), baca dan pertimbangkan teks tersebut sebagai bagian dari konten pesan. ` : ""; @@ -550,7 +612,9 @@ Ini adalah server Discord komunitas Indonesia. Kamu harus memahami: - Singkatan umum: "gw", "lo", "emg", "kyk", "tdk", "krn", "jgn", dll. - Konteks budaya lokal: SARA (Suku, Agama, Ras, Antar-golongan), hoaks, ujaran kebencian berbasis konteks Indonesia. - Perbedaan antara humor/banter biasa vs konten yang benar-benar melanggar. -- Kalimat ambigu dalam bahasa Indonesia harus ditafsirkan dengan charitable intent kecuali ada indikator kuat sebaliknya. +- Kalimat ambigu dalam bahasa Indonesia harus diberi keputusan final: "clean" bila bukti pelanggaran tidak jelas, "flagged" bila bukti pelanggaran jelas. +- Jangan pernah menulis analisis yang meminta admin/moderator memeriksa ulang, menyebut kurang konteks, atau tidak bisa menentukan. Berikan kesimpulan langsung berdasarkan teks + media + konteks yang tersedia. +- Gambar, sticker, embed, dan preview link adalah evidence utama yang setara dengan teks, bukan sekadar URL teks. ${imageInstructions} ## Konteks Percakapan ${contextText} @@ -571,10 +635,14 @@ Struktur wajib: } Kriteria status: -- "clean": tidak ada pelanggaran yang terdeteksi -- "warn": konten berpotensi melanggar atau memerlukan perhatian moderator +- "clean": tidak ada pelanggaran yang terdeteksi, atau kasus masih ambigu setelah semua evidence dianalisis +- "warn": risiko ringan yang konkret terdeteksi, misalnya spam borderline atau harassment ringan; BUKAN untuk kurang konteks/perlu admin cek - "flagged": pelanggaran jelas terdeteksi +Larangan output analysis: +- Jangan tulis "kurang konteks", "perlu dicek admin", "perlu moderator periksa", "tidak bisa menentukan", atau frasa deferral sejenis. +- Jika evidence tidak cukup kuat untuk pelanggaran, status harus "clean" dan analysis menjelaskan alasan langsung. + Flag yang valid: spam, hate_speech, sara, hoaks, harassment, sexual_content, violence, self_harm, doxxing, scam, misinformation, nsfw_image, gore_image, illegal_content CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan perlakukan ID sebagai angka — ini snowflake Discord yang bisa kehilangan presisi jika diparse sebagai number.`; @@ -618,7 +686,20 @@ CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan const webTexts = messageWebTextMap.get(msg.id) ?? []; const webContext = webTexts.length > 0 ? `\n${webTexts.join("\n")}` : ""; - return `[target] id=${msg.id} user=${msg.username}: ${content}${webContext}`; + const mediaEvidence = extractMessageMediaEvidence(msg.metadata); + const mediaContext = [ + mediaEvidence.stickers.length > 0 + ? `[sticker evidence: ${mediaEvidence.stickers.map((s) => `${s.name} (${s.url})`).join(" | ")}]` + : null, + mediaEvidence.embeds.length > 0 + ? `[embed evidence: ${mediaEvidence.embeds + .map((e) => [e.title, e.description, e.url, e.image, e.thumbnail].filter(Boolean).join(" | ")) + .join(" || ")}]` + : null, + ] + .filter(Boolean) + .join(" "); + return `[target] id=${msg.id} user=${msg.username}: ${content}${mediaContext ? ` ${mediaContext}` : ""}${webContext}`; }) .join("\n"); @@ -638,21 +719,32 @@ CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan const webTexts = messageWebTextMap.get(msg.id) ?? []; const webContext = webTexts.length > 0 ? `\n${webTexts.join("\n")}` : ""; - const msgText = `[target] id=${msg.id} user=${msg.username}: ${content}${webContext}`; + const mediaEvidence = extractMessageMediaEvidence(msg.metadata); + const mediaContext = [ + mediaEvidence.stickers.length > 0 + ? `[sticker evidence: ${mediaEvidence.stickers.map((s) => `${s.name} (${s.url})`).join(" | ")}]` + : null, + mediaEvidence.embeds.length > 0 + ? `[embed evidence: ${mediaEvidence.embeds + .map((e) => [e.title, e.description, e.url, e.image, e.thumbnail].filter(Boolean).join(" | ")) + .join(" || ")}]` + : null, + ] + .filter(Boolean) + .join(" "); + const msgText = `[target] id=${msg.id} user=${msg.username}: ${content}${mediaContext ? ` ${mediaContext}` : ""}${webContext}`; parts.push({ type: "text", text: msgText }); // Immediately follow the message text with its images const imgs = messageImageMap.get(msg.id); if (imgs && imgs.length > 0) { for (const img of imgs) { - parts.push(img); + parts.push({ type: "image_url", image_url: img.image_url }); + parts.push({ + type: "text", + text: img.sourceLabel, + }); } - // Anchor label after the image(s) so the model's attention window - // links this image block back to the message ID above it - parts.push({ - type: "text", - text: `[gambar di atas adalah lampiran dari pesan id=${msg.id}]`, - }); } } diff --git a/src/moderation/messageMetadata.ts b/src/moderation/messageMetadata.ts index e4a753f..b30e41c 100644 --- a/src/moderation/messageMetadata.ts +++ b/src/moderation/messageMetadata.ts @@ -11,35 +11,47 @@ export interface MessageLocation { channelName: string | null; } -export interface RichMessageMetadata { - stickers: Array<{ - id: string; - name: string; - url: string; - format: string | null; - }>; - embeds: Array<{ - title: string | null; - description: string | null; +export interface StickerEvidence { + id: string; + name: string; + url: string; + format: string | null; +} + +export interface EmbedEvidence { + title: string | null; + description: string | null; + url: string | null; + color: number | null; + image: string | null; + thumbnail: string | null; + author: { + name: string | null; url: string | null; - color: number | null; - image: string | null; - thumbnail: string | null; - author: { - name: string | null; - url: string | null; - iconURL: string | null; - } | null; - footer: { text: string | null; iconURL: string | null } | null; - fields: Array<{ name: string; value: string; inline: boolean }>; - }>; - attachments: Array<{ - id: string; - name: string; - url: string; - contentType: string | null; - size: number; - }>; + iconURL: string | null; + } | null; + footer: { text: string | null; iconURL: string | null } | null; + fields: Array<{ name: string; value: string; inline: boolean }>; +} + +export interface AttachmentEvidence { + id: string; + name: string; + url: string; + contentType: string | null; + size: number; +} + +export interface MessageMediaEvidence { + stickers: StickerEvidence[]; + embeds: EmbedEvidence[]; + attachments: AttachmentEvidence[]; +} + +export interface RichMessageMetadata { + stickers: Array; + embeds: Array; + attachments: Array; author: { id: string; username: string; @@ -167,6 +179,97 @@ export function getMessageMetadata(message: Message): RichMessageMetadata { }; } +export function parseRichMessageMetadata( + metadata: string | null | undefined, +): RichMessageMetadata | null { + if (!metadata) return null; + + try { + const parsed = JSON.parse(metadata) as Partial; + return { + stickers: Array.isArray(parsed.stickers) ? parsed.stickers : [], + embeds: Array.isArray(parsed.embeds) ? parsed.embeds : [], + attachments: Array.isArray(parsed.attachments) ? parsed.attachments : [], + author: parsed.author as RichMessageMetadata["author"], + member: (parsed.member ?? null) as RichMessageMetadata["member"], + channel: parsed.channel as RichMessageMetadata["channel"], + reference: (parsed.reference ?? null) as RichMessageMetadata["reference"], + }; + } catch { + return null; + } +} + +export function extractMessageMediaEvidence( + metadata: string | null | undefined, +): MessageMediaEvidence { + const parsed = parseRichMessageMetadata(metadata); + return { + stickers: parsed?.stickers ?? [], + embeds: parsed?.embeds ?? [], + attachments: parsed?.attachments ?? [], + }; +} + +export function formatMediaEvidenceForPrompt( + metadata: string | null | undefined, +): string { + const evidence = extractMessageMediaEvidence(metadata); + const parts: string[] = []; + + if (evidence.stickers.length > 0) { + parts.push( + `[stickers: ${evidence.stickers + .map((sticker) => + [`name=${sticker.name}`, sticker.url ? `url=${sticker.url}` : null] + .filter(Boolean) + .join(", "), + ) + .join(" | ")}]`, + ); + } + + if (evidence.embeds.length > 0) { + parts.push( + `[embeds: ${evidence.embeds + .map((embed) => + [ + embed.title ? `title=${embed.title}` : null, + embed.description ? `description=${embed.description}` : null, + embed.url ? `url=${embed.url}` : null, + embed.image ? `image=${embed.image}` : null, + embed.thumbnail ? `thumbnail=${embed.thumbnail}` : null, + embed.fields.length > 0 + ? `fields=${embed.fields.map((field) => `${field.name}: ${field.value}`).join("; ")}` + : null, + ] + .filter(Boolean) + .join(", "), + ) + .join(" | ")}]`, + ); + } + + if (evidence.attachments.length > 0) { + parts.push( + `[attachments: ${evidence.attachments + .map((attachment) => + [ + `name=${attachment.name}`, + attachment.contentType ? `type=${attachment.contentType}` : null, + `size=${attachment.size}`, + attachment.url ? `url=${attachment.url}` : null, + ] + .filter(Boolean) + .join(", "), + ) + .join(" | ")}]`, + ); + } + + return parts.join(" "); +} + export function getDisplayContent(message: Message): string { if (message.content.trim().length > 0) return message.content; diff --git a/tests/moderation/llmModerationClient.test.ts b/tests/moderation/llmModerationClient.test.ts index 611df21..102e1ed 100644 --- a/tests/moderation/llmModerationClient.test.ts +++ b/tests/moderation/llmModerationClient.test.ts @@ -77,6 +77,25 @@ describe("parseModerationResponse", () => { ]); }); + it("rejects deferral analysis text", () => { + expect(() => + parseModerationResponse( + JSON.stringify({ + results: [ + { + message_id: "m1", + status: "warn", + flags: [], + score: 0.4, + analysis: "Kurang konteks, perlu dicek admin.", + }, + ], + }), + ["m1"], + ), + ).toThrow(/deferral/i); + }); + it("handles missing target ids gracefully", () => { const result = parseModerationResponse(JSON.stringify({ results: [] }), [ "m1",