From 60084b3cc3bbd835dc35604f7bb98a31cf23751b Mon Sep 17 00:00:00 2001 From: Developer Date: Fri, 31 Jul 2026 19:11:13 +0700 Subject: [PATCH] fix(automod): flow real LLM analysis + descriptive fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: ai-analysis-worker read llmResult.explanation and llmResult.toxicityScore — fields the LLM pipeline never produces (canonical AnalysisResult uses analysis/score). Every message fell back to the bare template "Tidak ada indikasi pelanggaran." and the stored score was always 0. - Map analysis/score correctly; fallback now quotes the message content - Prompt: ban generic analysis phrasing, require reply context - LLM context: include replied-to message content (metadata.reference) so the model can explain what the user is replying to - Frontend: show thread/channel names from metadata instead of raw IDs (message card, detail views, search overlay); detail panel now displays the ai_analysis text - Auto-delete log/DM include the descriptive analysis as the reason --- .../ai-moderation/ai-analysis-worker.ts | 57 +++++++++++-------- .../modules/ai-moderation/autoDeleteLogger.ts | 2 + .../modules/ai-moderation/autoDeleteNotify.ts | 8 ++- .../ai-moderation/conversationContext.ts | 36 +++++++++++- .../modules/ai-moderation/prompts/output.ts | 2 + .../components/messages/ai-analysis-panel.tsx | 8 +++ .../src/components/messages/message-card.tsx | 13 ++++- .../messages/message-detail-view.tsx | 9 ++- .../components/messages/message-detail.tsx | 9 ++- .../components/messages/search-overlay.tsx | 3 +- services/frontend/src/lib/format.ts | 29 ++++++++++ 11 files changed, 142 insertions(+), 34 deletions(-) diff --git a/services/discord-gateway/src/modules/ai-moderation/ai-analysis-worker.ts b/services/discord-gateway/src/modules/ai-moderation/ai-analysis-worker.ts index 08726f8..faf4c7f 100644 --- a/services/discord-gateway/src/modules/ai-moderation/ai-analysis-worker.ts +++ b/services/discord-gateway/src/modules/ai-moderation/ai-analysis-worker.ts @@ -58,11 +58,8 @@ export interface AnalysisResult { | "review" | "delete" | "escalate"; - toxicityScore: number; - harmScore: number; - jailbreakScore: number; - safetyScore: number; - explanation: string; + score: number; + analysis: string; correctedFlags?: string[]; } @@ -182,7 +179,7 @@ export default async function workerRouter( * Runs the LLM moderation analysis on a single message. * * The LLM verdict IS the result — confidence, severity, flags and - * explanation all come from the model. On failure the message is marked + * analysis all come from the model. On failure the message is marked * "error" (explicit, retryable) instead of receiving a heuristic verdict. */ async function runLLMAnalysis( @@ -214,15 +211,10 @@ async function runLLMAnalysis( severity: llmResult.severity ?? "none", confidence: normalizeConfidence(llmResult.confidence), recommendedAction: llmResult.recommendedAction ?? "none", - toxicityScore: llmResult.toxicityScore ?? 0, - harmScore: llmResult.harmScore ?? 0, - jailbreakScore: llmResult.jailbreakScore ?? 0, - safetyScore: llmResult.safetyScore ?? 0, - explanation: - llmResult.explanation?.trim() || - (llmResult.status === "clean" - ? "Tidak ada indikasi pelanggaran." - : "Pesan terindikasi melanggar kebijakan (analisis AI)."), + score: llmResult.score ?? 0, + analysis: + llmResult.analysis?.trim() || + buildFallbackAnalysis(message, llmResult.status ?? "clean"), }; } catch (error) { const errorMsg = error instanceof Error ? error.message : String(error); @@ -242,6 +234,28 @@ function normalizeConfidence(raw: number | undefined | null): number { return 0.7; } +/** + * Content-aware fallback when the LLM returns an empty `analysis` field. + * Quotes the actual message content so the log still explains WHAT was said + * instead of a bare template like "Tidak ada indikasi pelanggaran." + */ +function buildFallbackAnalysis( + message: MessageRecord, + status: string, +): string { + const raw = (message.edited_content ?? message.content ?? "").trim(); + const snippet = raw.length > 120 ? `${raw.slice(0, 120).trimEnd()}…` : raw; + + if (status === "clean") { + return snippet + ? `Tidak ada indikasi pelanggaran. Pesan: "${snippet}" dinilai wajar dalam konteks percakapan.` + : "Tidak ada indikasi pelanggaran. Isi pesan dinilai wajar dalam konteks percakapan."; + } + return snippet + ? `Pesan terindikasi melanggar kebijakan: "${snippet}".` + : "Pesan terindikasi melanggar kebijakan (analisis AI)."; +} + function buildFallbackResult( messageId: string, reason: string, @@ -254,11 +268,8 @@ function buildFallbackResult( severity: "none", confidence: 0, recommendedAction: "review", - toxicityScore: 0, - harmScore: 0, - jailbreakScore: 0, - safetyScore: 0, - explanation: reason, + score: 0, + analysis: reason, }; } @@ -319,14 +330,14 @@ async function processBatch(job: { result: { status: result.status, flags: JSON.stringify(result.flags), - score: result.toxicityScore, - analysis: result.explanation, + score: result.score, + analysis: result.analysis, categories: result.categories, severity: result.severity, confidence: result.confidence, recommendedAction: result.recommendedAction, analyzedAt: Date.now(), - error: result.status === "error" ? result.explanation : null, + error: result.status === "error" ? result.analysis : null, }, })); diff --git a/services/discord-gateway/src/modules/ai-moderation/autoDeleteLogger.ts b/services/discord-gateway/src/modules/ai-moderation/autoDeleteLogger.ts index d56f379..149fa26 100644 --- a/services/discord-gateway/src/modules/ai-moderation/autoDeleteLogger.ts +++ b/services/discord-gateway/src/modules/ai-moderation/autoDeleteLogger.ts @@ -33,6 +33,7 @@ export async function logDeletionToChannel( const severity = message.ai_severity ?? "none"; const categories = message.ai_categories ?? message.ai_moderation_flags ?? "—"; + const reason = message.ai_analysis ?? "—"; const snippet = (message.edited_content ?? message.content).substring( 0, 200, @@ -42,6 +43,7 @@ export async function logDeletionToChannel( `**Status:** ${message.ai_status}\n` + `**Severitas:** ${severity}\n` + `**Kategori:** ${categories}\n` + + `**Alasan:** ${reason}\n` + `**Isi:** ${snippet}\n` + `**Waktu:** `, ); diff --git a/services/discord-gateway/src/modules/ai-moderation/autoDeleteNotify.ts b/services/discord-gateway/src/modules/ai-moderation/autoDeleteNotify.ts index fff8ec9..9fc78b5 100644 --- a/services/discord-gateway/src/modules/ai-moderation/autoDeleteNotify.ts +++ b/services/discord-gateway/src/modules/ai-moderation/autoDeleteNotify.ts @@ -20,8 +20,14 @@ export async function sendDeletionNotification( try { const targetUser = await client.users.fetch(message.user_id); if (targetUser) { + // Prefer the descriptive LLM analysis so the user understands WHY; + // fall back to category/flag labels when it is unavailable. + const analysis = (message.ai_analysis ?? "").trim(); const reason: string = - message.ai_categories ?? message.ai_moderation_flags ?? "(unknown)"; + (analysis.length > 240 ? `${analysis.slice(0, 240)}…` : analysis) || + message.ai_categories ?? + message.ai_moderation_flags ?? + "(unknown)"; await targetUser.send( `Pesan Anda di **${guildName}** telah dihapus oleh sistem moderasi otomatis.\n` + `Alasan: ${reason}\n` + diff --git a/services/discord-gateway/src/modules/ai-moderation/conversationContext.ts b/services/discord-gateway/src/modules/ai-moderation/conversationContext.ts index 8f831c9..fdc5439 100644 --- a/services/discord-gateway/src/modules/ai-moderation/conversationContext.ts +++ b/services/discord-gateway/src/modules/ai-moderation/conversationContext.ts @@ -42,12 +42,44 @@ export function estimateTokens(text: string): number { } /** - * Formats reference info for a message (reply/forward/crosspost) + * Formats reference info for a message (reply/forward/crosspost). + * + * The replied-to content is stored in the message metadata + * (`metadata.reference.content` / `repliedUsername`) at capture time, so + * include it here — the LLM can then explain WHAT the user is replying to + * instead of only seeing a raw message ID it cannot resolve. */ function formatReferenceInfo(msg: MessageRecord): string { const parts: string[] = []; + + let repliedContent: string | null = null; + let repliedUsername: string | null = null; + try { + const meta = JSON.parse(msg.metadata ?? "") as { + reference?: { + content?: string | null; + repliedUsername?: string | null; + } | null; + }; + repliedContent = meta?.reference?.content ?? null; + repliedUsername = meta?.reference?.repliedUsername ?? null; + } catch { + // metadata malformed — fall back to ID-only reference + } + + const repliedText = (repliedContent ?? "").trim(); + const repliedSnippet = repliedText + ? sanitizeDiscordTokens( + repliedText.length > 200 + ? `${repliedText.slice(0, 200)}…` + : repliedText, + ) + : null; + if (msg.is_reply && msg.reference_message_id) { - parts.push(`[reply_to: ${msg.reference_message_id}]`); + const who = repliedUsername ? ` oleh ${repliedUsername}` : ""; + const what = repliedSnippet ? `: "${repliedSnippet}"` : ""; + parts.push(`[reply_to: ${msg.reference_message_id}${who}${what}]`); if (msg.reference_channel_id) { parts.push(`(reply_channel: ${msg.reference_channel_id})`); } diff --git a/services/discord-gateway/src/modules/ai-moderation/prompts/output.ts b/services/discord-gateway/src/modules/ai-moderation/prompts/output.ts index 15c9ef0..13f99b3 100644 --- a/services/discord-gateway/src/modules/ai-moderation/prompts/output.ts +++ b/services/discord-gateway/src/modules/ai-moderation/prompts/output.ts @@ -153,9 +153,11 @@ Contoh buruk: "Pengirim bercanda tentang agama." (JANGAN menggunakan kata "berca CRITICAL: - JANGAN PERNAH menulis "Pesan hanya berisi..." atau "Pesan tidak mengandung..." sebagai analysis. +- JANGAN PERNAH menulis "Tidak ada indikasi pelanggaran" atau frasa generik serupa sebagai analysis — wajib sebutkan TOPIK/ISI pesan secara spesifik apa yang sedang dibicarakan pengirim. - JANGAN PERNAH menulis template generik seperti "Pengirim mengirimkan sebuah file GIF tanpa pelanggaran". Kamu WAJIB mendeskripsikan isi visualnya secara spesifik berdasarkan Media analysis. - JANGAN PERNAH menyebutkan nama / username pengguna secara langsung. Selalu gunakan kata "Pengirim" atau "Pengguna". - Selalu sebutkan ISI KONTEN secara spesifik — apa yang dibicarakan, apa yang terlihat di gambar. +- Jika pesan adalah BALASAN (reply) ke pesan lain, jelaskan konteks balasannya: apa yang sedang dibicarakan, siapa yang dibalas (tanpa nama, cukup peran/isi pesan yang dibalas), dan bagaimana tanggapan pengirim terhadapnya. - Gunakan informasi dari Media analysis untuk mendeskripsikan gambar. - Analisis harus MEMBERI KONTEKS, bukan hanya menyatakan status. - GUNAKAN untuk personalisasi analysis — jadikan analysis terasa seperti sistem "mengenal" pengguna. diff --git a/services/frontend/src/components/messages/ai-analysis-panel.tsx b/services/frontend/src/components/messages/ai-analysis-panel.tsx index cd9f495..5300eb1 100644 --- a/services/frontend/src/components/messages/ai-analysis-panel.tsx +++ b/services/frontend/src/components/messages/ai-analysis-panel.tsx @@ -11,6 +11,7 @@ interface AiAnalysisPanelProps { categories?: string[] | string | null; action?: string | null; score?: number | null; + analysis?: string | null; } const severityColor: Record = { @@ -29,6 +30,7 @@ export function AiAnalysisPanel({ categories, action, score, + analysis, }: AiAnalysisPanelProps) { if (!status || status === "pending") { return ( @@ -93,6 +95,12 @@ export function AiAnalysisPanel({ )} + {analysis && ( +

+ {analysis} +

+ )} + {action && action !== "none" && (
Recommended: diff --git a/services/frontend/src/components/messages/message-card.tsx b/services/frontend/src/components/messages/message-card.tsx index c498e07..38c0bb5 100644 --- a/services/frontend/src/components/messages/message-card.tsx +++ b/services/frontend/src/components/messages/message-card.tsx @@ -6,7 +6,7 @@ import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; import { Progress } from "@/components/ui/progress"; -import { safeParseJsonArray } from "@/lib/format"; +import { safeParseJsonArray, getMessageChannelLabel } from "@/lib/format"; import type { MessageRecord } from "@/lib/types"; import { cn } from "@/lib/utils"; import { AiStatusBadge } from "./ai-status-badge"; @@ -52,9 +52,16 @@ export function MessageCard({ {new Date(msg.created_at).toLocaleString()} - + - {msg.channel_id.slice(0, 8)} + {getMessageChannelLabel(msg)} {msg.ai_severity && msg.ai_severity !== "none" && ( diff --git a/services/frontend/src/components/messages/message-detail-view.tsx b/services/frontend/src/components/messages/message-detail-view.tsx index e1c7333..708d883 100644 --- a/services/frontend/src/components/messages/message-detail-view.tsx +++ b/services/frontend/src/components/messages/message-detail-view.tsx @@ -1,9 +1,10 @@ "use client"; -import { ArrowLeft, MessageSquare } from "lucide-react"; +import { ArrowLeft, MessageSquare, MessagesSquare } from "lucide-react"; import { GlassCard } from "@/components/glass/card"; import { AttachmentsGrid } from "./attachments-grid"; import { AiAnalysisPanel } from "./ai-analysis-panel"; +import { getMessageChannelLabel } from "@/lib/format"; import type { AttachmentRecord, MessageRecord } from "@/lib/types"; interface MessageDetailViewProps { @@ -25,7 +26,10 @@ export function MessageDetailView({ message, attachments, onBack }: MessageDetai
{message.username} - {message.channel_id?.slice(0, 8)} + + {message.thread_id && } + {getMessageChannelLabel(message)} +
{/* Content */} @@ -49,6 +53,7 @@ export function MessageDetailView({ message, attachments, onBack }: MessageDetai categories={message.ai_categories} action={message.ai_recommended_action} score={message.ai_moderation_score} + analysis={message.ai_analysis} /> ); diff --git a/services/frontend/src/components/messages/message-detail.tsx b/services/frontend/src/components/messages/message-detail.tsx index d6c7153..8696e4d 100644 --- a/services/frontend/src/components/messages/message-detail.tsx +++ b/services/frontend/src/components/messages/message-detail.tsx @@ -1,9 +1,10 @@ "use client"; -import { ArrowLeft, MessageSquare } from "lucide-react"; +import { ArrowLeft, MessageSquare, MessagesSquare } from "lucide-react"; import { GlassCard } from "@/components/glass/card"; import { AttachmentsGrid } from "./attachments-grid"; import { AiAnalysisPanel } from "./ai-analysis-panel"; +import { getMessageChannelLabel } from "@/lib/format"; import type { AttachmentRecord, MessageRecord } from "@/lib/types"; interface MessageDetailProps { @@ -25,7 +26,10 @@ export function MessageDetail({ message, attachments, onBack }: MessageDetailPro
{message.username} - {message.channel_id?.slice(0, 8)} + + {message.thread_id && } + {getMessageChannelLabel(message)} +
{/* Content */} @@ -49,6 +53,7 @@ export function MessageDetail({ message, attachments, onBack }: MessageDetailPro categories={message.ai_categories} action={message.ai_recommended_action} score={message.ai_moderation_score} + analysis={message.ai_analysis} /> ); diff --git a/services/frontend/src/components/messages/search-overlay.tsx b/services/frontend/src/components/messages/search-overlay.tsx index 154fa18..f6bbe65 100644 --- a/services/frontend/src/components/messages/search-overlay.tsx +++ b/services/frontend/src/components/messages/search-overlay.tsx @@ -4,6 +4,7 @@ import { Search, X } from "lucide-react"; import { useEffect, useRef, useState } from "react"; import { useQuery } from "@tanstack/react-query"; import { messagesApi } from "@/lib/api"; +import { getMessageChannelLabel } from "@/lib/format"; import type { MessageRecord } from "@/lib/types"; interface SearchOverlayProps { @@ -83,7 +84,7 @@ export function SearchOverlay({ open, onClose, onSelect }: SearchOverlayProps) { >
{msg.username} - {msg.channel_id?.slice(0, 8)} + {getMessageChannelLabel(msg)}

{msg.content}

diff --git a/services/frontend/src/lib/format.ts b/services/frontend/src/lib/format.ts index 1554f22..33f860e 100644 --- a/services/frontend/src/lib/format.ts +++ b/services/frontend/src/lib/format.ts @@ -43,3 +43,32 @@ export function safeParseJsonObject( return {}; } } + +/** + * Resolve a human-readable channel/thread label for a message. + * + * The channel name (and thread name, when the message lives in a thread) + * is captured by the gateway into the message metadata JSON under + * `metadata.channel.{channelName,threadName}`. Prefer names over raw IDs: + * a thread message shows its thread name, otherwise the channel name, + * falling back to a truncated channel ID only when names are unavailable. + */ +export function getMessageChannelLabel(msg: { + channel_id?: string; + metadata?: string | null; +}): string { + let channelName: string | undefined; + let threadName: string | undefined; + try { + const m = JSON.parse(msg.metadata ?? ""); + const ch = m?.channel; + channelName = + typeof ch?.channelName === "string" ? ch.channelName : undefined; + threadName = typeof ch?.threadName === "string" ? ch.threadName : undefined; + } catch { + // metadata malformed — fall through to ID fallback + } + if (threadName) return threadName; + if (channelName) return channelName; + return msg.channel_id?.slice(0, 8) ?? ""; +}