fix: false positive gambling detection + route collision + WS events (#1)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-06-02 10:44:27 +07:00
co-authored by Claude Opus 4.8
parent 34719d3366
commit f1ddca5eee
11 changed files with 285 additions and 33 deletions
@@ -24,8 +24,9 @@ export function createMessagesRouter(): Router {
// GET /api/messages/:channelId/attachments - Get attachments by channel // GET /api/messages/:channelId/attachments - Get attachments by channel
router.get("/messages/:channelId/attachments", handleGetAttachmentsByChannel); router.get("/messages/:channelId/attachments", handleGetAttachmentsByChannel);
// GET /api/messages/:id - Get single message by ID // GET /api/messages/detail/:id - Get single message by ID
router.get("/messages/:id", handleGetMessageById); // (uses /detail/ prefix to avoid collision with :channelId route above)
router.get("/messages/detail/:id", handleGetMessageById);
// POST /api/messages/:id/reanalyze - Mark message for re-analysis // POST /api/messages/:id/reanalyze - Mark message for re-analysis
router.post( router.post(
+21 -3
View File
@@ -15,10 +15,15 @@ const SUBSCRIPTIONS: ChannelMapping[] = [
{ channel: "discord:message:updated", eventType: "message_updated" }, { channel: "discord:message:updated", eventType: "message_updated" },
{ channel: "discord:message:deleted", eventType: "message_deleted" }, { channel: "discord:message:deleted", eventType: "message_deleted" },
{ channel: "discord:message:analyzed", eventType: "message_analyzed" }, { channel: "discord:message:analyzed", eventType: "message_analyzed" },
{ channel: "discord:attachment:created", eventType: "attachment_created" },
{ channel: "discord:attachment:uploaded", eventType: "attachment_uploaded" }, { channel: "discord:attachment:uploaded", eventType: "attachment_uploaded" },
{ channel: "discord:voice:started", eventType: "voice_recording_started" }, { channel: "discord:voice:started", eventType: "voice_recording_started" },
{ channel: "discord:voice:stopped", eventType: "voice_recording_stopped" }, { channel: "discord:voice:stopped", eventType: "voice_recording_stopped" },
{ channel: "discord:voice:uploaded", eventType: "voice_recording_uploaded" }, { channel: "discord:voice:uploaded", eventType: "voice_recording_uploaded" },
{
channel: "discord:analysis:queue_status",
eventType: "analysis_queue_status",
},
]; ];
let subscriber: Redis | null = null; let subscriber: Redis | null = null;
@@ -41,15 +46,28 @@ function handleSubscriptionMessage(channel: string, message: string): void {
return; return;
} }
let data: unknown; let envelope: {
type?: string;
data?: unknown;
timestamp?: number;
source?: string;
};
try { try {
data = JSON.parse(message); envelope = JSON.parse(message);
} catch (err) { } catch (err) {
logger.error({ channel, err }, "Failed to parse Redis message as JSON"); logger.error({ channel, err }, "Failed to parse Redis message as JSON");
return; return;
} }
logger.debug({ channel, eventType: mapping.eventType }, "Broadcasting Redis event"); // Unwrap DiscordGatewayEvent envelope — the gateway publishes:
// { type, data: <actual payload>, timestamp, source }
// We only want <actual payload>, not the full envelope.
const data = envelope.data !== undefined ? envelope.data : envelope;
logger.debug(
{ channel, eventType: mapping.eventType },
"Broadcasting Redis event",
);
broadcastRaw(mapping.eventType, data); broadcastRaw(mapping.eventType, data);
} }
+56 -13
View File
@@ -1,5 +1,5 @@
import { WebSocketServer, WebSocket } from "ws";
import type { Server } from "node:http"; import type { Server } from "node:http";
import { WebSocket, WebSocketServer } from "ws";
import { createChildLogger } from "../shared/logger/index.js"; import { createChildLogger } from "../shared/logger/index.js";
const logger = createChildLogger("ws.server"); const logger = createChildLogger("ws.server");
@@ -12,7 +12,6 @@ interface BroadcastEvent {
// Extend globalThis with broadcast function types // Extend globalThis with broadcast function types
declare global { declare global {
// biome-ignore lint/suspicious/noAssignInExpressions: intentional global broadcast registry
var __broadcastFns: var __broadcastFns:
| { | {
messageCreated: (data: unknown) => void; messageCreated: (data: unknown) => void;
@@ -24,6 +23,46 @@ declare global {
| undefined; | undefined;
} }
async function sendInitialStates(ws: WebSocket): Promise<void> {
// Send initial user state
ws.send(
JSON.stringify({
type: "user_state",
users: [],
}),
);
// Send initial UI state from database
try {
const { uiStateService } = await import(
"../modules/ui-state/ui-state.service.js"
);
const uiState = await uiStateService.getState();
ws.send(
JSON.stringify({
type: "ui_state",
state: uiState,
}),
);
} catch (err) {
logger.warn({ err }, "Failed to send initial ui_state");
}
// Send initial media state
try {
const { getStatus } = await import("../modules/media/media.service.js");
const mediaState = await getStatus();
ws.send(
JSON.stringify({
type: "media_state",
state: mediaState,
}),
);
} catch (err) {
logger.warn({ err }, "Failed to send initial media_state");
}
}
export function createWebSocketServer(server: Server): WebSocketServer { export function createWebSocketServer(server: Server): WebSocketServer {
const clients = new Set<WebSocket>(); const clients = new Set<WebSocket>();
@@ -33,19 +72,19 @@ export function createWebSocketServer(server: Server): WebSocketServer {
clients.add(ws); clients.add(ws);
logger.info(`Client connected (${clients.size} total)`); logger.info(`Client connected (${clients.size} total)`);
// Send initial user state // Send initial states (user, ui, media) — fire-and-forget
ws.send( sendInitialStates(ws).catch((err) =>
JSON.stringify({ logger.error({ err }, "sendInitialStates failed"),
type: "user_state",
users: [],
}),
); );
ws.on("message", (data: Buffer) => { ws.on("message", (data: Buffer) => {
// Binary PCM data received from browser. // Binary PCM data received from browser.
// Since backend has no Discord client to relay to, drop it. // Since backend has no Discord client to relay to, drop it.
if (Buffer.isBuffer(data) && data.length > 0) { if (Buffer.isBuffer(data) && data.length > 0) {
logger.debug({ bytes: data.length }, "Dropping binary PCM (no Discord client)"); logger.debug(
{ bytes: data.length },
"Dropping binary PCM (no Discord client)",
);
} }
}); });
@@ -91,10 +130,14 @@ export function createWebSocketServer(server: Server): WebSocketServer {
} }
globalThis.__broadcastFns = { globalThis.__broadcastFns = {
messageCreated: (data: unknown) => broadcast({ type: "message_created", data }), messageCreated: (data: unknown) =>
messageUpdated: (data: unknown) => broadcast({ type: "message_updated", data }), broadcast({ type: "message_created", data }),
messageDeleted: (data: unknown) => broadcast({ type: "message_deleted", data }), messageUpdated: (data: unknown) =>
attachmentUploaded: (data: unknown) => broadcast({ type: "attachment_uploaded", data }), broadcast({ type: "message_updated", data }),
messageDeleted: (data: unknown) =>
broadcast({ type: "message_deleted", data }),
attachmentUploaded: (data: unknown) =>
broadcast({ type: "attachment_uploaded", data }),
raw: (type: string, data: unknown) => broadcast({ type, data }), raw: (type: string, data: unknown) => broadcast({ type, data }),
}; };
@@ -4,10 +4,15 @@ import { z } from "zod";
import { config } from "../../shared/config/config.js"; import { config } from "../../shared/config/config.js";
import { createChildLogger } from "../../shared/logger/logger.js"; import { createChildLogger } from "../../shared/logger/logger.js";
import { retryWithBackoff } from "../../shared/utils/retry.js"; import { retryWithBackoff } from "../../shared/utils/retry.js";
import { withLlmConcurrency } from "./concurrencyLimiter.js";
import { resizeImageForVision } from "../attachment-upload/imageResizer.js"; import { resizeImageForVision } from "../attachment-upload/imageResizer.js";
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js"; import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
import type {
AnalysisResult,
AttachmentRecord,
MessageRecord,
} from "../message-capture/types.js";
import { withLlmConcurrency } from "./concurrencyLimiter.js";
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js"; import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
import { import {
getStickerFromCache, getStickerFromCache,
@@ -27,11 +32,6 @@ import {
makeStickerCacheKey, makeStickerCacheKey,
upsertCachedMediaAnalysis, upsertCachedMediaAnalysis,
} from "./textCacheStore.js"; } from "./textCacheStore.js";
import type {
AnalysisResult,
AttachmentRecord,
MessageRecord,
} from "../message-capture/types.js";
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js"; import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]); const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]);
@@ -587,7 +587,34 @@ const analyzeSingleMediaImage = async (
? buildStickerVisionPrompt(image.stickerName, messageId) ? buildStickerVisionPrompt(image.stickerName, messageId)
: image.customEmojiName : image.customEmojiName
? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId) ? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId)
: `Analisis media Discord berikut sebagai evidence moderasi. ${image.sourceLabel}\nJelaskan isi visual, teks yang terlihat, konteks risiko, dan apakah ada indikasi spam, scam, SARA, harassment, sexual content, violence, self-harm, doxxing, NSFW, gore, atau illegal content. Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek; berikan observasi langsung dari media.`; : [
`Analisis media Discord berikut sebagai evidence moderasi. ${image.sourceLabel}`,
``,
`Jelaskan SECARA SPESIFIK apa yang TERLIHAT di gambar (objek, teks, warna dominan, layout).`,
`Jangan menebak-nebak atau mengasumsikan konten yang tidak terlihat langsung.`,
``,
`HANYA flag jika gambar secara JELAS dan TIDAK AMBIGU menampilkan:`,
`- Antarmuka situs judi yang JELAS TERLIHAT (ada chip, kartu, meja taruhan, odds, deposit/withdraw)`,
`- Konten seksual eksplisit/NSFW/pornografi`,
`- Darah/luka/gore nyata (BUKAN kartun/meme/animasi)`,
`- Narkoba atau obat terlarang yang bisa diidentifikasi spesifik`,
`- Simbol kebencian (swastika, simbol teroris) yang JELAS`,
`- Ajakan bunuh diri atau self-harm eksplisit`,
`- Informasi pribadi (alamat, nomor telepon, KTP) yang bocor`,
``,
`Jika gambar HANYA berisi: chat biasa, meme lucu, foto makanan, selfie,`,
`screenshot discord/wa/media sosial biasa, landscape, hewan, screenshot game,`,
`atau konten sehari-hari lainnya → JELASKAN ISI YANG TERLIHAT dan KONFIRMASI AMAN.`,
``,
`JANGAN PERNAH mengklaim gambar adalah "antarmuka judi" atau "situs perjudian"`,
`KECUALI ada BUKTI VISUAL SPESIFIK seperti chip, kartu remi, meja taruhan,`,
`odds, deposit/withdraw, atau logo situs judi yang DIKENALI.`,
`Screenshot aplikasi chat atau media sosial BUKAN situs judi.`,
``,
`Jawab Bahasa Indonesia, maksimal 3 kalimat. Berikan deskripsi objektif,`,
`bukan asumsi. Jika aman, katakan "aman" dengan jelas.`,
].join("\n");
try { try {
const completion = await withLlmConcurrency(async () => const completion = await withLlmConcurrency(async () =>
@@ -50,7 +50,13 @@ Prioritas rendah (PELANGGARAN RINGAN):
3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high 3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high
4. Apakah ada spam/promosi borderline? → warn, low-medium 4. Apakah ada spam/promosi borderline? → warn, low-medium
5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean 5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`; Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.
## ATURAN ANTI FALSE-POSITIVE UNTUK GAMBAR
- Jika teks pesan adalah percakapan normal sehari-hari (seperti "Aku suka nasgor loh", "Halo guys", "Pagi semua", dll), maka gambar yang menyertainya SANGAT TIDAK MUNGKIN adalah konten judi atau pelanggaran berat.
- Vision model sering salah mengklasifikasi screenshot biasa sebagai "antarmuka situs judi." Abaikan klaim vision model tentang judi KECUALI teks pesan juga mengandung promosi/link judi.
- Jika teks clean dan vision bilang "gambling" tapi TANPA menyebutkan bukti spesifik (chip, kartu, odds, logo judi dikenal) → OVERRIDE ke clean.
- Percaya teks terlebih dahulu. Bukti visual hanya pelengkap, bukan penentu utama.`;
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Section: Media Instructions (conditional — injected when media present) // Section: Media Instructions (conditional — injected when media present)
@@ -58,7 +64,17 @@ Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`;
const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media
Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama. Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama.
Gunakan baris "Media analysis" sebagai evidence visual utama dalam keputusan moderasi batch ini. Baris "Media analysis" memberikan DESKRIPSI VISUAL tentang apa yang terlihat di gambar, BUKAN keputusan moderasi final.
## ATURAN KRITIS — Jangan Flag Gambar Biasa sebagai Judi
- **Screenshot Discord/WhatsApp/Telegram/media sosial biasa BUKAN situs judi.** Jangan pernah mengklaim screenshot chat sebagai "antarmuka perjudian".
- **Foto makanan, selfie, pemandangan, hewan, meme, screenshot game BUKAN konten melanggar.**
- **Gambar dengan teks bahasa Indonesia biasa BUKAN promosi judi.**
- HANYA flag "gambling" jika Media analysis secara SPESIFIK menyebutkan melihat elemen judi NYATA: chip, kartu remi, meja taruhan, odds, deposit/withdraw, atau logo situs judi yang DIKENALI.
- Jika Media analysis hanya mendeskripsikan "antarmuka aplikasi" atau "tampilan web" tanpa menyebutkan elemen judi spesifik → jangan flag gambling.
- Jika Media analysis bilang "aman" atau "tidak ada konten melanggar" → percaya itu.
- **Bukti teks LEBIH PENTING dari bukti visual.** Jika teks pesan adalah percakapan biasa ("Aku suka nasgor loh", "Halo guys", "Pagi semua") dan tidak mengandung promosi judi/narkoba/scam, maka gambar tersebut SANGAT TIDAK MUNGKIN adalah pelanggaran. JANGAN kontradiksi antara teks dan gambar — percaya teks terlebih dahulu.
- **Cross-check wajib**: Jika teks pesan clean/aman, maka analisis gambar yang mengklaim ada pelanggaran berat HARUS diabaikan kecuali ada bukti visual yang SANGAT JELAS dan TIDAK TERBANTAHKAN.
## Panduan Khusus Sticker ## Panduan Khusus Sticker
- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata. - Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata.
@@ -84,7 +100,15 @@ Output: {"results":[{"message_id":"67890","status":"flagged","flags":["harassmen
Contoh 3 — Sticker kartun dengan nama provokatif: Contoh 3 — Sticker kartun dengan nama provokatif:
Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"] Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"]
Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}`; Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}
Contoh 4 — Pesan biasa dengan gambar (JANGAN flag sebagai judi):
Input: [target] id=22222 user=rina: Aku suka nasgor loh [Media analysis for message 22222] [gambar di atas adalah attachment foto.jpg dari pesan id=22222]: Gambar menampilkan tangkapan layar aplikasi chat dengan teks percakapan biasa. Tidak ada konten melanggar terlihat. Aman.
Output: {"results":[{"message_id":"22222","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Pesan berisi percakapan sehari-hari tentang makanan. Gambar menunjukkan screenshot chat biasa tanpa pelanggaran."}]}
Contoh 5 — Pesan promosi judi dengan gambar situs judi:
Input: [target] id=33333 user=spammer: MAIN DI SINI GACOR PARAH https://judionline.xyz [Media analysis for message 33333] [gambar di atas adalah attachment slot.jpg dari pesan id=33333]: Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit. Terlihat logo "JudiOnline" dan odds taruhan.
Output: {"results":[{"message_id":"33333","status":"flagged","flags":["gambling"],"score":0.92,"categories":["gambling"],"severity":"high","confidence":0.92,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["MAIN DI SINI GACOR PARAH","https://judionline.xyz","Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit"],"analysis":"Promosi situs judi online dengan link, teks promosi, dan gambar antarmuka judi yang jelas."}]}`;
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Section: Output Schema + XML Delimiter Instructions // Section: Output Schema + XML Delimiter Instructions
@@ -10,6 +10,10 @@ export interface WsEventMap {
ui_state: { state: unknown }; ui_state: { state: unknown };
media_state: { state: unknown }; media_state: { state: unknown };
voice_recording_uploaded: { data: unknown }; voice_recording_uploaded: { data: unknown };
voice_recording_started: { data: unknown };
voice_recording_stopped: { data: unknown };
attachment_created: { data: unknown };
analysis_queue_status: { data: unknown };
} }
export type WsEventType = keyof WsEventMap; export type WsEventType = keyof WsEventMap;
+19
View File
@@ -15,6 +15,8 @@ export interface WsHandlers {
onUserState?: (users: unknown[]) => void; onUserState?: (users: unknown[]) => void;
onUiState?: (state: unknown) => void; onUiState?: (state: unknown) => void;
onMediaState?: (state: unknown) => void; onMediaState?: (state: unknown) => void;
onVoiceRecordingStarted?: (data: unknown) => void;
onVoiceRecordingStopped?: (data: unknown) => void;
onVoiceRecordingUploaded?: (data: unknown) => void; onVoiceRecordingUploaded?: (data: unknown) => void;
} }
@@ -82,6 +84,19 @@ function doConnect(): WebSocket {
case "voice_recording_uploaded": case "voice_recording_uploaded":
h.onVoiceRecordingUploaded?.(msg.data); h.onVoiceRecordingUploaded?.(msg.data);
break; break;
case "voice_recording_started":
h.onVoiceRecordingStarted?.(msg.data);
break;
case "voice_recording_stopped":
h.onVoiceRecordingStopped?.(msg.data);
break;
case "attachment_created":
// attachment_created is informational — same data shape as message_created
h.onMessageCreated?.(msg.data);
break;
case "analysis_queue_status":
// analysis_queue_status is monitoring-only — no UI action needed
break;
} }
} }
} catch { } catch {
@@ -130,6 +145,10 @@ export function useDashboardSocket(handlers: WsHandlers) {
onMediaState: (s) => handlersRef.current.onMediaState?.(s), onMediaState: (s) => handlersRef.current.onMediaState?.(s),
onVoiceRecordingUploaded: (d) => onVoiceRecordingUploaded: (d) =>
handlersRef.current.onVoiceRecordingUploaded?.(d), handlersRef.current.onVoiceRecordingUploaded?.(d),
onVoiceRecordingStarted: (d) =>
handlersRef.current.onVoiceRecordingStarted?.(d),
onVoiceRecordingStopped: (d) =>
handlersRef.current.onVoiceRecordingStopped?.(d),
}; };
_listeners.add(wrapper); _listeners.add(wrapper);
+2 -1
View File
@@ -17,6 +17,7 @@ import {
} from "./stickerCache.js"; } from "./stickerCache.js";
import { import {
buildCustomEmojiVisionPrompt, buildCustomEmojiVisionPrompt,
buildGeneralImageVisionPrompt,
buildStickerTextOnlyWarning, buildStickerTextOnlyWarning,
buildStickerVisionPrompt, buildStickerVisionPrompt,
} from "./stickerPrompt.js"; } from "./stickerPrompt.js";
@@ -587,7 +588,7 @@ const analyzeSingleMediaImage = async (
? buildStickerVisionPrompt(image.stickerName, messageId) ? buildStickerVisionPrompt(image.stickerName, messageId)
: image.customEmojiName : image.customEmojiName
? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId) ? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId)
: `Analisis media Discord berikut sebagai evidence moderasi. ${image.sourceLabel}\nJelaskan isi visual, teks yang terlihat, konteks risiko, dan apakah ada indikasi spam, scam, SARA, harassment, sexual content, violence, self-harm, doxxing, NSFW, gore, atau illegal content. Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek; berikan observasi langsung dari media.`; : buildGeneralImageVisionPrompt(image.sourceLabel, messageId);
try { try {
const completion = await withLlmConcurrency(async () => const completion = await withLlmConcurrency(async () =>
+27 -3
View File
@@ -50,7 +50,13 @@ Prioritas rendah (PELANGGARAN RINGAN):
3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high 3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high
4. Apakah ada spam/promosi borderline? → warn, low-medium 4. Apakah ada spam/promosi borderline? → warn, low-medium
5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean 5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`; Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.
## ATURAN ANTI FALSE-POSITIVE UNTUK GAMBAR
- Jika teks pesan adalah percakapan normal sehari-hari (seperti "Aku suka nasgor loh", "Halo guys", "Pagi semua", dll), maka gambar yang menyertainya SANGAT TIDAK MUNGKIN adalah konten judi atau pelanggaran berat.
- Vision model sering salah mengklasifikasi screenshot biasa sebagai "antarmuka situs judi." Abaikan klaim vision model tentang judi KECUALI teks pesan juga mengandung promosi/link judi.
- Jika teks clean dan vision bilang "gambling" tapi TANPA menyebutkan bukti spesifik (chip, kartu, odds, logo judi dikenal) → OVERRIDE ke clean.
- Percaya teks terlebih dahulu. Bukti visual hanya pelengkap, bukan penentu utama.`;
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Section: Media Instructions (conditional — injected when media present) // Section: Media Instructions (conditional — injected when media present)
@@ -58,7 +64,17 @@ Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`;
const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media
Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama. Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama.
Gunakan baris "Media analysis" sebagai evidence visual utama dalam keputusan moderasi batch ini. Baris "Media analysis" memberikan DESKRIPSI VISUAL tentang apa yang terlihat di gambar, BUKAN keputusan moderasi final.
## ATURAN KRITIS — Jangan Flag Gambar Biasa sebagai Judi
- **Screenshot Discord/WhatsApp/Telegram/media sosial biasa BUKAN situs judi.** Jangan pernah mengklaim screenshot chat sebagai "antarmuka perjudian".
- **Foto makanan, selfie, pemandangan, hewan, meme, screenshot game BUKAN konten melanggar.**
- **Gambar dengan teks bahasa Indonesia biasa BUKAN promosi judi.**
- HANYA flag "gambling" jika Media analysis secara SPESIFIK menyebutkan melihat elemen judi NYATA: chip, kartu remi, meja taruhan, odds, deposit/withdraw, atau logo situs judi yang DIKENALI.
- Jika Media analysis hanya mendeskripsikan "antarmuka aplikasi" atau "tampilan web" tanpa menyebutkan elemen judi spesifik → jangan flag gambling.
- Jika Media analysis bilang "aman" atau "tidak ada konten melanggar" → percaya itu.
- **Bukti teks LEBIH PENTING dari bukti visual.** Jika teks pesan adalah percakapan biasa ("Aku suka nasgor loh", "Halo guys", "Pagi semua") dan tidak mengandung promosi judi/narkoba/scam, maka gambar tersebut SANGAT TIDAK MUNGKIN adalah pelanggaran. JANGAN kontradiksi antara teks dan gambar — percaya teks terlebih dahulu.
- **Cross-check wajib**: Jika teks pesan clean/aman, maka analisis gambar yang mengklaim ada pelanggaran berat HARUS diabaikan kecuali ada bukti visual yang SANGAT JELAS dan TIDAK TERBANTAHKAN.
## Panduan Khusus Sticker ## Panduan Khusus Sticker
- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata. - Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata.
@@ -84,7 +100,15 @@ Output: {"results":[{"message_id":"67890","status":"flagged","flags":["harassmen
Contoh 3 — Sticker kartun dengan nama provokatif: Contoh 3 — Sticker kartun dengan nama provokatif:
Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"] Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"]
Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}`; Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}
Contoh 4 — Pesan biasa dengan gambar (JANGAN flag sebagai judi):
Input: [target] id=22222 user=rina: Aku suka nasgor loh [Media analysis for message 22222] [gambar di atas adalah attachment foto.jpg dari pesan id=22222]: Gambar menampilkan tangkapan layar aplikasi chat dengan teks percakapan biasa. Tidak ada konten melanggar terlihat. Aman.
Output: {"results":[{"message_id":"22222","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Pesan berisi percakapan sehari-hari tentang makanan. Gambar menunjukkan screenshot chat biasa tanpa pelanggaran."}]}
Contoh 5 — Pesan promosi judi dengan gambar situs judi:
Input: [target] id=33333 user=spammer: MAIN DI SINI GACOR PARAH https://judionline.xyz [Media analysis for message 33333] [gambar di atas adalah attachment slot.jpg dari pesan id=33333]: Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit. Terlihat logo "JudiOnline" dan odds taruhan.
Output: {"results":[{"message_id":"33333","status":"flagged","flags":["gambling"],"score":0.92,"categories":["gambling"],"severity":"high","confidence":0.92,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["MAIN DI SINI GACOR PARAH","https://judionline.xyz","Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit"],"analysis":"Promosi situs judi online dengan link, teks promosi, dan gambar antarmuka judi yang jelas."}]}`;
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Section: Output Schema + XML Delimiter Instructions // Section: Output Schema + XML Delimiter Instructions
+42
View File
@@ -94,3 +94,45 @@ export function buildCustomEmojiTextOnlyFallback(emojiName: string): string {
`Custom emoji di Discord adalah ekspresi/emosi umum, bukan konten ofensif.]` `Custom emoji di Discord adalah ekspresi/emosi umum, bukan konten ofensif.]`
); );
} }
/**
* Prompt used for general attachment/web images (not stickers or emojis).
*
* This is the default prompt for analyzing regular images. It includes
* explicit guidance on gambling detection to prevent false positives
* (e.g., flagging screenshots of food, finance, or casual content as gambling).
*/
export function buildGeneralImageVisionPrompt(
sourceLabel: string,
messageId: string,
): string {
return [
`Analisis media Discord berikut sebagai evidence moderasi.`,
`${sourceLabel}`,
``,
`PENTING — Panduan Deteksi Pelanggaran:`,
``,
`**Gambling/Judi:**`,
`- HANYA flag jika gambar JELAS menampilkan promosi atau antarmuka situs judi/taruhan nyata (poker, slots, sports betting).`,
`- Screenshot Discord/WhatsApp/Telegram/media sosial/chat biasa BUKAN situs judi — jangan flag.`,
`- Cryptocurrency UI atau finance apps (Coinbase, Binance, exchange) BUKAN judi — jangan flag sebagai gambling.`,
`- Screenshot ekonomi game, NFT marketplace, atau trading UI BUKAN judi — jangan flag.`,
`- Gambar makanan, meme, selfie, pemandangan atau konten casual TIDAK PERNAH gambling, bahkan jika ada teks finansial.`,
`- Hanya flag gambling jika ada BUKTI VISUAL JELAS: tombol taruhan, odds, chips, roulette wheel, atau branding situs judi terkenal.`,
`- Jika tidak yakin apakah sebuah gambar menampilkan situs judi atau sekadar aplikasi biasa → pilih "Tidak ada indikasi pelanggaran."`,
``,
`**Illegal Content:**`,
`- Jangan flag berdasarkan kecurigaan. HANYA flag jika ada BUKTI VISUAL JELAS dari konten ilegal (narkoba, senjata, dokumen palsu).`,
``,
`**Sexual Content & NSFW:**`,
`- Jangan flag foto casual/fashion hanya karena ada bagian tubuh. Flag HANYA jika gambar eksplisit atau pornografi.`,
``,
`**Violence:**`,
`- Screenshot game/anime action BUKAN kekerasan nyata — jangan flag.`,
`- Jangan flag meme atau karya seni yang menampilkan adegan dramatik.`,
``,
`Jelaskan isi visual, teks yang terlihat, dan konteks risiko.`,
`Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek; berikan observasi langsung dari media.`,
`Jika gambar tampak aman/tidak ada risiko, tulis: "Tidak ada indikasi pelanggaran terdeteksi."`,
].join("\n");
}
@@ -3,6 +3,7 @@ import {
parseModerationResponse, parseModerationResponse,
runModerationAnalysis, runModerationAnalysis,
} from "../../src/moderation/llmModerationClient"; } from "../../src/moderation/llmModerationClient";
import { buildGeneralImageVisionPrompt } from "../../src/moderation/stickerPrompt";
import type { MessageRecord } from "../../src/moderation/types"; import type { MessageRecord } from "../../src/moderation/types";
vi.mock("../../src/retry", () => ({ vi.mock("../../src/retry", () => ({
@@ -1670,4 +1671,52 @@ describe("runModerationAnalysis", () => {
expect(result[0].score).toBe(0.0); expect(result[0].score).toBe(0.0);
}); });
}); });
describe("Vision prompts", () => {
it("buildGeneralImageVisionPrompt includes gambling false positive prevention", () => {
const prompt = buildGeneralImageVisionPrompt(
"[gambar dari attachment food.jpg]",
"msg123",
);
// Should explicitly warn against false positives
expect(prompt).toContain("Cryptocurrency UI");
expect(prompt).toContain("Binance");
expect(prompt).toContain("BUKAN judi");
expect(prompt).toContain("finance apps");
// Should be clear about what IS gambling
expect(prompt).toContain("promosi atau antarmuka situs judi");
expect(prompt).toContain("BUKTI VISUAL JELAS");
// Should warn against hallucinations
expect(prompt).toContain("Jangan flag berdasarkan kecurigaan");
expect(prompt).toContain("BUKTI VISUAL JELAS");
});
it("buildGeneralImageVisionPrompt warns against hallucinating gambling from casual images", () => {
const prompt = buildGeneralImageVisionPrompt(
"[gambar nasgor/fried rice]",
"msg123",
);
// Should explicitly state casual images are not gambling
expect(prompt).toContain("Gambar makanan");
expect(prompt).toContain("TIDAK PERNAH gambling");
expect(prompt).toContain("bahkan jika ada teks finansial");
});
it("buildGeneralImageVisionPrompt warns against false positive crypto flagging", () => {
const prompt = buildGeneralImageVisionPrompt(
"[screenshot dari exchange]",
"msg123",
);
// Explicitly state crypto/finance UI is NOT gambling
expect(prompt).toContain("Cryptocurrency UI atau finance apps");
expect(prompt).toContain("Coinbase");
expect(prompt).toContain("exchange");
expect(prompt).toContain("BUKAN judi");
});
});
}); });