fix: false positive gambling detection + route collision + WS events (#1)
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
34719d3366
commit
f1ddca5eee
@@ -24,8 +24,9 @@ export function createMessagesRouter(): Router {
|
|||||||
// GET /api/messages/:channelId/attachments - Get attachments by channel
|
// GET /api/messages/:channelId/attachments - Get attachments by channel
|
||||||
router.get("/messages/:channelId/attachments", handleGetAttachmentsByChannel);
|
router.get("/messages/:channelId/attachments", handleGetAttachmentsByChannel);
|
||||||
|
|
||||||
// GET /api/messages/:id - Get single message by ID
|
// GET /api/messages/detail/:id - Get single message by ID
|
||||||
router.get("/messages/:id", handleGetMessageById);
|
// (uses /detail/ prefix to avoid collision with :channelId route above)
|
||||||
|
router.get("/messages/detail/:id", handleGetMessageById);
|
||||||
|
|
||||||
// POST /api/messages/:id/reanalyze - Mark message for re-analysis
|
// POST /api/messages/:id/reanalyze - Mark message for re-analysis
|
||||||
router.post(
|
router.post(
|
||||||
|
|||||||
@@ -15,10 +15,15 @@ const SUBSCRIPTIONS: ChannelMapping[] = [
|
|||||||
{ channel: "discord:message:updated", eventType: "message_updated" },
|
{ channel: "discord:message:updated", eventType: "message_updated" },
|
||||||
{ channel: "discord:message:deleted", eventType: "message_deleted" },
|
{ channel: "discord:message:deleted", eventType: "message_deleted" },
|
||||||
{ channel: "discord:message:analyzed", eventType: "message_analyzed" },
|
{ channel: "discord:message:analyzed", eventType: "message_analyzed" },
|
||||||
|
{ channel: "discord:attachment:created", eventType: "attachment_created" },
|
||||||
{ channel: "discord:attachment:uploaded", eventType: "attachment_uploaded" },
|
{ channel: "discord:attachment:uploaded", eventType: "attachment_uploaded" },
|
||||||
{ channel: "discord:voice:started", eventType: "voice_recording_started" },
|
{ channel: "discord:voice:started", eventType: "voice_recording_started" },
|
||||||
{ channel: "discord:voice:stopped", eventType: "voice_recording_stopped" },
|
{ channel: "discord:voice:stopped", eventType: "voice_recording_stopped" },
|
||||||
{ channel: "discord:voice:uploaded", eventType: "voice_recording_uploaded" },
|
{ channel: "discord:voice:uploaded", eventType: "voice_recording_uploaded" },
|
||||||
|
{
|
||||||
|
channel: "discord:analysis:queue_status",
|
||||||
|
eventType: "analysis_queue_status",
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
let subscriber: Redis | null = null;
|
let subscriber: Redis | null = null;
|
||||||
@@ -41,15 +46,28 @@ function handleSubscriptionMessage(channel: string, message: string): void {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let data: unknown;
|
let envelope: {
|
||||||
|
type?: string;
|
||||||
|
data?: unknown;
|
||||||
|
timestamp?: number;
|
||||||
|
source?: string;
|
||||||
|
};
|
||||||
try {
|
try {
|
||||||
data = JSON.parse(message);
|
envelope = JSON.parse(message);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error({ channel, err }, "Failed to parse Redis message as JSON");
|
logger.error({ channel, err }, "Failed to parse Redis message as JSON");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.debug({ channel, eventType: mapping.eventType }, "Broadcasting Redis event");
|
// Unwrap DiscordGatewayEvent envelope — the gateway publishes:
|
||||||
|
// { type, data: <actual payload>, timestamp, source }
|
||||||
|
// We only want <actual payload>, not the full envelope.
|
||||||
|
const data = envelope.data !== undefined ? envelope.data : envelope;
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
{ channel, eventType: mapping.eventType },
|
||||||
|
"Broadcasting Redis event",
|
||||||
|
);
|
||||||
broadcastRaw(mapping.eventType, data);
|
broadcastRaw(mapping.eventType, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { WebSocketServer, WebSocket } from "ws";
|
|
||||||
import type { Server } from "node:http";
|
import type { Server } from "node:http";
|
||||||
|
import { WebSocket, WebSocketServer } from "ws";
|
||||||
import { createChildLogger } from "../shared/logger/index.js";
|
import { createChildLogger } from "../shared/logger/index.js";
|
||||||
|
|
||||||
const logger = createChildLogger("ws.server");
|
const logger = createChildLogger("ws.server");
|
||||||
@@ -12,7 +12,6 @@ interface BroadcastEvent {
|
|||||||
|
|
||||||
// Extend globalThis with broadcast function types
|
// Extend globalThis with broadcast function types
|
||||||
declare global {
|
declare global {
|
||||||
// biome-ignore lint/suspicious/noAssignInExpressions: intentional global broadcast registry
|
|
||||||
var __broadcastFns:
|
var __broadcastFns:
|
||||||
| {
|
| {
|
||||||
messageCreated: (data: unknown) => void;
|
messageCreated: (data: unknown) => void;
|
||||||
@@ -24,6 +23,46 @@ declare global {
|
|||||||
| undefined;
|
| undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function sendInitialStates(ws: WebSocket): Promise<void> {
|
||||||
|
// Send initial user state
|
||||||
|
ws.send(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "user_state",
|
||||||
|
users: [],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Send initial UI state from database
|
||||||
|
try {
|
||||||
|
const { uiStateService } = await import(
|
||||||
|
"../modules/ui-state/ui-state.service.js"
|
||||||
|
);
|
||||||
|
const uiState = await uiStateService.getState();
|
||||||
|
ws.send(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "ui_state",
|
||||||
|
state: uiState,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn({ err }, "Failed to send initial ui_state");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send initial media state
|
||||||
|
try {
|
||||||
|
const { getStatus } = await import("../modules/media/media.service.js");
|
||||||
|
const mediaState = await getStatus();
|
||||||
|
ws.send(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "media_state",
|
||||||
|
state: mediaState,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn({ err }, "Failed to send initial media_state");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function createWebSocketServer(server: Server): WebSocketServer {
|
export function createWebSocketServer(server: Server): WebSocketServer {
|
||||||
const clients = new Set<WebSocket>();
|
const clients = new Set<WebSocket>();
|
||||||
|
|
||||||
@@ -33,19 +72,19 @@ export function createWebSocketServer(server: Server): WebSocketServer {
|
|||||||
clients.add(ws);
|
clients.add(ws);
|
||||||
logger.info(`Client connected (${clients.size} total)`);
|
logger.info(`Client connected (${clients.size} total)`);
|
||||||
|
|
||||||
// Send initial user state
|
// Send initial states (user, ui, media) — fire-and-forget
|
||||||
ws.send(
|
sendInitialStates(ws).catch((err) =>
|
||||||
JSON.stringify({
|
logger.error({ err }, "sendInitialStates failed"),
|
||||||
type: "user_state",
|
|
||||||
users: [],
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
ws.on("message", (data: Buffer) => {
|
ws.on("message", (data: Buffer) => {
|
||||||
// Binary PCM data received from browser.
|
// Binary PCM data received from browser.
|
||||||
// Since backend has no Discord client to relay to, drop it.
|
// Since backend has no Discord client to relay to, drop it.
|
||||||
if (Buffer.isBuffer(data) && data.length > 0) {
|
if (Buffer.isBuffer(data) && data.length > 0) {
|
||||||
logger.debug({ bytes: data.length }, "Dropping binary PCM (no Discord client)");
|
logger.debug(
|
||||||
|
{ bytes: data.length },
|
||||||
|
"Dropping binary PCM (no Discord client)",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -91,10 +130,14 @@ export function createWebSocketServer(server: Server): WebSocketServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
globalThis.__broadcastFns = {
|
globalThis.__broadcastFns = {
|
||||||
messageCreated: (data: unknown) => broadcast({ type: "message_created", data }),
|
messageCreated: (data: unknown) =>
|
||||||
messageUpdated: (data: unknown) => broadcast({ type: "message_updated", data }),
|
broadcast({ type: "message_created", data }),
|
||||||
messageDeleted: (data: unknown) => broadcast({ type: "message_deleted", data }),
|
messageUpdated: (data: unknown) =>
|
||||||
attachmentUploaded: (data: unknown) => broadcast({ type: "attachment_uploaded", data }),
|
broadcast({ type: "message_updated", data }),
|
||||||
|
messageDeleted: (data: unknown) =>
|
||||||
|
broadcast({ type: "message_deleted", data }),
|
||||||
|
attachmentUploaded: (data: unknown) =>
|
||||||
|
broadcast({ type: "attachment_uploaded", data }),
|
||||||
raw: (type: string, data: unknown) => broadcast({ type, data }),
|
raw: (type: string, data: unknown) => broadcast({ type, data }),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -4,10 +4,15 @@ import { z } from "zod";
|
|||||||
import { config } from "../../shared/config/config.js";
|
import { config } from "../../shared/config/config.js";
|
||||||
import { createChildLogger } from "../../shared/logger/logger.js";
|
import { createChildLogger } from "../../shared/logger/logger.js";
|
||||||
import { retryWithBackoff } from "../../shared/utils/retry.js";
|
import { retryWithBackoff } from "../../shared/utils/retry.js";
|
||||||
import { withLlmConcurrency } from "./concurrencyLimiter.js";
|
|
||||||
import { resizeImageForVision } from "../attachment-upload/imageResizer.js";
|
import { resizeImageForVision } from "../attachment-upload/imageResizer.js";
|
||||||
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
|
|
||||||
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
|
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
|
||||||
|
import type {
|
||||||
|
AnalysisResult,
|
||||||
|
AttachmentRecord,
|
||||||
|
MessageRecord,
|
||||||
|
} from "../message-capture/types.js";
|
||||||
|
import { withLlmConcurrency } from "./concurrencyLimiter.js";
|
||||||
|
import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js";
|
||||||
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
|
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
|
||||||
import {
|
import {
|
||||||
getStickerFromCache,
|
getStickerFromCache,
|
||||||
@@ -27,11 +32,6 @@ import {
|
|||||||
makeStickerCacheKey,
|
makeStickerCacheKey,
|
||||||
upsertCachedMediaAnalysis,
|
upsertCachedMediaAnalysis,
|
||||||
} from "./textCacheStore.js";
|
} from "./textCacheStore.js";
|
||||||
import type {
|
|
||||||
AnalysisResult,
|
|
||||||
AttachmentRecord,
|
|
||||||
MessageRecord,
|
|
||||||
} from "../message-capture/types.js";
|
|
||||||
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
|
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
|
||||||
|
|
||||||
const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]);
|
const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]);
|
||||||
@@ -587,7 +587,34 @@ const analyzeSingleMediaImage = async (
|
|||||||
? buildStickerVisionPrompt(image.stickerName, messageId)
|
? buildStickerVisionPrompt(image.stickerName, messageId)
|
||||||
: image.customEmojiName
|
: image.customEmojiName
|
||||||
? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId)
|
? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId)
|
||||||
: `Analisis media Discord berikut sebagai evidence moderasi. ${image.sourceLabel}\nJelaskan isi visual, teks yang terlihat, konteks risiko, dan apakah ada indikasi spam, scam, SARA, harassment, sexual content, violence, self-harm, doxxing, NSFW, gore, atau illegal content. Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek; berikan observasi langsung dari media.`;
|
: [
|
||||||
|
`Analisis media Discord berikut sebagai evidence moderasi. ${image.sourceLabel}`,
|
||||||
|
``,
|
||||||
|
`Jelaskan SECARA SPESIFIK apa yang TERLIHAT di gambar (objek, teks, warna dominan, layout).`,
|
||||||
|
`Jangan menebak-nebak atau mengasumsikan konten yang tidak terlihat langsung.`,
|
||||||
|
``,
|
||||||
|
`HANYA flag jika gambar secara JELAS dan TIDAK AMBIGU menampilkan:`,
|
||||||
|
`- Antarmuka situs judi yang JELAS TERLIHAT (ada chip, kartu, meja taruhan, odds, deposit/withdraw)`,
|
||||||
|
|
||||||
|
`- Konten seksual eksplisit/NSFW/pornografi`,
|
||||||
|
`- Darah/luka/gore nyata (BUKAN kartun/meme/animasi)`,
|
||||||
|
`- Narkoba atau obat terlarang yang bisa diidentifikasi spesifik`,
|
||||||
|
`- Simbol kebencian (swastika, simbol teroris) yang JELAS`,
|
||||||
|
`- Ajakan bunuh diri atau self-harm eksplisit`,
|
||||||
|
`- Informasi pribadi (alamat, nomor telepon, KTP) yang bocor`,
|
||||||
|
``,
|
||||||
|
`Jika gambar HANYA berisi: chat biasa, meme lucu, foto makanan, selfie,`,
|
||||||
|
`screenshot discord/wa/media sosial biasa, landscape, hewan, screenshot game,`,
|
||||||
|
`atau konten sehari-hari lainnya → JELASKAN ISI YANG TERLIHAT dan KONFIRMASI AMAN.`,
|
||||||
|
``,
|
||||||
|
`JANGAN PERNAH mengklaim gambar adalah "antarmuka judi" atau "situs perjudian"`,
|
||||||
|
`KECUALI ada BUKTI VISUAL SPESIFIK seperti chip, kartu remi, meja taruhan,`,
|
||||||
|
`odds, deposit/withdraw, atau logo situs judi yang DIKENALI.`,
|
||||||
|
`Screenshot aplikasi chat atau media sosial BUKAN situs judi.`,
|
||||||
|
``,
|
||||||
|
`Jawab Bahasa Indonesia, maksimal 3 kalimat. Berikan deskripsi objektif,`,
|
||||||
|
`bukan asumsi. Jika aman, katakan "aman" dengan jelas.`,
|
||||||
|
].join("\n");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const completion = await withLlmConcurrency(async () =>
|
const completion = await withLlmConcurrency(async () =>
|
||||||
|
|||||||
@@ -50,7 +50,13 @@ Prioritas rendah (PELANGGARAN RINGAN):
|
|||||||
3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high
|
3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high
|
||||||
4. Apakah ada spam/promosi borderline? → warn, low-medium
|
4. Apakah ada spam/promosi borderline? → warn, low-medium
|
||||||
5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean
|
5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean
|
||||||
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`;
|
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.
|
||||||
|
|
||||||
|
## ATURAN ANTI FALSE-POSITIVE UNTUK GAMBAR
|
||||||
|
- Jika teks pesan adalah percakapan normal sehari-hari (seperti "Aku suka nasgor loh", "Halo guys", "Pagi semua", dll), maka gambar yang menyertainya SANGAT TIDAK MUNGKIN adalah konten judi atau pelanggaran berat.
|
||||||
|
- Vision model sering salah mengklasifikasi screenshot biasa sebagai "antarmuka situs judi." Abaikan klaim vision model tentang judi KECUALI teks pesan juga mengandung promosi/link judi.
|
||||||
|
- Jika teks clean dan vision bilang "gambling" tapi TANPA menyebutkan bukti spesifik (chip, kartu, odds, logo judi dikenal) → OVERRIDE ke clean.
|
||||||
|
- Percaya teks terlebih dahulu. Bukti visual hanya pelengkap, bukan penentu utama.`;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Section: Media Instructions (conditional — injected when media present)
|
// Section: Media Instructions (conditional — injected when media present)
|
||||||
@@ -58,7 +64,17 @@ Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`;
|
|||||||
|
|
||||||
const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media
|
const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media
|
||||||
Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama.
|
Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama.
|
||||||
Gunakan baris "Media analysis" sebagai evidence visual utama dalam keputusan moderasi batch ini.
|
Baris "Media analysis" memberikan DESKRIPSI VISUAL tentang apa yang terlihat di gambar, BUKAN keputusan moderasi final.
|
||||||
|
|
||||||
|
## ATURAN KRITIS — Jangan Flag Gambar Biasa sebagai Judi
|
||||||
|
- **Screenshot Discord/WhatsApp/Telegram/media sosial biasa BUKAN situs judi.** Jangan pernah mengklaim screenshot chat sebagai "antarmuka perjudian".
|
||||||
|
- **Foto makanan, selfie, pemandangan, hewan, meme, screenshot game BUKAN konten melanggar.**
|
||||||
|
- **Gambar dengan teks bahasa Indonesia biasa BUKAN promosi judi.**
|
||||||
|
- HANYA flag "gambling" jika Media analysis secara SPESIFIK menyebutkan melihat elemen judi NYATA: chip, kartu remi, meja taruhan, odds, deposit/withdraw, atau logo situs judi yang DIKENALI.
|
||||||
|
- Jika Media analysis hanya mendeskripsikan "antarmuka aplikasi" atau "tampilan web" tanpa menyebutkan elemen judi spesifik → jangan flag gambling.
|
||||||
|
- Jika Media analysis bilang "aman" atau "tidak ada konten melanggar" → percaya itu.
|
||||||
|
- **Bukti teks LEBIH PENTING dari bukti visual.** Jika teks pesan adalah percakapan biasa ("Aku suka nasgor loh", "Halo guys", "Pagi semua") dan tidak mengandung promosi judi/narkoba/scam, maka gambar tersebut SANGAT TIDAK MUNGKIN adalah pelanggaran. JANGAN kontradiksi antara teks dan gambar — percaya teks terlebih dahulu.
|
||||||
|
- **Cross-check wajib**: Jika teks pesan clean/aman, maka analisis gambar yang mengklaim ada pelanggaran berat HARUS diabaikan kecuali ada bukti visual yang SANGAT JELAS dan TIDAK TERBANTAHKAN.
|
||||||
|
|
||||||
## Panduan Khusus Sticker
|
## Panduan Khusus Sticker
|
||||||
- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata.
|
- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata.
|
||||||
@@ -84,7 +100,15 @@ Output: {"results":[{"message_id":"67890","status":"flagged","flags":["harassmen
|
|||||||
|
|
||||||
Contoh 3 — Sticker kartun dengan nama provokatif:
|
Contoh 3 — Sticker kartun dengan nama provokatif:
|
||||||
Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"]
|
Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"]
|
||||||
Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}`;
|
Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}
|
||||||
|
|
||||||
|
Contoh 4 — Pesan biasa dengan gambar (JANGAN flag sebagai judi):
|
||||||
|
Input: [target] id=22222 user=rina: Aku suka nasgor loh [Media analysis for message 22222] [gambar di atas adalah attachment foto.jpg dari pesan id=22222]: Gambar menampilkan tangkapan layar aplikasi chat dengan teks percakapan biasa. Tidak ada konten melanggar terlihat. Aman.
|
||||||
|
Output: {"results":[{"message_id":"22222","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Pesan berisi percakapan sehari-hari tentang makanan. Gambar menunjukkan screenshot chat biasa tanpa pelanggaran."}]}
|
||||||
|
|
||||||
|
Contoh 5 — Pesan promosi judi dengan gambar situs judi:
|
||||||
|
Input: [target] id=33333 user=spammer: MAIN DI SINI GACOR PARAH https://judionline.xyz [Media analysis for message 33333] [gambar di atas adalah attachment slot.jpg dari pesan id=33333]: Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit. Terlihat logo "JudiOnline" dan odds taruhan.
|
||||||
|
Output: {"results":[{"message_id":"33333","status":"flagged","flags":["gambling"],"score":0.92,"categories":["gambling"],"severity":"high","confidence":0.92,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["MAIN DI SINI GACOR PARAH","https://judionline.xyz","Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit"],"analysis":"Promosi situs judi online dengan link, teks promosi, dan gambar antarmuka judi yang jelas."}]}`;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Section: Output Schema + XML Delimiter Instructions
|
// Section: Output Schema + XML Delimiter Instructions
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ export interface WsEventMap {
|
|||||||
ui_state: { state: unknown };
|
ui_state: { state: unknown };
|
||||||
media_state: { state: unknown };
|
media_state: { state: unknown };
|
||||||
voice_recording_uploaded: { data: unknown };
|
voice_recording_uploaded: { data: unknown };
|
||||||
|
voice_recording_started: { data: unknown };
|
||||||
|
voice_recording_stopped: { data: unknown };
|
||||||
|
attachment_created: { data: unknown };
|
||||||
|
analysis_queue_status: { data: unknown };
|
||||||
}
|
}
|
||||||
|
|
||||||
export type WsEventType = keyof WsEventMap;
|
export type WsEventType = keyof WsEventMap;
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ export interface WsHandlers {
|
|||||||
onUserState?: (users: unknown[]) => void;
|
onUserState?: (users: unknown[]) => void;
|
||||||
onUiState?: (state: unknown) => void;
|
onUiState?: (state: unknown) => void;
|
||||||
onMediaState?: (state: unknown) => void;
|
onMediaState?: (state: unknown) => void;
|
||||||
|
onVoiceRecordingStarted?: (data: unknown) => void;
|
||||||
|
onVoiceRecordingStopped?: (data: unknown) => void;
|
||||||
onVoiceRecordingUploaded?: (data: unknown) => void;
|
onVoiceRecordingUploaded?: (data: unknown) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -82,6 +84,19 @@ function doConnect(): WebSocket {
|
|||||||
case "voice_recording_uploaded":
|
case "voice_recording_uploaded":
|
||||||
h.onVoiceRecordingUploaded?.(msg.data);
|
h.onVoiceRecordingUploaded?.(msg.data);
|
||||||
break;
|
break;
|
||||||
|
case "voice_recording_started":
|
||||||
|
h.onVoiceRecordingStarted?.(msg.data);
|
||||||
|
break;
|
||||||
|
case "voice_recording_stopped":
|
||||||
|
h.onVoiceRecordingStopped?.(msg.data);
|
||||||
|
break;
|
||||||
|
case "attachment_created":
|
||||||
|
// attachment_created is informational — same data shape as message_created
|
||||||
|
h.onMessageCreated?.(msg.data);
|
||||||
|
break;
|
||||||
|
case "analysis_queue_status":
|
||||||
|
// analysis_queue_status is monitoring-only — no UI action needed
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
@@ -130,6 +145,10 @@ export function useDashboardSocket(handlers: WsHandlers) {
|
|||||||
onMediaState: (s) => handlersRef.current.onMediaState?.(s),
|
onMediaState: (s) => handlersRef.current.onMediaState?.(s),
|
||||||
onVoiceRecordingUploaded: (d) =>
|
onVoiceRecordingUploaded: (d) =>
|
||||||
handlersRef.current.onVoiceRecordingUploaded?.(d),
|
handlersRef.current.onVoiceRecordingUploaded?.(d),
|
||||||
|
onVoiceRecordingStarted: (d) =>
|
||||||
|
handlersRef.current.onVoiceRecordingStarted?.(d),
|
||||||
|
onVoiceRecordingStopped: (d) =>
|
||||||
|
handlersRef.current.onVoiceRecordingStopped?.(d),
|
||||||
};
|
};
|
||||||
|
|
||||||
_listeners.add(wrapper);
|
_listeners.add(wrapper);
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import {
|
|||||||
} from "./stickerCache.js";
|
} from "./stickerCache.js";
|
||||||
import {
|
import {
|
||||||
buildCustomEmojiVisionPrompt,
|
buildCustomEmojiVisionPrompt,
|
||||||
|
buildGeneralImageVisionPrompt,
|
||||||
buildStickerTextOnlyWarning,
|
buildStickerTextOnlyWarning,
|
||||||
buildStickerVisionPrompt,
|
buildStickerVisionPrompt,
|
||||||
} from "./stickerPrompt.js";
|
} from "./stickerPrompt.js";
|
||||||
@@ -587,7 +588,7 @@ const analyzeSingleMediaImage = async (
|
|||||||
? buildStickerVisionPrompt(image.stickerName, messageId)
|
? buildStickerVisionPrompt(image.stickerName, messageId)
|
||||||
: image.customEmojiName
|
: image.customEmojiName
|
||||||
? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId)
|
? buildCustomEmojiVisionPrompt(image.customEmojiName, messageId)
|
||||||
: `Analisis media Discord berikut sebagai evidence moderasi. ${image.sourceLabel}\nJelaskan isi visual, teks yang terlihat, konteks risiko, dan apakah ada indikasi spam, scam, SARA, harassment, sexual content, violence, self-harm, doxxing, NSFW, gore, atau illegal content. Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek; berikan observasi langsung dari media.`;
|
: buildGeneralImageVisionPrompt(image.sourceLabel, messageId);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const completion = await withLlmConcurrency(async () =>
|
const completion = await withLlmConcurrency(async () =>
|
||||||
|
|||||||
@@ -50,7 +50,13 @@ Prioritas rendah (PELANGGARAN RINGAN):
|
|||||||
3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high
|
3. Apakah ada harassment terarah/hate speech/sara? → flagged, medium-high
|
||||||
4. Apakah ada spam/promosi borderline? → warn, low-medium
|
4. Apakah ada spam/promosi borderline? → warn, low-medium
|
||||||
5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean
|
5. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean
|
||||||
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`;
|
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.
|
||||||
|
|
||||||
|
## ATURAN ANTI FALSE-POSITIVE UNTUK GAMBAR
|
||||||
|
- Jika teks pesan adalah percakapan normal sehari-hari (seperti "Aku suka nasgor loh", "Halo guys", "Pagi semua", dll), maka gambar yang menyertainya SANGAT TIDAK MUNGKIN adalah konten judi atau pelanggaran berat.
|
||||||
|
- Vision model sering salah mengklasifikasi screenshot biasa sebagai "antarmuka situs judi." Abaikan klaim vision model tentang judi KECUALI teks pesan juga mengandung promosi/link judi.
|
||||||
|
- Jika teks clean dan vision bilang "gambling" tapi TANPA menyebutkan bukti spesifik (chip, kartu, odds, logo judi dikenal) → OVERRIDE ke clean.
|
||||||
|
- Percaya teks terlebih dahulu. Bukti visual hanya pelengkap, bukan penentu utama.`;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Section: Media Instructions (conditional — injected when media present)
|
// Section: Media Instructions (conditional — injected when media present)
|
||||||
@@ -58,7 +64,17 @@ Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks.`;
|
|||||||
|
|
||||||
const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media
|
const MEDIA_INSTRUCTIONS = `## Instruksi Analisis Media
|
||||||
Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama.
|
Gambar, sticker, embed image, preview link, dan attachment sudah dianalisis lewat request media terpisah sebelum batch utama.
|
||||||
Gunakan baris "Media analysis" sebagai evidence visual utama dalam keputusan moderasi batch ini.
|
Baris "Media analysis" memberikan DESKRIPSI VISUAL tentang apa yang terlihat di gambar, BUKAN keputusan moderasi final.
|
||||||
|
|
||||||
|
## ATURAN KRITIS — Jangan Flag Gambar Biasa sebagai Judi
|
||||||
|
- **Screenshot Discord/WhatsApp/Telegram/media sosial biasa BUKAN situs judi.** Jangan pernah mengklaim screenshot chat sebagai "antarmuka perjudian".
|
||||||
|
- **Foto makanan, selfie, pemandangan, hewan, meme, screenshot game BUKAN konten melanggar.**
|
||||||
|
- **Gambar dengan teks bahasa Indonesia biasa BUKAN promosi judi.**
|
||||||
|
- HANYA flag "gambling" jika Media analysis secara SPESIFIK menyebutkan melihat elemen judi NYATA: chip, kartu remi, meja taruhan, odds, deposit/withdraw, atau logo situs judi yang DIKENALI.
|
||||||
|
- Jika Media analysis hanya mendeskripsikan "antarmuka aplikasi" atau "tampilan web" tanpa menyebutkan elemen judi spesifik → jangan flag gambling.
|
||||||
|
- Jika Media analysis bilang "aman" atau "tidak ada konten melanggar" → percaya itu.
|
||||||
|
- **Bukti teks LEBIH PENTING dari bukti visual.** Jika teks pesan adalah percakapan biasa ("Aku suka nasgor loh", "Halo guys", "Pagi semua") dan tidak mengandung promosi judi/narkoba/scam, maka gambar tersebut SANGAT TIDAK MUNGKIN adalah pelanggaran. JANGAN kontradiksi antara teks dan gambar — percaya teks terlebih dahulu.
|
||||||
|
- **Cross-check wajib**: Jika teks pesan clean/aman, maka analisis gambar yang mengklaim ada pelanggaran berat HARUS diabaikan kecuali ada bukti visual yang SANGAT JELAS dan TIDAK TERBANTAHKAN.
|
||||||
|
|
||||||
## Panduan Khusus Sticker
|
## Panduan Khusus Sticker
|
||||||
- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata.
|
- Sticker Discord adalah media kartun/meme/ilustrasi, BUKAN foto atau video nyata.
|
||||||
@@ -84,7 +100,15 @@ Output: {"results":[{"message_id":"67890","status":"flagged","flags":["harassmen
|
|||||||
|
|
||||||
Contoh 3 — Sticker kartun dengan nama provokatif:
|
Contoh 3 — Sticker kartun dengan nama provokatif:
|
||||||
Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"]
|
Input: [target] id=11111 user=citra: <:singa_injek:123456> [sticker: "Singa injek pejabat"]
|
||||||
Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}`;
|
Output: {"results":[{"message_id":"11111","status":"clean","flags":[],"score":0.1,"categories":[],"severity":"none","confidence":0.8,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Sticker kartun satir dengan nama provokatif namun bukan ancaman nyata."}]}
|
||||||
|
|
||||||
|
Contoh 4 — Pesan biasa dengan gambar (JANGAN flag sebagai judi):
|
||||||
|
Input: [target] id=22222 user=rina: Aku suka nasgor loh [Media analysis for message 22222] [gambar di atas adalah attachment foto.jpg dari pesan id=22222]: Gambar menampilkan tangkapan layar aplikasi chat dengan teks percakapan biasa. Tidak ada konten melanggar terlihat. Aman.
|
||||||
|
Output: {"results":[{"message_id":"22222","status":"clean","flags":[],"score":0.0,"categories":[],"severity":"none","confidence":0.95,"recommended_action":"none","policy_version":"default-2026-05-30","evidence":[],"analysis":"Pesan berisi percakapan sehari-hari tentang makanan. Gambar menunjukkan screenshot chat biasa tanpa pelanggaran."}]}
|
||||||
|
|
||||||
|
Contoh 5 — Pesan promosi judi dengan gambar situs judi:
|
||||||
|
Input: [target] id=33333 user=spammer: MAIN DI SINI GACOR PARAH https://judionline.xyz [Media analysis for message 33333] [gambar di atas adalah attachment slot.jpg dari pesan id=33333]: Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit. Terlihat logo "JudiOnline" dan odds taruhan.
|
||||||
|
Output: {"results":[{"message_id":"33333","status":"flagged","flags":["gambling"],"score":0.92,"categories":["gambling"],"severity":"high","confidence":0.92,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["MAIN DI SINI GACOR PARAH","https://judionline.xyz","Gambar menampilkan antarmuka situs judi online dengan mesin slot, chip, dan tombol deposit"],"analysis":"Promosi situs judi online dengan link, teks promosi, dan gambar antarmuka judi yang jelas."}]}`;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Section: Output Schema + XML Delimiter Instructions
|
// Section: Output Schema + XML Delimiter Instructions
|
||||||
|
|||||||
@@ -94,3 +94,45 @@ export function buildCustomEmojiTextOnlyFallback(emojiName: string): string {
|
|||||||
`Custom emoji di Discord adalah ekspresi/emosi umum, bukan konten ofensif.]`
|
`Custom emoji di Discord adalah ekspresi/emosi umum, bukan konten ofensif.]`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prompt used for general attachment/web images (not stickers or emojis).
|
||||||
|
*
|
||||||
|
* This is the default prompt for analyzing regular images. It includes
|
||||||
|
* explicit guidance on gambling detection to prevent false positives
|
||||||
|
* (e.g., flagging screenshots of food, finance, or casual content as gambling).
|
||||||
|
*/
|
||||||
|
export function buildGeneralImageVisionPrompt(
|
||||||
|
sourceLabel: string,
|
||||||
|
messageId: string,
|
||||||
|
): string {
|
||||||
|
return [
|
||||||
|
`Analisis media Discord berikut sebagai evidence moderasi.`,
|
||||||
|
`${sourceLabel}`,
|
||||||
|
``,
|
||||||
|
`PENTING — Panduan Deteksi Pelanggaran:`,
|
||||||
|
``,
|
||||||
|
`**Gambling/Judi:**`,
|
||||||
|
`- HANYA flag jika gambar JELAS menampilkan promosi atau antarmuka situs judi/taruhan nyata (poker, slots, sports betting).`,
|
||||||
|
`- Screenshot Discord/WhatsApp/Telegram/media sosial/chat biasa BUKAN situs judi — jangan flag.`,
|
||||||
|
`- Cryptocurrency UI atau finance apps (Coinbase, Binance, exchange) BUKAN judi — jangan flag sebagai gambling.`,
|
||||||
|
`- Screenshot ekonomi game, NFT marketplace, atau trading UI BUKAN judi — jangan flag.`,
|
||||||
|
`- Gambar makanan, meme, selfie, pemandangan atau konten casual TIDAK PERNAH gambling, bahkan jika ada teks finansial.`,
|
||||||
|
`- Hanya flag gambling jika ada BUKTI VISUAL JELAS: tombol taruhan, odds, chips, roulette wheel, atau branding situs judi terkenal.`,
|
||||||
|
`- Jika tidak yakin apakah sebuah gambar menampilkan situs judi atau sekadar aplikasi biasa → pilih "Tidak ada indikasi pelanggaran."`,
|
||||||
|
``,
|
||||||
|
`**Illegal Content:**`,
|
||||||
|
`- Jangan flag berdasarkan kecurigaan. HANYA flag jika ada BUKTI VISUAL JELAS dari konten ilegal (narkoba, senjata, dokumen palsu).`,
|
||||||
|
``,
|
||||||
|
`**Sexual Content & NSFW:**`,
|
||||||
|
`- Jangan flag foto casual/fashion hanya karena ada bagian tubuh. Flag HANYA jika gambar eksplisit atau pornografi.`,
|
||||||
|
``,
|
||||||
|
`**Violence:**`,
|
||||||
|
`- Screenshot game/anime action BUKAN kekerasan nyata — jangan flag.`,
|
||||||
|
`- Jangan flag meme atau karya seni yang menampilkan adegan dramatik.`,
|
||||||
|
``,
|
||||||
|
`Jelaskan isi visual, teks yang terlihat, dan konteks risiko.`,
|
||||||
|
`Jawab Bahasa Indonesia, maksimal 3 kalimat. Jangan bilang kurang konteks atau perlu admin cek; berikan observasi langsung dari media.`,
|
||||||
|
`Jika gambar tampak aman/tidak ada risiko, tulis: "Tidak ada indikasi pelanggaran terdeteksi."`,
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import {
|
|||||||
parseModerationResponse,
|
parseModerationResponse,
|
||||||
runModerationAnalysis,
|
runModerationAnalysis,
|
||||||
} from "../../src/moderation/llmModerationClient";
|
} from "../../src/moderation/llmModerationClient";
|
||||||
|
import { buildGeneralImageVisionPrompt } from "../../src/moderation/stickerPrompt";
|
||||||
import type { MessageRecord } from "../../src/moderation/types";
|
import type { MessageRecord } from "../../src/moderation/types";
|
||||||
|
|
||||||
vi.mock("../../src/retry", () => ({
|
vi.mock("../../src/retry", () => ({
|
||||||
@@ -1670,4 +1671,52 @@ describe("runModerationAnalysis", () => {
|
|||||||
expect(result[0].score).toBe(0.0);
|
expect(result[0].score).toBe(0.0);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("Vision prompts", () => {
|
||||||
|
it("buildGeneralImageVisionPrompt includes gambling false positive prevention", () => {
|
||||||
|
const prompt = buildGeneralImageVisionPrompt(
|
||||||
|
"[gambar dari attachment food.jpg]",
|
||||||
|
"msg123",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Should explicitly warn against false positives
|
||||||
|
expect(prompt).toContain("Cryptocurrency UI");
|
||||||
|
expect(prompt).toContain("Binance");
|
||||||
|
expect(prompt).toContain("BUKAN judi");
|
||||||
|
expect(prompt).toContain("finance apps");
|
||||||
|
|
||||||
|
// Should be clear about what IS gambling
|
||||||
|
expect(prompt).toContain("promosi atau antarmuka situs judi");
|
||||||
|
expect(prompt).toContain("BUKTI VISUAL JELAS");
|
||||||
|
|
||||||
|
// Should warn against hallucinations
|
||||||
|
expect(prompt).toContain("Jangan flag berdasarkan kecurigaan");
|
||||||
|
expect(prompt).toContain("BUKTI VISUAL JELAS");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("buildGeneralImageVisionPrompt warns against hallucinating gambling from casual images", () => {
|
||||||
|
const prompt = buildGeneralImageVisionPrompt(
|
||||||
|
"[gambar nasgor/fried rice]",
|
||||||
|
"msg123",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Should explicitly state casual images are not gambling
|
||||||
|
expect(prompt).toContain("Gambar makanan");
|
||||||
|
expect(prompt).toContain("TIDAK PERNAH gambling");
|
||||||
|
expect(prompt).toContain("bahkan jika ada teks finansial");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("buildGeneralImageVisionPrompt warns against false positive crypto flagging", () => {
|
||||||
|
const prompt = buildGeneralImageVisionPrompt(
|
||||||
|
"[screenshot dari exchange]",
|
||||||
|
"msg123",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Explicitly state crypto/finance UI is NOT gambling
|
||||||
|
expect(prompt).toContain("Cryptocurrency UI atau finance apps");
|
||||||
|
expect(prompt).toContain("Coinbase");
|
||||||
|
expect(prompt).toContain("exchange");
|
||||||
|
expect(prompt).toContain("BUKAN judi");
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user