From fb09ac81c5c3074f1daec1d4dbfd5fdc25fea584 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Fri, 29 May 2026 18:39:10 +0700 Subject: [PATCH] feat(moderation): Indonesian slang normalizer and false-positive prevention - Add indonesian-badwords dependency for local lexical signal - Add Indonesian slang lexicon with woy/woi/hadeh as safe casual terms - Normalize Discord custom emoji <:name:id> to [emoji:name] in prompts - Wire normalization evidence into both conversationContext and llmModerationClient prompts - Harden system prompt: woy/woi are casual greetings, not SARA/hate - Add tests for emoji normalization, slang mapping, badword detection Co-Authored-By: Claude Opus 4.8 --- package.json | 1 + pnpm-lock.yaml | 82 ++++++++------ src/moderation/conversationContext.ts | 5 +- src/moderation/indonesianTextNormalizer.ts | 107 ++++++++++++++++++ src/moderation/llmModerationClient.ts | 10 +- .../resources/indonesianSlangLexicon.ts | 45 ++++++++ src/types/indonesian-badwords.d.ts | 25 ++++ .../indonesianTextNormalizer.test.ts | 92 +++++++++++++++ 8 files changed, 327 insertions(+), 40 deletions(-) create mode 100644 src/moderation/indonesianTextNormalizer.ts create mode 100644 src/moderation/resources/indonesianSlangLexicon.ts create mode 100644 src/types/indonesian-badwords.d.ts create mode 100644 tests/moderation/indonesianTextNormalizer.test.ts diff --git a/package.json b/package.json index 378c066..04b60ef 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "drizzle-orm": "^0.45.2", "express": "^5.2.1", "helmet": "^8.1.0", + "indonesian-badwords": "^1.0.1", "libsodium-wrappers": "^0.8.4", "lucide-react": "^1.16.0", "openai": "^6.38.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f813180..adb1506 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: helmet: specifier: ^8.1.0 version: 8.1.0 + indonesian-badwords: + specifier: ^1.0.1 + version: 1.0.1 libsodium-wrappers: specifier: ^0.8.4 version: 0.8.4 @@ -110,7 +113,7 @@ importers: devDependencies: '@biomejs/biome': specifier: latest - version: 2.4.15 + version: 2.4.16 '@tailwindcss/postcss': specifier: ^4.3.0 version: 4.3.0 @@ -250,7 +253,7 @@ importers: devDependencies: '@biomejs/biome': specifier: latest - version: 2.4.15 + version: 2.4.16 '@discordjs/docgen': specifier: ^0.11.1 version: 0.11.1 @@ -321,8 +324,8 @@ packages: engines: {node: '>=14.21.3'} hasBin: true - '@biomejs/biome@2.4.15': - resolution: {integrity: sha512-j5VH3a/h/HXTKBM50MDMxRCzkeLv9S2XJcW2WgnZT1+xyisi+0bISrXR82gCX+8S9lvK0skEvHJRN+3Ktr2hlw==} + '@biomejs/biome@2.4.16': + resolution: {integrity: sha512-x9ajFh1zChVybCiM3TN6OD4phAqLgtPZjFrZF+aTMYCPjwBO+k529TX7PPsAqtGNLeV4UgzwQnowEgS7bGmzcA==} engines: {node: '>=14.21.3'} hasBin: true @@ -332,8 +335,8 @@ packages: cpu: [arm64] os: [darwin] - '@biomejs/cli-darwin-arm64@2.4.15': - resolution: {integrity: sha512-rF3PPqLq1yoST79zaQbDjVJwsuIeci/O+9bgNmC5QpgOqz6aqYuzA4abyAGx+mgyiDXn4A049xAN8gijbuR1Qg==} + '@biomejs/cli-darwin-arm64@2.4.16': + resolution: {integrity: sha512-wxPvu4XOA85YJk9ixSWUmq/QBHbid85BISbOAqqBM/5xQpPk9ayjk5375tOlSC0BeCwNSbPFafQBm+vBumXq0A==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] @@ -344,8 +347,8 @@ packages: cpu: [x64] os: [darwin] - '@biomejs/cli-darwin-x64@2.4.15': - resolution: {integrity: sha512-/5KHXYMfSJs1fNXiX30xFtI8JcCFV6zaVVLxOa0M2sfqBKHkpQhRTv94yxQWxeTY2lzo2OuTlNvPC+hDQt2wcQ==} + '@biomejs/cli-darwin-x64@2.4.16': + resolution: {integrity: sha512-xFCqGPwYusQJp4N4NJLi1XJiZqjwFdjhT+KqtNy+Ug3qgfczqnTa6MSDvxJF6TkuDLoYJItMapz6tAf7kCekFw==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] @@ -357,8 +360,8 @@ packages: os: [linux] libc: [musl] - '@biomejs/cli-linux-arm64-musl@2.4.15': - resolution: {integrity: sha512-ZPcxznxm0pogHBLZhYntyR3sR+MrZjqJIKEr7ZqVen0Rl+P/4upVmfYXjftizi9RoqZntg33fv/1fbdhbYXpEQ==} + '@biomejs/cli-linux-arm64-musl@2.4.16': + resolution: {integrity: sha512-oYxnW0ARfJkr72ezzF2OR8N/rtkgLUQeYtF8cFhVswbknHxtTcmzSsanVJP8yQKnGpGpc2ck6c5zLvHahL6Cbg==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] @@ -371,8 +374,8 @@ packages: os: [linux] libc: [glibc] - '@biomejs/cli-linux-arm64@2.4.15': - resolution: {integrity: sha512-owaAMZD/T4LrD0ELNCk0Km3qrRHuM0X6EAyVE1FSqGY0rbLoiDLrO4Us2tllm6cAeB2Ioa9C2C08NZPdr8+0Ug==} + '@biomejs/cli-linux-arm64@2.4.16': + resolution: {integrity: sha512-2kFb4//jxfZaP6D+Rj5VkHkxgyD9EoRAVBEQb8PKRv+s4NO2zYNJKXFaJmK1CmhufJOWEfpHKaRbOja7qjmdhQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] @@ -385,8 +388,8 @@ packages: os: [linux] libc: [musl] - '@biomejs/cli-linux-x64-musl@2.4.15': - resolution: {integrity: sha512-CNq/9W38SYSH023lfcQ4KKU8K0YX8T//FZUhcgtMMRABDojx5XsMV7jlweAvGSl389wJQB29Qo6Zb/a+jdvt+w==} + '@biomejs/cli-linux-x64-musl@2.4.16': + resolution: {integrity: sha512-iHDS+MCM65DPqWGu+ECC3uoALyj2H7F4nVUPxIPjz/PIl94EUu+EDfGZDzFP+NY1EOPVt9NQvwFqq7HdMmowdg==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] @@ -399,8 +402,8 @@ packages: os: [linux] libc: [glibc] - '@biomejs/cli-linux-x64@2.4.15': - resolution: {integrity: sha512-0jj7THz12GbUOLmMibktK6DZjqz2zV64KFxyBtcFTKPiiOIY0a7vns1elpO1dERvxpsZ5ik0oFfz0oGwFde1+g==} + '@biomejs/cli-linux-x64@2.4.16': + resolution: {integrity: sha512-NbcBbi/nJqn5baae6wqRXdS7Gadf2uRpehSh6vMSYpG8OhkXl/Xg8aorWrJ+9VWqAT5ml90alLvorkpMW0nBwQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] @@ -412,8 +415,8 @@ packages: cpu: [arm64] os: [win32] - '@biomejs/cli-win32-arm64@2.4.15': - resolution: {integrity: sha512-ouhkYdlhp/1GghEJPdWwD/Vi3gQ1nFxuSpMolWsbq3Lsq3QUR4jl6UdhhscdCugKU5vOEuMiJhvKj66O0OCq+w==} + '@biomejs/cli-win32-arm64@2.4.16': + resolution: {integrity: sha512-0rgImMsNb5v/chhkIFe3wu7PEFClS6RBAYUijGL9UsYN3PanSaoK24HSSuSJb1pYbYYVjzAyZTl3gtjJ84BM8A==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] @@ -424,8 +427,8 @@ packages: cpu: [x64] os: [win32] - '@biomejs/cli-win32-x64@2.4.15': - resolution: {integrity: sha512-zBrGq5mx5wwpnow4+2BxUvleDM+GNd4sLbPaMapsSLQLD0NGRCquqPBTgN+7XkUteHvj7M+BstuI8tmnV7+HgQ==} + '@biomejs/cli-win32-x64@2.4.16': + resolution: {integrity: sha512-Kp85jgoBHa05gix6UIRjfCDiUV3w/8VIdZ247VyyO2gEjaw12WEVhdIjlxp/AMzXxqxQwbxNTDVZ3Mwd2RG5rw==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] @@ -3098,6 +3101,9 @@ packages: resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} engines: {node: '>=8'} + indonesian-badwords@1.0.1: + resolution: {integrity: sha512-A8V6hklYqql15yelhnx5jkoDLru9hPr/q7TE6S4fICZvb7lbluWUXqGjcWpgmDCsbXpecs38/fbuhvM0em/7CQ==} + inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. @@ -4935,63 +4941,63 @@ snapshots: '@biomejs/cli-win32-arm64': 2.3.8 '@biomejs/cli-win32-x64': 2.3.8 - '@biomejs/biome@2.4.15': + '@biomejs/biome@2.4.16': optionalDependencies: - '@biomejs/cli-darwin-arm64': 2.4.15 - '@biomejs/cli-darwin-x64': 2.4.15 - '@biomejs/cli-linux-arm64': 2.4.15 - '@biomejs/cli-linux-arm64-musl': 2.4.15 - '@biomejs/cli-linux-x64': 2.4.15 - '@biomejs/cli-linux-x64-musl': 2.4.15 - '@biomejs/cli-win32-arm64': 2.4.15 - '@biomejs/cli-win32-x64': 2.4.15 + '@biomejs/cli-darwin-arm64': 2.4.16 + '@biomejs/cli-darwin-x64': 2.4.16 + '@biomejs/cli-linux-arm64': 2.4.16 + '@biomejs/cli-linux-arm64-musl': 2.4.16 + '@biomejs/cli-linux-x64': 2.4.16 + '@biomejs/cli-linux-x64-musl': 2.4.16 + '@biomejs/cli-win32-arm64': 2.4.16 + '@biomejs/cli-win32-x64': 2.4.16 '@biomejs/cli-darwin-arm64@2.3.8': optional: true - '@biomejs/cli-darwin-arm64@2.4.15': + '@biomejs/cli-darwin-arm64@2.4.16': optional: true '@biomejs/cli-darwin-x64@2.3.8': optional: true - '@biomejs/cli-darwin-x64@2.4.15': + '@biomejs/cli-darwin-x64@2.4.16': optional: true '@biomejs/cli-linux-arm64-musl@2.3.8': optional: true - '@biomejs/cli-linux-arm64-musl@2.4.15': + '@biomejs/cli-linux-arm64-musl@2.4.16': optional: true '@biomejs/cli-linux-arm64@2.3.8': optional: true - '@biomejs/cli-linux-arm64@2.4.15': + '@biomejs/cli-linux-arm64@2.4.16': optional: true '@biomejs/cli-linux-x64-musl@2.3.8': optional: true - '@biomejs/cli-linux-x64-musl@2.4.15': + '@biomejs/cli-linux-x64-musl@2.4.16': optional: true '@biomejs/cli-linux-x64@2.3.8': optional: true - '@biomejs/cli-linux-x64@2.4.15': + '@biomejs/cli-linux-x64@2.4.16': optional: true '@biomejs/cli-win32-arm64@2.3.8': optional: true - '@biomejs/cli-win32-arm64@2.4.15': + '@biomejs/cli-win32-arm64@2.4.16': optional: true '@biomejs/cli-win32-x64@2.3.8': optional: true - '@biomejs/cli-win32-x64@2.4.15': + '@biomejs/cli-win32-x64@2.4.16': optional: true '@colors/colors@1.6.0': {} @@ -7370,6 +7376,8 @@ snapshots: indent-string@4.0.0: {} + indonesian-badwords@1.0.1: {} + inflight@1.0.6: dependencies: once: 1.4.0 diff --git a/src/moderation/conversationContext.ts b/src/moderation/conversationContext.ts index 5ed955c..1256dd1 100644 --- a/src/moderation/conversationContext.ts +++ b/src/moderation/conversationContext.ts @@ -1,3 +1,4 @@ +import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js"; import { formatMediaEvidenceForPrompt } from "./messageMetadata.js"; import type { MessageRecord } from "./types.js"; @@ -30,9 +31,11 @@ export function formatMessageForPrompt( ): string { const content = msg.edited_content ?? msg.content; const timestamp = formatTimestamp(msg.created_at); + const textEvidence = formatModerationTextEvidenceForPrompt(content); + const textSuffix = textEvidence ? ` ${textEvidence}` : ""; const mediaEvidence = formatMediaEvidenceForPrompt(msg.metadata); const mediaSuffix = mediaEvidence ? ` ${mediaEvidence}` : ""; - return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}${mediaSuffix}`; + return `[${label}] id=${msg.id} time=${timestamp} user=${msg.username}: ${content}${textSuffix}${mediaSuffix}`; } /** diff --git a/src/moderation/indonesianTextNormalizer.ts b/src/moderation/indonesianTextNormalizer.ts new file mode 100644 index 0000000..ede35ca --- /dev/null +++ b/src/moderation/indonesianTextNormalizer.ts @@ -0,0 +1,107 @@ +import badwordsModule from "indonesian-badwords"; +import { INDONESIAN_SLANG_LEXICON } from "./resources/indonesianSlangLexicon.js"; + +const CUSTOM_EMOJI_PATTERN = //g; +const WORD_PATTERN = /[\p{L}\p{N}_]+/gu; + +interface BadwordAnalyzeResult { + badwords?: string[]; + count?: number; +} + +interface BadwordsModule { + analyze?: (text: string) => BadwordAnalyzeResult; + flag?: (text: string) => boolean; +} + +const badwords = badwordsModule as BadwordsModule; + +export interface ModerationTextEvidence { + raw: string; + normalized: string; + notes: string[]; + badwords: string[]; + hasBadwords: boolean; +} + +export function normalizeDiscordCustomEmoji(text: string): { + text: string; + emojiNames: string[]; +} { + const emojiNames: string[] = []; + const normalized = text.replace(CUSTOM_EMOJI_PATTERN, (_match, name: string) => { + emojiNames.push(name); + return `[emoji:${name}]`; + }); + + return { text: normalized, emojiNames }; +} + +export function normalizeIndonesianSlang(text: string): { + text: string; + notes: string[]; +} { + const notes: string[] = []; + const normalized = text.replace(WORD_PATTERN, (word) => { + const entry = INDONESIAN_SLANG_LEXICON[word.toLowerCase()]; + if (!entry) return word; + + notes.push(`${word}=${entry.normalized} (${entry.note})`); + return entry.normalized; + }); + + return { text: normalized, notes: Array.from(new Set(notes)) }; +} + +export function detectIndonesianBadwords(text: string): string[] { + try { + const result = badwords.analyze?.(text); + if (Array.isArray(result?.badwords)) { + return Array.from(new Set(result.badwords.map((word) => word.toLowerCase()))); + } + } catch { + // Keep moderation pipeline resilient if dependency changes shape. + } + return []; +} + +export function buildModerationTextEvidence(text: string): ModerationTextEvidence { + const emojiNormalized = normalizeDiscordCustomEmoji(text); + const slangNormalized = normalizeIndonesianSlang(emojiNormalized.text); + const badwordHits = detectIndonesianBadwords(slangNormalized.text); + const notes = [...slangNormalized.notes]; + + for (const emojiName of emojiNormalized.emojiNames) { + notes.push( + `emoji:${emojiName}=Discord custom emoji/expression; not text offense by default`, + ); + } + + if (badwordHits.length > 0) { + notes.push(`local lexical check: Indonesian badword detected: ${badwordHits.join(", ")}`); + } else { + notes.push("local lexical check: no Indonesian badword detected"); + } + + return { + raw: text, + normalized: slangNormalized.text, + notes: Array.from(new Set(notes)), + badwords: badwordHits, + hasBadwords: badwordHits.length > 0, + }; +} + +export function formatModerationTextEvidenceForPrompt(text: string): string { + const evidence = buildModerationTextEvidence(text); + if (evidence.normalized === evidence.raw && evidence.notes.length === 0) { + return ""; + } + + return [ + `[normalized_text: ${evidence.normalized}]`, + evidence.notes.length > 0 ? `[normalization_notes: ${evidence.notes.join("; ")}]` : null, + ] + .filter(Boolean) + .join(" "); +} diff --git a/src/moderation/llmModerationClient.ts b/src/moderation/llmModerationClient.ts index 1ae81a6..15adbda 100644 --- a/src/moderation/llmModerationClient.ts +++ b/src/moderation/llmModerationClient.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { config } from "../config.js"; import { createChildLogger } from "../logger.js"; import { retryWithBackoff } from "../retry.js"; +import { formatModerationTextEvidenceForPrompt } from "./indonesianTextNormalizer.js"; import { extractMessageMediaEvidence } from "./messageMetadata.js"; import type { AnalysisResult, @@ -663,10 +664,13 @@ Bahasa utama komunitas ini adalah BAHASA INDONESIA. Bahasa Inggris adalah bahasa ## Konteks Server Ini adalah server Discord komunitas Indonesia. Kamu harus memahami: -- Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", dll. +- Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll. - Singkatan umum: "gw", "lo", "emg", "kyk", "tdk", "krn", "jgn", dll. - Konteks budaya lokal: SARA (Suku, Agama, Ras, Antar-golongan), hoaks, ujaran kebencian berbasis konteks Indonesia. - Perbedaan antara humor/banter biasa vs konten yang benar-benar melanggar. +- "woy"/"woi" adalah sapaan/interjeksi informal Indonesia dan tidak boleh dianggap SARA, hate speech, atau harassment tanpa target hinaan/ancaman jelas. +- Discord custom emoji seperti <:hadeh:123> atau [emoji:hadeh] adalah ekspresi/emoji, bukan pelanggaran teks. Gunakan sebagai konteks ekspresi saja. +- Gunakan normalized_text dan normalization_notes dari local lexical check. Jika notes menyatakan no Indonesian badword detected dan hanya ada slang/emoji aman, jangan flag karena kata slang itu saja. - Kalimat ambigu dalam bahasa Indonesia harus diberi keputusan final: "clean" bila bukti pelanggaran tidak jelas, "flagged" bila bukti pelanggaran jelas. - Jangan pernah menulis analisis yang meminta admin/moderator memeriksa ulang, menyebut kurang konteks, atau tidak bisa menentukan. Berikan kesimpulan langsung berdasarkan teks + media + konteks yang tersedia. - Gambar, sticker, embed, dan preview link adalah evidence utama yang setara dengan teks, bukan sekadar URL teks. @@ -733,6 +737,8 @@ CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan const webTexts = messageWebTextMap.get(msg.id) ?? []; const mediaAnalyses = messageMediaAnalysisMap.get(msg.id) ?? []; const webContext = webTexts.length > 0 ? `\n${webTexts.join("\n")}` : ""; + const textEvidence = formatModerationTextEvidenceForPrompt(content); + const textContext = textEvidence ? `\n${textEvidence}` : ""; const mediaAnalysisContext = mediaAnalyses.length > 0 ? `\n${mediaAnalyses.join("\n")}` : ""; const mediaEvidence = extractMessageMediaEvidence(msg.metadata); @@ -748,7 +754,7 @@ CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan ] .filter(Boolean) .join(" "); - return `[target] id=${msg.id} user=${msg.username}: ${content}${mediaContext ? ` ${mediaContext}` : ""}${webContext}${mediaAnalysisContext}`; + return `[target] id=${msg.id} user=${msg.username}: ${content}${mediaContext ? ` ${mediaContext}` : ""}${textContext}${webContext}${mediaAnalysisContext}`; }) .join("\n"); diff --git a/src/moderation/resources/indonesianSlangLexicon.ts b/src/moderation/resources/indonesianSlangLexicon.ts new file mode 100644 index 0000000..aa9f3ba --- /dev/null +++ b/src/moderation/resources/indonesianSlangLexicon.ts @@ -0,0 +1,45 @@ +export interface SlangLexiconEntry { + normalized: string; + note: string; + safeByDefault?: boolean; +} + +export const INDONESIAN_SLANG_LEXICON: Record = { + gw: { normalized: "gue", note: "first-person informal pronoun" }, + gue: { normalized: "gue", note: "first-person informal pronoun" }, + gua: { normalized: "gue", note: "first-person informal pronoun" }, + lo: { normalized: "lu", note: "second-person informal pronoun" }, + lu: { normalized: "lu", note: "second-person informal pronoun" }, + loe: { normalized: "lu", note: "second-person informal pronoun" }, + yg: { normalized: "yang", note: "common abbreviation" }, + emg: { normalized: "memang", note: "common abbreviation" }, + kyk: { normalized: "kayak", note: "common abbreviation" }, + tdk: { normalized: "tidak", note: "common abbreviation" }, + krn: { normalized: "karena", note: "common abbreviation" }, + jgn: { normalized: "jangan", note: "common abbreviation" }, + woy: { + normalized: "woy", + note: "casual Indonesian interjection/greeting; not SARA/hate/harassment by default", + safeByDefault: true, + }, + woi: { + normalized: "woi", + note: "casual Indonesian interjection/greeting; not SARA/hate/harassment by default", + safeByDefault: true, + }, + oi: { + normalized: "oi", + note: "casual call/interjection; not offensive by default", + safeByDefault: true, + }, + hadeh: { + normalized: "hadeh", + note: "facepalm/tired expression; not offensive by default", + safeByDefault: true, + }, + hadeuh: { + normalized: "hadeh", + note: "facepalm/tired expression; not offensive by default", + safeByDefault: true, + }, +}; diff --git a/src/types/indonesian-badwords.d.ts b/src/types/indonesian-badwords.d.ts new file mode 100644 index 0000000..61a5ca6 --- /dev/null +++ b/src/types/indonesian-badwords.d.ts @@ -0,0 +1,25 @@ +declare module "indonesian-badwords" { + export interface BadwordAnalyzeResult { + text?: string; + words?: number; + censored?: string; + badwords?: string[]; + count?: number; + locations?: Array<{ word: string; index: number }>; + } + + export function analyze(text: string): BadwordAnalyzeResult; + export function flag(text: string): boolean; + export function filter(text: string): string; + export function censor(text: string): string; + + const value: { + analyze: typeof analyze; + flag: typeof flag; + filter: typeof filter; + censor: typeof censor; + dict?: unknown; + badwords?: unknown; + }; + export default value; +} diff --git a/tests/moderation/indonesianTextNormalizer.test.ts b/tests/moderation/indonesianTextNormalizer.test.ts new file mode 100644 index 0000000..065ee50 --- /dev/null +++ b/tests/moderation/indonesianTextNormalizer.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { + buildModerationTextEvidence, + detectIndonesianBadwords, + formatModerationTextEvidenceForPrompt, + normalizeDiscordCustomEmoji, + normalizeIndonesianSlang, +} from "../../src/moderation/indonesianTextNormalizer"; + +describe("normalizeDiscordCustomEmoji", () => { + it("replaces static custom emoji", () => { + const result = normalizeDiscordCustomEmoji("Bersiaplah woy <:hadeh:1217434294281048185>"); + expect(result.text).toBe("Bersiaplah woy [emoji:hadeh]"); + expect(result.emojiNames).toContain("hadeh"); + }); + + it("replaces animated custom emoji", () => { + const result = normalizeDiscordCustomEmoji("cek dulu"); + expect(result.text).toBe("cek [emoji:speen] dulu"); + expect(result.emojiNames).toContain("speen"); + }); + + it("handles text without emoji", () => { + const result = normalizeDiscordCustomEmoji("halo semua"); + expect(result.text).toBe("halo semua"); + expect(result.emojiNames).toHaveLength(0); + }); +}); + +describe("normalizeIndonesianSlang", () => { + it("maps common slang to normalized form", () => { + const { text, notes } = normalizeIndonesianSlang("gw emg kyk gitu"); + expect(text).toBe("gue memang kayak gitu"); + expect(notes.some((n) => n.startsWith("gw="))).toBe(true); + }); + + it("marks woy as safe casual interjection", () => { + const { text, notes } = normalizeIndonesianSlang("woy hadeh"); + expect(text).toBe("woy hadeh"); + expect(notes.some((n) => n.includes("casual"))).toBe(true); + }); +}); + +describe("detectIndonesianBadwords", () => { + it("detects known badword", () => { + const badwords = detectIndonesianBadwords("kontol banget"); + expect(badwords).toContain("kontol"); + }); + + it("returns empty array for safe slang", () => { + const badwords = detectIndonesianBadwords("woy hadeh gua"); + expect(badwords).toHaveLength(0); + }); +}); + +describe("buildModerationTextEvidence", () => { + it("produces correct evidence for woy with emoji", () => { + const evidence = buildModerationTextEvidence( + "Bersiaplah woy <:hadeh:1217434294281048185>", + ); + expect(evidence.normalized).toContain("[emoji:hadeh]"); + expect(evidence.badwords).toHaveLength(0); + expect(evidence.hasBadwords).toBe(false); + expect(evidence.notes.some((n) => n.includes("no Indonesian badword"))).toBe(true); + expect(evidence.notes.some((n) => n.includes("emoji:hadeh"))).toBe(true); + expect(evidence.notes.some((n) => n.includes("casual"))).toBe(true); + }); + + it("detects badword when present", () => { + const evidence = buildModerationTextEvidence("anjing loe kontol"); + expect(evidence.hasBadwords).toBe(true); + expect(evidence.notes.some((n) => n.includes("badword detected"))).toBe(true); + }); +}); + +describe("formatModerationTextEvidenceForPrompt", () => { + it("returns prompt evidence for slang + emoji", () => { + const formatted = formatModerationTextEvidenceForPrompt( + "Bersiaplah woy <:hadeh:1217434294281048185>", + ); + expect(formatted).toContain("[normalized_text:"); + expect(formatted).toContain("[emoji:hadeh]"); + expect(formatted).toContain("[normalization_notes:"); + expect(formatted).toContain("no Indonesian badword detected"); + }); + + it("includes normalized text even for clean input", () => { + const formatted = formatModerationTextEvidenceForPrompt("Halo semua"); + expect(formatted).toContain("[normalized_text:"); + expect(formatted).toContain("no Indonesian badword detected"); + }); +});