fix(moderation): remove manual badword list and fix deferral regex false positives

- Remove LOCAL_BADWORDS array (25 hardcoded words) and FALSE_POSITIVE_WHITELISTS
- Remove detectLocalBadwords function — all detection now goes through API pipeline
- Fix DEFERRAL_ANALYSIS_PATTERN: remove overly broad patterns (admin perlu, bisa berpotensi, maaf/sorry, saya tidak yakin)
- Expand DEFERRAL_EXCEPTION_PATTERN to catch more decisive-deferral variations
- Update tests to reflect API-only detection (local fallback removed)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/moderation/indonesianTextNormalizer.ts

@@ -118,84 +118,9 @@ export function normalizeDiscordCustomEmoji(text: string): {
   return { text: normalized, emojiNames };
 }
 
-// ---------------------------------------------------------------------------
+// Local badword detection removed (lines 121-198).
-// Local fallback badword list (used when NVIDIA API is unavailable)
+// All detection now goes through the API pipeline (NVIDIA → Primary AI → Groq)
-// ---------------------------------------------------------------------------
+// to eliminate false positives from substring matching and hardcoded whitelists.
-
-const LOCAL_BADWORDS = [
-  "anjing",
-  "bangsat",
-  "brengsek",
-  "bajingan",
-  "kontol",
-  "memek",
-  "tai",
-  "goblok",
-  "tolol",
-  "bego",
-  "sialan",
-  "jancuk",
-  "kampret",
-  "pepek",
-  "jembut",
-  "ngentot",
-  "ngewe",
-  "coli",
-  "celaka",
-  "laknat",
-  "pantek",
-  "entod",
-  "ndasmu",
-  "ndas",
-  "piyo",
-  "asu",
-];
-
-const FALSE_POSITIVE_WHITELISTS: Record<string, string[]> = {
-  asu: [
-    "asus",
-    "masuk",
-    "termasuk",
-    "dimasukkan",
-    "memasukkan",
-    "kasur",
-    "asumsi",
-    "asuransi",
-    "asupan",
-    "pasukan",
-    "pasundan",
-  ],
-  goblok: ["goblok"],
-  kontol: ["kontol"],
-  memek: ["memek"],
-  tolol: ["tolol"],
-};
-
-function detectLocalBadwords(text: string): string[] {
-  const lowerText = text.toLowerCase();
-  const words = lowerText.match(/[\p{L}\p{N}_]+/gu) || [];
-
-  const isRealHit = (hit: string, whitelist: string[]): boolean => {
-    for (const w of words) {
-      if (w.includes(hit)) {
-        if (w === hit) return true;
-        if (!whitelist.includes(w)) return true;
-      }
-    }
-    return false;
-  };
-
-  const hits: string[] = [];
-
-  for (const badword of LOCAL_BADWORDS) {
-    const whitelist = FALSE_POSITIVE_WHITELISTS[badword] ?? [badword];
-    if (isRealHit(badword, whitelist)) {
-      hits.push(badword);
-    }
-  }
-
-  return Array.from(new Set(hits));
-}
 
 function normalizeBadwordCacheKey(text: string): string {
   return text.trim().replace(/\s+/g, " ").toLowerCase();
@@ -496,15 +421,18 @@ async function callNemotronContentSafety(text: string): Promise<string[]> {
 // ---------------------------------------------------------------------------
 
 /**
- * Detect badwords in text using a **three-tier cache strategy**:
+ * Detect badwords in text using a **two-tier cache + API pipeline**:
  *
  * 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path,
  *    keyed by the full normalized text string.
  * 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted
  *    across restarts. Uses the FULL normalized text (not per-word) because
  *    context matters: "kau" alone is clean, but "awas kau" can be a threat.
- * 3. **API/fallback pipeline** (NVIDIA → Primary AI → Groq → local lexical)
+ * 3. **API pipeline** (NVIDIA → Primary AI → Groq)
  *    only runs when both cache layers miss.
+ *
+ * No local hardcoded badword list — all detection goes through AI APIs
+ * to eliminate false positives from substring matching.
  */
 export async function detectIndonesianBadwords(
   text: string,
@@ -532,25 +460,12 @@ export async function detectIndonesianBadwords(
       return flags;
     }
 
-    // ── Tier 3: API / fallback pipeline ──
+    // ── Tier 3: API pipeline ──
-
-    // 3a. Local lexical check (instant, no network)
-    const localHits = detectLocalBadwords(text);
-    if (localHits.length > 0) {
-      setCachedBadwords(cacheKey, localHits);
-      await upsertCachedText(
-        cacheKey,
-        localHits,
-        "local",
-        Date.now() + DB_CACHE_TTL_MS,
-      );
-      return localHits;
-    }
 
     const hits = new Set<string>();
-    let sourceUsed: "local" | "nvidia" | "primary_ai" | "groq" = "local";
+    let sourceUsed: "nvidia" | "primary_ai" | "groq" = "primary_ai";
 
-    // 3b. Try NVIDIA API if key is configured and not rate limited.
+    // 3a. Try NVIDIA API if key is configured and not rate limited.
     const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
     if (apiKey && Date.now() >= nemotronUnavailableUntil) {
       try {
@@ -569,12 +484,12 @@ export async function detectIndonesianBadwords(
         }
         log.warn(
           { error },
-          "NVIDIA Nemotron API call failed, falling back to primary AI then local detection",
+          "NVIDIA Nemotron API call failed, falling back to primary AI",
         );
       }
     }
 
-    // 3c. Try the main AI model next.
+    // 3b. Try the main AI model next.
     if (hits.size === 0 && Date.now() >= primaryAiUnavailableUntil) {
       try {
         const primaryHits = await callPrimaryAiModeration(text);
@@ -592,12 +507,12 @@ export async function detectIndonesianBadwords(
         }
         log.warn(
           { error },
-          "Primary AI badword detection failed, falling back to Groq then local detection",
+          "Primary AI badword detection failed, falling back to Groq",
         );
       }
     }
 
-    // 3d. Try Groq Llama Prompt Guard as final API fallback.
+    // 3c. Try Groq Llama Prompt Guard as final API fallback.
     if (hits.size === 0 && Date.now() >= groqUnavailableUntil) {
       const groqKey = config.GROQ_API_KEY;
       if (groqKey) {
@@ -616,7 +531,7 @@ export async function detectIndonesianBadwords(
           }
           log.warn(
             { error },
-            "Groq Llama Prompt Guard moderation failed, falling back to local detection",
+            "Groq Llama Prompt Guard moderation failed",
           );
         }
       }

src/moderation/llmModerationClient.ts

@@ -66,17 +66,24 @@ const log = createChildLogger("llmModerationClient");
 
 /**
  * Enhanced deferral detection pattern (R9).
- * Covers more variations and multi-word combinations that the LLM might use
+ *
- * to defer judgment instead of making a decision.
+ * Only matches patterns where the model explicitly states it cannot make
+ * a decision and needs human review. Removed overly broad patterns that
+ * caused false positives:
+ * - "admin (perlu|harus|sebaiknya)" → common in regular sentences
+ * - "bisa (berpotensi|mengandung)" → decisive statements, not deferral
+ * - "maaf|sorry" → opinions/apologies, not deferral
+ * - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral
  */
 const DEFERRAL_ANALYSIS_PATTERN =
-  /(?:kurang (?:konteks|bukti|informasi|data)|kekurangan (?:konteks|bukti)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi).*(?:admin|moderator|manusia|human)|admin (?:perlu|harus|sebaiknya)|moderator (?:perlu|harus|sebaiknya)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|mengevaluasi)|cannot determine|insufficient (?:context|evidence|information)|(?:mungkin|sepertinya|tampaknya) (?:perlu|harus|sebaiknya) (?:dicek|diperiksa|ditinjau)|tidak (?:cukup|memadai) (?:bukti|informasi|konteks)|bisa (?:berpotensi|mengandung)|(?:(?:maaf|sorry|抱歉|ขออภัย))[,.\s]|(?:saya (?:tidak|kurang|belum) (?:yakin|pasti|tahu|paham)))/i;
+  /(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i;
 
 /**
  * Exceptions: patterns that look like deferral but are actually decisive.
+ * Expanded to catch more variations where the model gives a clear verdict.
  */
 const DEFERRAL_EXCEPTION_PATTERN =
-  /tidak bisa menentukan.*(?:karena|sebab|dengan alasan).*(?:clean|tidak (?:ada|terdapat).*(?:pelanggaran|masalah)|aman)/i;
+  /tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i;
 
 function hasDeferralAnalysis(analysis: string): boolean {
   if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false;

tests/moderation/indonesianTextNormalizer.test.ts

@@ -52,9 +52,12 @@ describe("normalizeDiscordCustomEmoji", () => {
 });
 
 describe("detectIndonesianBadwords", () => {
-  it("detects known badword via local fallback", async () => {
+  it("returns empty array when no APIs are configured", async () => {
+    // Local badword list removed; all detection now requires an API.
+    // With all APIs disabled (see disableRemoteModeration above),
+    // the function should return empty without throwing.
     const badwords = await detectIndonesianBadwords("kontol banget");
-    expect(badwords).toContain("kontol");
+    expect(badwords).toHaveLength(0);
   });
 
   it("returns empty array for safe slang", async () => {
@@ -75,7 +78,10 @@ describe("buildModerationTextEvidence", () => {
 
   it("detects badword when present", async () => {
     const evidence = await buildModerationTextEvidence("anjing loe kontol");
-    expect(evidence.hasBadwords).toBe(true);
+    // With all APIs disabled, local detection is removed so hasBadwords will be false.
+    // This test now verifies that the evidence builder does not crash and always
+    // returns a valid structure.
+    expect(evidence.normalized).toBeDefined();
     expect(evidence.notes.some((n) => n.includes("badword detected"))).toBe(
       true,
     );