fix(moderation): remove manual badword list and fix deferral regex false positives
- Remove LOCAL_BADWORDS array (25 hardcoded words) and FALSE_POSITIVE_WHITELISTS - Remove detectLocalBadwords function — all detection now goes through API pipeline - Fix DEFERRAL_ANALYSIS_PATTERN: remove overly broad patterns (admin perlu, bisa berpotensi, maaf/sorry, saya tidak yakin) - Expand DEFERRAL_EXCEPTION_PATTERN to catch more decisive-deferral variations - Update tests to reflect API-only detection (local fallback removed) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
a643125c7b
commit
13efb576d6
@@ -118,84 +118,9 @@ export function normalizeDiscordCustomEmoji(text: string): {
|
||||
return { text: normalized, emojiNames };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Local fallback badword list (used when NVIDIA API is unavailable)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const LOCAL_BADWORDS = [
|
||||
"anjing",
|
||||
"bangsat",
|
||||
"brengsek",
|
||||
"bajingan",
|
||||
"kontol",
|
||||
"memek",
|
||||
"tai",
|
||||
"goblok",
|
||||
"tolol",
|
||||
"bego",
|
||||
"sialan",
|
||||
"jancuk",
|
||||
"kampret",
|
||||
"pepek",
|
||||
"jembut",
|
||||
"ngentot",
|
||||
"ngewe",
|
||||
"coli",
|
||||
"celaka",
|
||||
"laknat",
|
||||
"pantek",
|
||||
"entod",
|
||||
"ndasmu",
|
||||
"ndas",
|
||||
"piyo",
|
||||
"asu",
|
||||
];
|
||||
|
||||
const FALSE_POSITIVE_WHITELISTS: Record<string, string[]> = {
|
||||
asu: [
|
||||
"asus",
|
||||
"masuk",
|
||||
"termasuk",
|
||||
"dimasukkan",
|
||||
"memasukkan",
|
||||
"kasur",
|
||||
"asumsi",
|
||||
"asuransi",
|
||||
"asupan",
|
||||
"pasukan",
|
||||
"pasundan",
|
||||
],
|
||||
goblok: ["goblok"],
|
||||
kontol: ["kontol"],
|
||||
memek: ["memek"],
|
||||
tolol: ["tolol"],
|
||||
};
|
||||
|
||||
function detectLocalBadwords(text: string): string[] {
|
||||
const lowerText = text.toLowerCase();
|
||||
const words = lowerText.match(/[\p{L}\p{N}_]+/gu) || [];
|
||||
|
||||
const isRealHit = (hit: string, whitelist: string[]): boolean => {
|
||||
for (const w of words) {
|
||||
if (w.includes(hit)) {
|
||||
if (w === hit) return true;
|
||||
if (!whitelist.includes(w)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const hits: string[] = [];
|
||||
|
||||
for (const badword of LOCAL_BADWORDS) {
|
||||
const whitelist = FALSE_POSITIVE_WHITELISTS[badword] ?? [badword];
|
||||
if (isRealHit(badword, whitelist)) {
|
||||
hits.push(badword);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(new Set(hits));
|
||||
}
|
||||
// Local badword detection removed (lines 121-198).
|
||||
// All detection now goes through the API pipeline (NVIDIA → Primary AI → Groq)
|
||||
// to eliminate false positives from substring matching and hardcoded whitelists.
|
||||
|
||||
function normalizeBadwordCacheKey(text: string): string {
|
||||
return text.trim().replace(/\s+/g, " ").toLowerCase();
|
||||
@@ -496,15 +421,18 @@ async function callNemotronContentSafety(text: string): Promise<string[]> {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Detect badwords in text using a **three-tier cache strategy**:
|
||||
* Detect badwords in text using a **two-tier cache + API pipeline**:
|
||||
*
|
||||
* 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path,
|
||||
* keyed by the full normalized text string.
|
||||
* 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted
|
||||
* across restarts. Uses the FULL normalized text (not per-word) because
|
||||
* context matters: "kau" alone is clean, but "awas kau" can be a threat.
|
||||
* 3. **API/fallback pipeline** (NVIDIA → Primary AI → Groq → local lexical)
|
||||
* 3. **API pipeline** (NVIDIA → Primary AI → Groq)
|
||||
* only runs when both cache layers miss.
|
||||
*
|
||||
* No local hardcoded badword list — all detection goes through AI APIs
|
||||
* to eliminate false positives from substring matching.
|
||||
*/
|
||||
export async function detectIndonesianBadwords(
|
||||
text: string,
|
||||
@@ -532,25 +460,12 @@ export async function detectIndonesianBadwords(
|
||||
return flags;
|
||||
}
|
||||
|
||||
// ── Tier 3: API / fallback pipeline ──
|
||||
|
||||
// 3a. Local lexical check (instant, no network)
|
||||
const localHits = detectLocalBadwords(text);
|
||||
if (localHits.length > 0) {
|
||||
setCachedBadwords(cacheKey, localHits);
|
||||
await upsertCachedText(
|
||||
cacheKey,
|
||||
localHits,
|
||||
"local",
|
||||
Date.now() + DB_CACHE_TTL_MS,
|
||||
);
|
||||
return localHits;
|
||||
}
|
||||
// ── Tier 3: API pipeline ──
|
||||
|
||||
const hits = new Set<string>();
|
||||
let sourceUsed: "local" | "nvidia" | "primary_ai" | "groq" = "local";
|
||||
let sourceUsed: "nvidia" | "primary_ai" | "groq" = "primary_ai";
|
||||
|
||||
// 3b. Try NVIDIA API if key is configured and not rate limited.
|
||||
// 3a. Try NVIDIA API if key is configured and not rate limited.
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (apiKey && Date.now() >= nemotronUnavailableUntil) {
|
||||
try {
|
||||
@@ -569,12 +484,12 @@ export async function detectIndonesianBadwords(
|
||||
}
|
||||
log.warn(
|
||||
{ error },
|
||||
"NVIDIA Nemotron API call failed, falling back to primary AI then local detection",
|
||||
"NVIDIA Nemotron API call failed, falling back to primary AI",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3c. Try the main AI model next.
|
||||
// 3b. Try the main AI model next.
|
||||
if (hits.size === 0 && Date.now() >= primaryAiUnavailableUntil) {
|
||||
try {
|
||||
const primaryHits = await callPrimaryAiModeration(text);
|
||||
@@ -592,12 +507,12 @@ export async function detectIndonesianBadwords(
|
||||
}
|
||||
log.warn(
|
||||
{ error },
|
||||
"Primary AI badword detection failed, falling back to Groq then local detection",
|
||||
"Primary AI badword detection failed, falling back to Groq",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3d. Try Groq Llama Prompt Guard as final API fallback.
|
||||
// 3c. Try Groq Llama Prompt Guard as final API fallback.
|
||||
if (hits.size === 0 && Date.now() >= groqUnavailableUntil) {
|
||||
const groqKey = config.GROQ_API_KEY;
|
||||
if (groqKey) {
|
||||
@@ -616,7 +531,7 @@ export async function detectIndonesianBadwords(
|
||||
}
|
||||
log.warn(
|
||||
{ error },
|
||||
"Groq Llama Prompt Guard moderation failed, falling back to local detection",
|
||||
"Groq Llama Prompt Guard moderation failed",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,17 +66,24 @@ const log = createChildLogger("llmModerationClient");
|
||||
|
||||
/**
|
||||
* Enhanced deferral detection pattern (R9).
|
||||
* Covers more variations and multi-word combinations that the LLM might use
|
||||
* to defer judgment instead of making a decision.
|
||||
*
|
||||
* Only matches patterns where the model explicitly states it cannot make
|
||||
* a decision and needs human review. Removed overly broad patterns that
|
||||
* caused false positives:
|
||||
* - "admin (perlu|harus|sebaiknya)" → common in regular sentences
|
||||
* - "bisa (berpotensi|mengandung)" → decisive statements, not deferral
|
||||
* - "maaf|sorry" → opinions/apologies, not deferral
|
||||
* - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral
|
||||
*/
|
||||
const DEFERRAL_ANALYSIS_PATTERN =
|
||||
/(?:kurang (?:konteks|bukti|informasi|data)|kekurangan (?:konteks|bukti)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi).*(?:admin|moderator|manusia|human)|admin (?:perlu|harus|sebaiknya)|moderator (?:perlu|harus|sebaiknya)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|mengevaluasi)|cannot determine|insufficient (?:context|evidence|information)|(?:mungkin|sepertinya|tampaknya) (?:perlu|harus|sebaiknya) (?:dicek|diperiksa|ditinjau)|tidak (?:cukup|memadai) (?:bukti|informasi|konteks)|bisa (?:berpotensi|mengandung)|(?:(?:maaf|sorry|抱歉|ขออภัย))[,.\s]|(?:saya (?:tidak|kurang|belum) (?:yakin|pasti|tahu|paham)))/i;
|
||||
/(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i;
|
||||
|
||||
/**
|
||||
* Exceptions: patterns that look like deferral but are actually decisive.
|
||||
* Expanded to catch more variations where the model gives a clear verdict.
|
||||
*/
|
||||
const DEFERRAL_EXCEPTION_PATTERN =
|
||||
/tidak bisa menentukan.*(?:karena|sebab|dengan alasan).*(?:clean|tidak (?:ada|terdapat).*(?:pelanggaran|masalah)|aman)/i;
|
||||
/tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i;
|
||||
|
||||
function hasDeferralAnalysis(analysis: string): boolean {
|
||||
if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false;
|
||||
|
||||
@@ -52,9 +52,12 @@ describe("normalizeDiscordCustomEmoji", () => {
|
||||
});
|
||||
|
||||
describe("detectIndonesianBadwords", () => {
|
||||
it("detects known badword via local fallback", async () => {
|
||||
it("returns empty array when no APIs are configured", async () => {
|
||||
// Local badword list removed; all detection now requires an API.
|
||||
// With all APIs disabled (see disableRemoteModeration above),
|
||||
// the function should return empty without throwing.
|
||||
const badwords = await detectIndonesianBadwords("kontol banget");
|
||||
expect(badwords).toContain("kontol");
|
||||
expect(badwords).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("returns empty array for safe slang", async () => {
|
||||
@@ -75,7 +78,10 @@ describe("buildModerationTextEvidence", () => {
|
||||
|
||||
it("detects badword when present", async () => {
|
||||
const evidence = await buildModerationTextEvidence("anjing loe kontol");
|
||||
expect(evidence.hasBadwords).toBe(true);
|
||||
// With all APIs disabled, local detection is removed so hasBadwords will be false.
|
||||
// This test now verifies that the evidence builder does not crash and always
|
||||
// returns a valid structure.
|
||||
expect(evidence.normalized).toBeDefined();
|
||||
expect(evidence.notes.some((n) => n.includes("badword detected"))).toBe(
|
||||
true,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user