feat: integrate NVIDIA Nemotron-3 Content Safety API for Indonesian badword detection
- Added configuration options for NVIDIA Nemotron API key, model, and base URL. - Refactored badword detection to utilize NVIDIA API, with a fallback to a local badword list. - Updated moderation functions to handle asynchronous operations for text evidence generation. - Removed dependency on the `indonesian-badwords` package and implemented custom detection logic. - Enhanced tests to accommodate asynchronous behavior and validate new detection methods.
This commit is contained in:
MythEclipse
@@ -1,20 +1,40 @@
|
||||
import badwordsModule from "indonesian-badwords";
|
||||
import axios from "axios";
|
||||
import { config } from "../config.js";
|
||||
import { INDONESIAN_SLANG_LEXICON } from "./resources/indonesianSlangLexicon.js";
|
||||
import { createChildLogger } from "../logger.js";
|
||||
|
||||
const log = createChildLogger("indonesianTextNormalizer");
|
||||
|
||||
const CUSTOM_EMOJI_PATTERN = /<a?:([a-zA-Z0-9_]+):(\d+)>/g;
|
||||
const WORD_PATTERN = /[\p{L}\p{N}_]+/gu;
|
||||
|
||||
interface BadwordAnalyzeResult {
|
||||
badwords?: string[];
|
||||
count?: number;
|
||||
}
|
||||
/** NVIDIA content safety categories that map to offensive/badword content. */
|
||||
const NVIDIA_BAD_CATEGORIES = new Set([
|
||||
"hate",
|
||||
"harassment",
|
||||
"sexual",
|
||||
"violence",
|
||||
"self-harm",
|
||||
"illicit",
|
||||
"profanity",
|
||||
"vulgar",
|
||||
"insult",
|
||||
]);
|
||||
|
||||
interface BadwordsModule {
|
||||
analyze?: (text: string) => BadwordAnalyzeResult;
|
||||
flag?: (text: string) => boolean;
|
||||
}
|
||||
|
||||
const badwords = badwordsModule as BadwordsModule;
|
||||
/**
|
||||
* Map NVIDIA Nemotron category labels to Indonesian badword-style labels.
|
||||
*/
|
||||
const CATEGORY_TO_BADWORD_LABEL: Record<string, string> = {
|
||||
hate: "hate_speech",
|
||||
harassment: "harassment",
|
||||
sexual: "sexual_content",
|
||||
violence: "violence",
|
||||
"self-harm": "self_harm",
|
||||
illicit: "illegal_content",
|
||||
profanity: "vulgar_language",
|
||||
vulgar: "vulgar_language",
|
||||
insult: "harassment",
|
||||
};
|
||||
|
||||
export interface ModerationTextEvidence {
|
||||
raw: string;
|
||||
@@ -24,6 +44,10 @@ export interface ModerationTextEvidence {
|
||||
hasBadwords: boolean;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sync helpers (unchanged)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function normalizeDiscordCustomEmoji(text: string): {
|
||||
text: string;
|
||||
emojiNames: string[];
|
||||
@@ -53,104 +77,155 @@ export function normalizeIndonesianSlang(text: string): {
|
||||
return { text: normalized, notes: Array.from(new Set(notes)) };
|
||||
}
|
||||
|
||||
export function detectIndonesianBadwords(text: string): string[] {
|
||||
try {
|
||||
const result = badwords.analyze?.(text);
|
||||
if (Array.isArray(result?.badwords)) {
|
||||
let hits = Array.from(new Set(result.badwords.map((word) => word.toLowerCase())));
|
||||
// ---------------------------------------------------------------------------
|
||||
// Local fallback badword list (used when NVIDIA API is unavailable)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const lowerText = text.toLowerCase();
|
||||
const LOCAL_BADWORDS = [
|
||||
"anjing", "bangsat", "brengsek", "bajingan", "kontol", "memek",
|
||||
"tai", "goblok", "tolol", "bego", "sialan", "jancuk", "kampret",
|
||||
"pepek", "jembut", "ngentot", "ngewe", "coli", "celaka", "laknat",
|
||||
"pantek", "entod", "ndasmu", "ndas", "piyo", "asu",
|
||||
];
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// False-positive filters — exclude badword hits that appear only as
|
||||
// substrings of longer innocent words. Each filter checks whether the
|
||||
// hit exists as a standalone word OR as part of a word that is NOT in
|
||||
// the whitelist.
|
||||
// -----------------------------------------------------------------------
|
||||
const words = lowerText.match(/[\p{L}\p{N}_]+/gu) || [];
|
||||
const FALSE_POSITIVE_WHITELISTS: Record<string, string[]> = {
|
||||
asu: [
|
||||
"asus", "masuk", "termasuk", "dimasukkan", "memasukkan",
|
||||
"kasur", "asumsi", "asuransi", "asupan", "pasukan", "pasundan",
|
||||
],
|
||||
goblok: ["goblok"],
|
||||
kontol: ["kontol"],
|
||||
memek: ["memek"],
|
||||
tolol: ["tolol"],
|
||||
};
|
||||
|
||||
/** Returns true if the given hit appears in the text as a standalone word
|
||||
* or inside a word that is NOT in the whitelist. */
|
||||
const isRealHit = (hit: string, whitelist: string[]): boolean => {
|
||||
for (const w of words) {
|
||||
if (w.includes(hit)) {
|
||||
// If the word IS an exact match, it's definitely a real hit.
|
||||
if (w === hit) return true;
|
||||
// If it's inside a longer word, check the whitelist.
|
||||
if (!whitelist.includes(w)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
function detectLocalBadwords(text: string): string[] {
|
||||
const lowerText = text.toLowerCase();
|
||||
const words = lowerText.match(/[\p{L}\p{N}_]+/gu) || [];
|
||||
|
||||
hits = hits.filter((hit) => {
|
||||
switch (hit) {
|
||||
case "asu":
|
||||
return isRealHit(hit, [
|
||||
"asus", "masuk", "termasuk", "dimasukkan", "memasukkan",
|
||||
"kasur", "asumsi", "asuransi", "asupan", "pasukan", "pasundan",
|
||||
]);
|
||||
case "goblok":
|
||||
return isRealHit(hit, [
|
||||
"goblok", // standalone is always flagged
|
||||
]);
|
||||
case "kontol":
|
||||
return isRealHit(hit, [
|
||||
"kontol", // standalone is always flagged
|
||||
]);
|
||||
case "memek":
|
||||
return isRealHit(hit, [
|
||||
"memek", // standalone is always flagged
|
||||
]);
|
||||
case "tolol":
|
||||
return isRealHit(hit, [
|
||||
"tolol", // standalone is always flagged
|
||||
]);
|
||||
case "beg":
|
||||
// Short substring — only flag if it appears as a standalone word
|
||||
// or in a known profanity context, not inside "bego" variants.
|
||||
return words.some(w => w === "beg" || w === "bgo" || w === "bgoo");
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Secondary detection: catch slang/vowelless forms the npm package misses.
|
||||
// These are words that appear standalone (not inside a longer word) after
|
||||
// normalization has already run.
|
||||
// -----------------------------------------------------------------------
|
||||
const SLANG_BADWORDS = [
|
||||
"anjing", "bangsat", "brengsek", "bajingan", "kontol", "memek",
|
||||
"tai", "goblok", "tolol", "bego", "sialan", "jancuk", "kampret",
|
||||
"pepek", "jembut", "ngentot", "ngewe", "coli", "celaka", "laknat",
|
||||
"pantek", "entod", "ndasmu", "ndas", "piyo",
|
||||
];
|
||||
|
||||
for (const slang of SLANG_BADWORDS) {
|
||||
if (hits.includes(slang)) continue;
|
||||
|
||||
const standalonePattern = new RegExp(
|
||||
`(?:^|\\s|[^\\p{L}])${slang}(?:$|\\s|[^\\p{L}])`,
|
||||
"iu",
|
||||
);
|
||||
if (standalonePattern.test(lowerText)) {
|
||||
hits.push(slang);
|
||||
}
|
||||
const isRealHit = (hit: string, whitelist: string[]): boolean => {
|
||||
for (const w of words) {
|
||||
if (w.includes(hit)) {
|
||||
if (w === hit) return true;
|
||||
if (!whitelist.includes(w)) return true;
|
||||
}
|
||||
|
||||
return Array.from(new Set(hits));
|
||||
}
|
||||
} catch {
|
||||
// Keep moderation pipeline resilient if dependency changes shape.
|
||||
return false;
|
||||
};
|
||||
|
||||
const hits: string[] = [];
|
||||
|
||||
for (const badword of LOCAL_BADWORDS) {
|
||||
const whitelist = FALSE_POSITIVE_WHITELISTS[badword] ?? [badword];
|
||||
if (isRealHit(badword, whitelist)) {
|
||||
hits.push(badword);
|
||||
}
|
||||
}
|
||||
return [];
|
||||
|
||||
return Array.from(new Set(hits));
|
||||
}
|
||||
|
||||
export function buildModerationTextEvidence(text: string): ModerationTextEvidence {
|
||||
// ---------------------------------------------------------------------------
|
||||
// NVIDIA Nemotron-3 Content Safety API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Call NVIDIA Nemotron-3 Content Safety API to detect harmful content.
|
||||
* Returns categories/flags from the API response.
|
||||
*/
|
||||
async function callNemotronContentSafety(text: string): Promise<string[]> {
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (!apiKey) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const response = await axios.post(
|
||||
config.NVIDIA_NEMOTRON_BASE_URL,
|
||||
{
|
||||
model: config.NVIDIA_NEMOTRON_MODEL,
|
||||
messages: [{ role: "user", content: text }],
|
||||
max_tokens: 897,
|
||||
temperature: 0.2,
|
||||
top_p: 0.7,
|
||||
stream: false,
|
||||
chat_template_kwargs: { request_categories: "/categories" },
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
Accept: "application/json",
|
||||
},
|
||||
timeout: 15_000,
|
||||
},
|
||||
);
|
||||
|
||||
const data = response.data;
|
||||
const categories: string[] = [];
|
||||
|
||||
// Parse the LLM response for category flags
|
||||
const content = data?.choices?.[0]?.message?.content ?? "";
|
||||
if (content) {
|
||||
const lowerContent = content.toLowerCase();
|
||||
for (const category of NVIDIA_BAD_CATEGORIES) {
|
||||
// Check if the category appears as a key in the response
|
||||
// The Nemotron content safety model returns structured data with category scores
|
||||
if (lowerContent.includes(category)) {
|
||||
categories.push(CATEGORY_TO_BADWORD_LABEL[category] ?? category);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also check for structured response fields
|
||||
const choice = data?.choices?.[0];
|
||||
if (choice?.message?.content) {
|
||||
try {
|
||||
const parsed = JSON.parse(choice.message.content);
|
||||
if (parsed.categories && Array.isArray(parsed.categories)) {
|
||||
for (const cat of parsed.categories) {
|
||||
if (NVIDIA_BAD_CATEGORIES.has(cat.name ?? cat)) {
|
||||
categories.push(CATEGORY_TO_BADWORD_LABEL[cat.name ?? cat] ?? cat);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Not JSON — already handled via text search above
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(new Set(categories));
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect badwords in text using NVIDIA Nemotron-3 Content Safety API.
|
||||
* Falls back to local lexical list if API key is missing or call fails.
|
||||
*/
|
||||
export async function detectIndonesianBadwords(text: string): Promise<string[]> {
|
||||
// Always run local detection first (fast, no network dependency)
|
||||
const localHits = detectLocalBadwords(text);
|
||||
|
||||
// Try NVIDIA API if key is configured
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (apiKey) {
|
||||
try {
|
||||
const apiCategories = await callNemotronContentSafety(text);
|
||||
const allHits = Array.from(new Set([...localHits, ...apiCategories]));
|
||||
return allHits;
|
||||
} catch (error) {
|
||||
log.warn({ error }, "NVIDIA Nemotron API call failed, falling back to local detection");
|
||||
}
|
||||
}
|
||||
|
||||
return localHits;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Async evidence builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function buildModerationTextEvidence(text: string): Promise<ModerationTextEvidence> {
|
||||
const emojiNormalized = normalizeDiscordCustomEmoji(text);
|
||||
const slangNormalized = normalizeIndonesianSlang(emojiNormalized.text);
|
||||
const badwordHits = detectIndonesianBadwords(slangNormalized.text);
|
||||
const badwordHits = await detectIndonesianBadwords(slangNormalized.text);
|
||||
const notes = [...slangNormalized.notes];
|
||||
|
||||
for (const emojiName of emojiNormalized.emojiNames) {
|
||||
@@ -160,9 +235,9 @@ export function buildModerationTextEvidence(text: string): ModerationTextEvidenc
|
||||
}
|
||||
|
||||
if (badwordHits.length > 0) {
|
||||
notes.push(`local lexical check: Indonesian badword detected: ${badwordHits.join(", ")}`);
|
||||
notes.push(`Indonesian badword detected: ${badwordHits.join(", ")}`);
|
||||
} else {
|
||||
notes.push("local lexical check: no Indonesian badword detected");
|
||||
notes.push("no Indonesian badword detected");
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -174,8 +249,8 @@ export function buildModerationTextEvidence(text: string): ModerationTextEvidenc
|
||||
};
|
||||
}
|
||||
|
||||
export function formatModerationTextEvidenceForPrompt(text: string): string {
|
||||
const evidence = buildModerationTextEvidence(text);
|
||||
export async function formatModerationTextEvidenceForPrompt(text: string): Promise<string> {
|
||||
const evidence = await buildModerationTextEvidence(text);
|
||||
if (evidence.normalized === evidence.raw && evidence.notes.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user