feat: integrate NVIDIA Nemotron-3 Content Safety API for Indonesian badword detection
- Added configuration options for NVIDIA Nemotron API key, model, and base URL. - Refactored badword detection to utilize NVIDIA API, with a fallback to a local badword list. - Updated moderation functions to handle asynchronous operations for text evidence generation. - Removed dependency on the `indonesian-badwords` package and implemented custom detection logic. - Enhanced tests to accommodate asynchronous behavior and validate new detection methods.
This commit is contained in:
MythEclipse
@@ -114,6 +114,17 @@ const configSchema = z
|
||||
.int()
|
||||
.positive()
|
||||
.default(10),
|
||||
/** NVIDIA Nemotron-3 Content Safety API key for badword detection. */
|
||||
NVIDIA_NEMOTRON_API_KEY: z.string().optional(),
|
||||
/** NVIDIA Nemotron model identifier. */
|
||||
NVIDIA_NEMOTRON_MODEL: z
|
||||
.string()
|
||||
.default("nvidia/nemotron-3-content-safety"),
|
||||
/** NVIDIA Nemotron API base URL. */
|
||||
NVIDIA_NEMOTRON_BASE_URL: z
|
||||
.string()
|
||||
.url()
|
||||
.default("https://integrate.api.nvidia.com/v1/chat/completions"),
|
||||
AUTO_DELETE_FLAGGED_ENABLED: z
|
||||
.string()
|
||||
.optional()
|
||||
|
||||
@@ -77,7 +77,7 @@ export default async function processAnalysisRequest({
|
||||
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
||||
});
|
||||
|
||||
const contextLines = buildConversationContext({
|
||||
const contextLines = await buildConversationContext({
|
||||
contextBefore,
|
||||
targets: messages,
|
||||
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
||||
|
||||
@@ -9,8 +9,6 @@ import { retryWithBackoff } from "../retry.js";
|
||||
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
|
||||
import {
|
||||
buildConversationContext,
|
||||
estimateTokens,
|
||||
formatMessageForPrompt,
|
||||
} from "./conversationContext.js";
|
||||
import { runModerationAnalysis } from "./llmModerationClient.js";
|
||||
import {
|
||||
@@ -157,6 +155,8 @@ export function getConversationKey(message: MessageRecord): string {
|
||||
/**
|
||||
* Picks a batch of messages within a token budget.
|
||||
* `tokensPerMessage` accounts for JSON structure overhead around each entry.
|
||||
* Uses a rough character-based token estimate (avoids async formatMessageForPrompt
|
||||
* since this function runs in a synchronous promise chain).
|
||||
*/
|
||||
export function pickBatchWithinBudget(
|
||||
messages: MessageRecord[],
|
||||
@@ -167,8 +167,9 @@ export function pickBatchWithinBudget(
|
||||
let usedTokens = 0;
|
||||
|
||||
for (const msg of messages) {
|
||||
const formatted = formatMessageForPrompt(msg, "target");
|
||||
const msgTokens = estimateTokens(formatted) + tokensPerMessage;
|
||||
const content = msg.edited_content ?? msg.content;
|
||||
// Rough token estimate: ~3 chars per token + metadata overhead
|
||||
const msgTokens = Math.ceil(content.length / 3) + tokensPerMessage;
|
||||
|
||||
if (usedTokens + msgTokens <= maxTokens) {
|
||||
batch.push(msg);
|
||||
@@ -238,7 +239,7 @@ async function processIndividualFallback(
|
||||
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
||||
});
|
||||
|
||||
const contextLines = buildConversationContext({
|
||||
const contextLines = await buildConversationContext({
|
||||
contextBefore,
|
||||
targets: [message],
|
||||
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
||||
|
||||
@@ -25,13 +25,13 @@ export function estimateTokens(text: string): number {
|
||||
/**
|
||||
* Formats a single message for context or target display
|
||||
*/
|
||||
export function formatMessageForPrompt(
|
||||
export async function formatMessageForPrompt(
|
||||
msg: MessageRecord,
|
||||
label: "context" | "target",
|
||||
): string {
|
||||
): Promise<string> {
|
||||
const content = msg.edited_content ?? msg.content;
|
||||
const timestamp = formatTimestamp(msg.created_at);
|
||||
const textEvidence = formatModerationTextEvidenceForPrompt(content);
|
||||
const textEvidence = await formatModerationTextEvidenceForPrompt(content);
|
||||
const textSuffix = textEvidence ? ` ${textEvidence}` : "";
|
||||
const mediaEvidence = formatMediaEvidenceForPrompt(msg.metadata);
|
||||
const mediaSuffix = mediaEvidence ? ` ${mediaEvidence}` : "";
|
||||
@@ -42,22 +42,23 @@ export function formatMessageForPrompt(
|
||||
* Builds conversation historical context without including targets.
|
||||
* Calculates how much token budget targets use, and fills the rest with context.
|
||||
*/
|
||||
export function buildConversationContext(
|
||||
export async function buildConversationContext(
|
||||
input: ConversationContextInput,
|
||||
): string[] {
|
||||
): Promise<string[]> {
|
||||
const { contextBefore, targets, maxTokens } = input;
|
||||
|
||||
// Calculate tokens used by targets
|
||||
let usedTokens = targets.reduce((sum, msg) => {
|
||||
return sum + estimateTokens(formatMessageForPrompt(msg, "target"));
|
||||
}, 0);
|
||||
// Calculate tokens used by targets (parallel)
|
||||
const targetLines = await Promise.all(
|
||||
targets.map((msg) => formatMessageForPrompt(msg, "target")),
|
||||
);
|
||||
let usedTokens = targetLines.reduce((sum, line) => sum + estimateTokens(line), 0);
|
||||
|
||||
const selectedContextLines: string[] = [];
|
||||
|
||||
// Go backwards through context, taking most recent first
|
||||
for (let i = contextBefore.length - 1; i >= 0; i--) {
|
||||
const msg = contextBefore[i];
|
||||
const line = formatMessageForPrompt(msg, "context");
|
||||
const line = await formatMessageForPrompt(msg, "context");
|
||||
const lineTokens = estimateTokens(line);
|
||||
|
||||
if (usedTokens + lineTokens <= maxTokens) {
|
||||
|
||||
@@ -1,20 +1,40 @@
|
||||
import badwordsModule from "indonesian-badwords";
|
||||
import axios from "axios";
|
||||
import { config } from "../config.js";
|
||||
import { INDONESIAN_SLANG_LEXICON } from "./resources/indonesianSlangLexicon.js";
|
||||
import { createChildLogger } from "../logger.js";
|
||||
|
||||
const log = createChildLogger("indonesianTextNormalizer");
|
||||
|
||||
const CUSTOM_EMOJI_PATTERN = /<a?:([a-zA-Z0-9_]+):(\d+)>/g;
|
||||
const WORD_PATTERN = /[\p{L}\p{N}_]+/gu;
|
||||
|
||||
interface BadwordAnalyzeResult {
|
||||
badwords?: string[];
|
||||
count?: number;
|
||||
}
|
||||
/** NVIDIA content safety categories that map to offensive/badword content. */
|
||||
const NVIDIA_BAD_CATEGORIES = new Set([
|
||||
"hate",
|
||||
"harassment",
|
||||
"sexual",
|
||||
"violence",
|
||||
"self-harm",
|
||||
"illicit",
|
||||
"profanity",
|
||||
"vulgar",
|
||||
"insult",
|
||||
]);
|
||||
|
||||
interface BadwordsModule {
|
||||
analyze?: (text: string) => BadwordAnalyzeResult;
|
||||
flag?: (text: string) => boolean;
|
||||
}
|
||||
|
||||
const badwords = badwordsModule as BadwordsModule;
|
||||
/**
|
||||
* Map NVIDIA Nemotron category labels to Indonesian badword-style labels.
|
||||
*/
|
||||
const CATEGORY_TO_BADWORD_LABEL: Record<string, string> = {
|
||||
hate: "hate_speech",
|
||||
harassment: "harassment",
|
||||
sexual: "sexual_content",
|
||||
violence: "violence",
|
||||
"self-harm": "self_harm",
|
||||
illicit: "illegal_content",
|
||||
profanity: "vulgar_language",
|
||||
vulgar: "vulgar_language",
|
||||
insult: "harassment",
|
||||
};
|
||||
|
||||
export interface ModerationTextEvidence {
|
||||
raw: string;
|
||||
@@ -24,6 +44,10 @@ export interface ModerationTextEvidence {
|
||||
hasBadwords: boolean;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sync helpers (unchanged)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function normalizeDiscordCustomEmoji(text: string): {
|
||||
text: string;
|
||||
emojiNames: string[];
|
||||
@@ -53,104 +77,155 @@ export function normalizeIndonesianSlang(text: string): {
|
||||
return { text: normalized, notes: Array.from(new Set(notes)) };
|
||||
}
|
||||
|
||||
export function detectIndonesianBadwords(text: string): string[] {
|
||||
try {
|
||||
const result = badwords.analyze?.(text);
|
||||
if (Array.isArray(result?.badwords)) {
|
||||
let hits = Array.from(new Set(result.badwords.map((word) => word.toLowerCase())));
|
||||
// ---------------------------------------------------------------------------
|
||||
// Local fallback badword list (used when NVIDIA API is unavailable)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const lowerText = text.toLowerCase();
|
||||
const LOCAL_BADWORDS = [
|
||||
"anjing", "bangsat", "brengsek", "bajingan", "kontol", "memek",
|
||||
"tai", "goblok", "tolol", "bego", "sialan", "jancuk", "kampret",
|
||||
"pepek", "jembut", "ngentot", "ngewe", "coli", "celaka", "laknat",
|
||||
"pantek", "entod", "ndasmu", "ndas", "piyo", "asu",
|
||||
];
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// False-positive filters — exclude badword hits that appear only as
|
||||
// substrings of longer innocent words. Each filter checks whether the
|
||||
// hit exists as a standalone word OR as part of a word that is NOT in
|
||||
// the whitelist.
|
||||
// -----------------------------------------------------------------------
|
||||
const words = lowerText.match(/[\p{L}\p{N}_]+/gu) || [];
|
||||
const FALSE_POSITIVE_WHITELISTS: Record<string, string[]> = {
|
||||
asu: [
|
||||
"asus", "masuk", "termasuk", "dimasukkan", "memasukkan",
|
||||
"kasur", "asumsi", "asuransi", "asupan", "pasukan", "pasundan",
|
||||
],
|
||||
goblok: ["goblok"],
|
||||
kontol: ["kontol"],
|
||||
memek: ["memek"],
|
||||
tolol: ["tolol"],
|
||||
};
|
||||
|
||||
/** Returns true if the given hit appears in the text as a standalone word
|
||||
* or inside a word that is NOT in the whitelist. */
|
||||
const isRealHit = (hit: string, whitelist: string[]): boolean => {
|
||||
for (const w of words) {
|
||||
if (w.includes(hit)) {
|
||||
// If the word IS an exact match, it's definitely a real hit.
|
||||
if (w === hit) return true;
|
||||
// If it's inside a longer word, check the whitelist.
|
||||
if (!whitelist.includes(w)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
function detectLocalBadwords(text: string): string[] {
|
||||
const lowerText = text.toLowerCase();
|
||||
const words = lowerText.match(/[\p{L}\p{N}_]+/gu) || [];
|
||||
|
||||
hits = hits.filter((hit) => {
|
||||
switch (hit) {
|
||||
case "asu":
|
||||
return isRealHit(hit, [
|
||||
"asus", "masuk", "termasuk", "dimasukkan", "memasukkan",
|
||||
"kasur", "asumsi", "asuransi", "asupan", "pasukan", "pasundan",
|
||||
]);
|
||||
case "goblok":
|
||||
return isRealHit(hit, [
|
||||
"goblok", // standalone is always flagged
|
||||
]);
|
||||
case "kontol":
|
||||
return isRealHit(hit, [
|
||||
"kontol", // standalone is always flagged
|
||||
]);
|
||||
case "memek":
|
||||
return isRealHit(hit, [
|
||||
"memek", // standalone is always flagged
|
||||
]);
|
||||
case "tolol":
|
||||
return isRealHit(hit, [
|
||||
"tolol", // standalone is always flagged
|
||||
]);
|
||||
case "beg":
|
||||
// Short substring — only flag if it appears as a standalone word
|
||||
// or in a known profanity context, not inside "bego" variants.
|
||||
return words.some(w => w === "beg" || w === "bgo" || w === "bgoo");
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Secondary detection: catch slang/vowelless forms the npm package misses.
|
||||
// These are words that appear standalone (not inside a longer word) after
|
||||
// normalization has already run.
|
||||
// -----------------------------------------------------------------------
|
||||
const SLANG_BADWORDS = [
|
||||
"anjing", "bangsat", "brengsek", "bajingan", "kontol", "memek",
|
||||
"tai", "goblok", "tolol", "bego", "sialan", "jancuk", "kampret",
|
||||
"pepek", "jembut", "ngentot", "ngewe", "coli", "celaka", "laknat",
|
||||
"pantek", "entod", "ndasmu", "ndas", "piyo",
|
||||
];
|
||||
|
||||
for (const slang of SLANG_BADWORDS) {
|
||||
if (hits.includes(slang)) continue;
|
||||
|
||||
const standalonePattern = new RegExp(
|
||||
`(?:^|\\s|[^\\p{L}])${slang}(?:$|\\s|[^\\p{L}])`,
|
||||
"iu",
|
||||
);
|
||||
if (standalonePattern.test(lowerText)) {
|
||||
hits.push(slang);
|
||||
}
|
||||
const isRealHit = (hit: string, whitelist: string[]): boolean => {
|
||||
for (const w of words) {
|
||||
if (w.includes(hit)) {
|
||||
if (w === hit) return true;
|
||||
if (!whitelist.includes(w)) return true;
|
||||
}
|
||||
|
||||
return Array.from(new Set(hits));
|
||||
}
|
||||
} catch {
|
||||
// Keep moderation pipeline resilient if dependency changes shape.
|
||||
return false;
|
||||
};
|
||||
|
||||
const hits: string[] = [];
|
||||
|
||||
for (const badword of LOCAL_BADWORDS) {
|
||||
const whitelist = FALSE_POSITIVE_WHITELISTS[badword] ?? [badword];
|
||||
if (isRealHit(badword, whitelist)) {
|
||||
hits.push(badword);
|
||||
}
|
||||
}
|
||||
return [];
|
||||
|
||||
return Array.from(new Set(hits));
|
||||
}
|
||||
|
||||
export function buildModerationTextEvidence(text: string): ModerationTextEvidence {
|
||||
// ---------------------------------------------------------------------------
|
||||
// NVIDIA Nemotron-3 Content Safety API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Call NVIDIA Nemotron-3 Content Safety API to detect harmful content.
|
||||
* Returns categories/flags from the API response.
|
||||
*/
|
||||
async function callNemotronContentSafety(text: string): Promise<string[]> {
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (!apiKey) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const response = await axios.post(
|
||||
config.NVIDIA_NEMOTRON_BASE_URL,
|
||||
{
|
||||
model: config.NVIDIA_NEMOTRON_MODEL,
|
||||
messages: [{ role: "user", content: text }],
|
||||
max_tokens: 897,
|
||||
temperature: 0.2,
|
||||
top_p: 0.7,
|
||||
stream: false,
|
||||
chat_template_kwargs: { request_categories: "/categories" },
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
Accept: "application/json",
|
||||
},
|
||||
timeout: 15_000,
|
||||
},
|
||||
);
|
||||
|
||||
const data = response.data;
|
||||
const categories: string[] = [];
|
||||
|
||||
// Parse the LLM response for category flags
|
||||
const content = data?.choices?.[0]?.message?.content ?? "";
|
||||
if (content) {
|
||||
const lowerContent = content.toLowerCase();
|
||||
for (const category of NVIDIA_BAD_CATEGORIES) {
|
||||
// Check if the category appears as a key in the response
|
||||
// The Nemotron content safety model returns structured data with category scores
|
||||
if (lowerContent.includes(category)) {
|
||||
categories.push(CATEGORY_TO_BADWORD_LABEL[category] ?? category);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also check for structured response fields
|
||||
const choice = data?.choices?.[0];
|
||||
if (choice?.message?.content) {
|
||||
try {
|
||||
const parsed = JSON.parse(choice.message.content);
|
||||
if (parsed.categories && Array.isArray(parsed.categories)) {
|
||||
for (const cat of parsed.categories) {
|
||||
if (NVIDIA_BAD_CATEGORIES.has(cat.name ?? cat)) {
|
||||
categories.push(CATEGORY_TO_BADWORD_LABEL[cat.name ?? cat] ?? cat);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Not JSON — already handled via text search above
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(new Set(categories));
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect badwords in text using NVIDIA Nemotron-3 Content Safety API.
|
||||
* Falls back to local lexical list if API key is missing or call fails.
|
||||
*/
|
||||
export async function detectIndonesianBadwords(text: string): Promise<string[]> {
|
||||
// Always run local detection first (fast, no network dependency)
|
||||
const localHits = detectLocalBadwords(text);
|
||||
|
||||
// Try NVIDIA API if key is configured
|
||||
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
|
||||
if (apiKey) {
|
||||
try {
|
||||
const apiCategories = await callNemotronContentSafety(text);
|
||||
const allHits = Array.from(new Set([...localHits, ...apiCategories]));
|
||||
return allHits;
|
||||
} catch (error) {
|
||||
log.warn({ error }, "NVIDIA Nemotron API call failed, falling back to local detection");
|
||||
}
|
||||
}
|
||||
|
||||
return localHits;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Async evidence builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function buildModerationTextEvidence(text: string): Promise<ModerationTextEvidence> {
|
||||
const emojiNormalized = normalizeDiscordCustomEmoji(text);
|
||||
const slangNormalized = normalizeIndonesianSlang(emojiNormalized.text);
|
||||
const badwordHits = detectIndonesianBadwords(slangNormalized.text);
|
||||
const badwordHits = await detectIndonesianBadwords(slangNormalized.text);
|
||||
const notes = [...slangNormalized.notes];
|
||||
|
||||
for (const emojiName of emojiNormalized.emojiNames) {
|
||||
@@ -160,9 +235,9 @@ export function buildModerationTextEvidence(text: string): ModerationTextEvidenc
|
||||
}
|
||||
|
||||
if (badwordHits.length > 0) {
|
||||
notes.push(`local lexical check: Indonesian badword detected: ${badwordHits.join(", ")}`);
|
||||
notes.push(`Indonesian badword detected: ${badwordHits.join(", ")}`);
|
||||
} else {
|
||||
notes.push("local lexical check: no Indonesian badword detected");
|
||||
notes.push("no Indonesian badword detected");
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -174,8 +249,8 @@ export function buildModerationTextEvidence(text: string): ModerationTextEvidenc
|
||||
};
|
||||
}
|
||||
|
||||
export function formatModerationTextEvidenceForPrompt(text: string): string {
|
||||
const evidence = buildModerationTextEvidence(text);
|
||||
export async function formatModerationTextEvidenceForPrompt(text: string): Promise<string> {
|
||||
const evidence = await buildModerationTextEvidence(text);
|
||||
if (evidence.normalized === evidence.raw && evidence.notes.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
@@ -805,7 +805,17 @@ CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan
|
||||
let lastParseError: string | null = null;
|
||||
let lastInvalidContent: string | null = null;
|
||||
|
||||
const buildMessageContent = (): string => {
|
||||
// Pre-compute text evidence for all targets in parallel
|
||||
const textEvidenceMap = new Map<string, string>();
|
||||
await Promise.all(
|
||||
targets.map(async (msg) => {
|
||||
const content = msg.edited_content ?? msg.content;
|
||||
const evidence = await formatModerationTextEvidenceForPrompt(content);
|
||||
textEvidenceMap.set(msg.id, evidence);
|
||||
}),
|
||||
);
|
||||
|
||||
const buildMessageContent = async (): Promise<string> => {
|
||||
const correction = lastParseError
|
||||
? {
|
||||
error: lastParseError,
|
||||
@@ -821,7 +831,7 @@ CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan
|
||||
const webTexts = messageWebTextMap.get(msg.id) ?? [];
|
||||
const mediaAnalyses = messageMediaAnalysisMap.get(msg.id) ?? [];
|
||||
const webContext = webTexts.length > 0 ? `\n${webTexts.join("\n")}` : "";
|
||||
const textEvidence = formatModerationTextEvidenceForPrompt(content);
|
||||
const textEvidence = textEvidenceMap.get(msg.id) ?? "";
|
||||
const textContext = textEvidence ? `\n${textEvidence}` : "";
|
||||
const mediaAnalysisContext =
|
||||
mediaAnalyses.length > 0 ? `\n${mediaAnalyses.join("\n")}` : "";
|
||||
@@ -856,7 +866,7 @@ CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: buildMessageContent(),
|
||||
content: await buildMessageContent(),
|
||||
},
|
||||
],
|
||||
temperature: 0.2,
|
||||
|
||||
Vendored
-25
@@ -1,25 +0,0 @@
|
||||
declare module "indonesian-badwords" {
|
||||
export interface BadwordAnalyzeResult {
|
||||
text?: string;
|
||||
words?: number;
|
||||
censored?: string;
|
||||
badwords?: string[];
|
||||
count?: number;
|
||||
locations?: Array<{ word: string; index: number }>;
|
||||
}
|
||||
|
||||
export function analyze(text: string): BadwordAnalyzeResult;
|
||||
export function flag(text: string): boolean;
|
||||
export function filter(text: string): string;
|
||||
export function censor(text: string): string;
|
||||
|
||||
const value: {
|
||||
analyze: typeof analyze;
|
||||
flag: typeof flag;
|
||||
filter: typeof filter;
|
||||
censor: typeof censor;
|
||||
dict?: unknown;
|
||||
badwords?: unknown;
|
||||
};
|
||||
export default value;
|
||||
}
|
||||
Reference in New Issue
Block a user