MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: session auto-commit

Browse Source
This commit is contained in:
MythEclipse
2026-06-02 17:12:42 +07:00
parent 498ab73d62
commit b23ccf0837
7 changed files with 76 additions and 625 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
-8
View File
@@ -51,14 +51,6 @@ AI_LLM_MODEL=free
# Vision model for image/video moderation (falls back to AI_LLM_MODEL if unset) # Vision model for image/video moderation (falls back to AI_LLM_MODEL if unset)
AI_LLM_VISION_MODEL=multimodal AI_LLM_VISION_MODEL=multimodal
# NVIDIA Nemotron Content Safety Configuration
NVIDIA_NEMOTRON_API_KEY=your_nvidia_api_key_here
# Groq Llama Prompt Guard Fallback Configuration
GROQ_API_KEY=your_groq_api_key_here
GROQ_MODERATION_MODEL=meta-llama/llama-prompt-guard-2-86m
GROQ_MODERATION_BASE_URL=https://api.groq.com/openai/v1/chat/completions
# Database Configuration (PostgreSQL) # Database Configuration (PostgreSQL)
# Option 1: Use DATABASE_URL for connection string # Option 1: Use DATABASE_URL for connection string
# DATABASE_URL=postgresql://user:password@localhost:5432/discord_bot # DATABASE_URL=postgresql://user:password@localhost:5432/discord_bot
services/discord-gateway/src/modules/ai-moderation/indonesianTextNormalizer.ts
+34 -328
View File
@@ -1,43 +1,12 @@
import axios from "axios";
import OpenAI from "openai"; import OpenAI from "openai";
import { AbortError } from "p-retry";
import { config } from "../../shared/config/config.js"; import { config } from "../../shared/config/config.js";
import { createChildLogger } from "../../shared/logger/logger.js"; import { createChildLogger } from "../../shared/logger/logger.js";
import { retryWithBackoff } from "../../shared/utils/retry.js";
import { getCachedText, upsertCachedText } from "./textCacheStore.js"; import { getCachedText, upsertCachedText } from "./textCacheStore.js";
const log = createChildLogger("indonesianTextNormalizer"); const log = createChildLogger("indonesianTextNormalizer");
const CUSTOM_EMOJI_PATTERN = /<a?:([a-zA-Z0-9_]+):(\d+)>/g; const CUSTOM_EMOJI_PATTERN = /<a?:([a-zA-Z0-9_]+):(\d+)>/g;
/** NVIDIA content safety categories that map to offensive/badword content. */
const NVIDIA_BAD_CATEGORIES = new Set([
"hate",
"harassment",
"sexual",
"violence",
"self-harm",
"illicit",
"profanity",
"vulgar",
"insult",
]);
/**
* Map NVIDIA Nemotron category labels to Indonesian badword-style labels.
*/
const CATEGORY_TO_BADWORD_LABEL: Record<string, string> = {
hate: "hate_speech",
harassment: "harassment",
sexual: "sexual_content",
violence: "violence",
"self-harm": "self_harm",
illicit: "illegal_content",
profanity: "vulgar_language",
vulgar: "vulgar_language",
insult: "harassment",
};
const VALID_PRIMARY_AI_FLAGS = new Set([ const VALID_PRIMARY_AI_FLAGS = new Set([
"spam", "spam",
"hate_speech", "hate_speech",
@@ -75,13 +44,6 @@ const BADWORD_CACHE_TTL_MS = 10 * 60 * 1000;
*/ */
const DB_CACHE_TTL_MS = 24 * 60 * 60 * 1000; const DB_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
const NEMOTRON_RATE_LIMIT_COOLDOWN_MS = 60_000; // 1 min backoff on 429
const PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS = 60_000; // 1 min backoff on 429
const GROQ_RATE_LIMIT_COOLDOWN_MS = 60_000; // 1 min backoff on 429
/** How long to mark a provider unavailable after a transient (5xx/timeout) error. */
const TRANSIENT_ERROR_COOLDOWN_MS = 30_000; // 30s backoff on 502/timeout
interface BadwordCacheEntry { interface BadwordCacheEntry {
value: string[]; value: string[];
expiresAt: number; expiresAt: number;
@@ -89,9 +51,6 @@ interface BadwordCacheEntry {
const badwordCache = new Map<string, BadwordCacheEntry>(); const badwordCache = new Map<string, BadwordCacheEntry>();
const inFlightBadwordLookups = new Map<string, Promise<string[]>>(); const inFlightBadwordLookups = new Map<string, Promise<string[]>>();
let nemotronUnavailableUntil = 0;
let primaryAiUnavailableUntil = 0;
let groqUnavailableUntil = 0;
let primaryModerationClient: OpenAI | null = null; let primaryModerationClient: OpenAI | null = null;
export interface ModerationTextEvidence { export interface ModerationTextEvidence {
@@ -103,7 +62,7 @@ export interface ModerationTextEvidence {
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Sync helpers (unchanged) // Sync helpers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
export function normalizeDiscordCustomEmoji(text: string): { export function normalizeDiscordCustomEmoji(text: string): {
@@ -122,10 +81,6 @@ export function normalizeDiscordCustomEmoji(text: string): {
return { text: normalized, emojiNames }; return { text: normalized, emojiNames };
} }
// Local badword detection removed (lines 121-198).
// All detection now goes through the API pipeline (NVIDIA → Primary AI → Groq)
// to eliminate false positives from substring matching and hardcoded whitelists.
function normalizeBadwordCacheKey(text: string): string { function normalizeBadwordCacheKey(text: string): string {
return text.trim().replace(/\s+/g, " ").toLowerCase(); return text.trim().replace(/\s+/g, " ").toLowerCase();
} }
@@ -194,7 +149,7 @@ function normalizePrimaryAiFlag(value: string): string | null {
return lower; return lower;
} }
return CATEGORY_TO_BADWORD_LABEL[lower] ?? null; return null;
} }
function extractFlagsFromPrimaryAiContent(content: string): string[] { function extractFlagsFromPrimaryAiContent(content: string): string[] {
@@ -240,13 +195,6 @@ function extractFlagsFromPrimaryAiContent(content: string): string[] {
} }
} }
for (const category of Object.keys(CATEGORY_TO_BADWORD_LABEL)) {
if (lowerContent.includes(category)) {
const mapped = CATEGORY_TO_BADWORD_LABEL[category];
if (mapped) flags.add(mapped);
}
}
return Array.from(flags); return Array.from(flags);
} }
@@ -256,36 +204,25 @@ async function callPrimaryAiModeration(text: string): Promise<string[]> {
return []; return [];
} }
const completion = await retryWithBackoff( const completion = await client.chat.completions.create({
async () => { model: config.AI_LLM_MODEL,
return client.chat.completions.create({ messages: [
model: config.AI_LLM_MODEL, {
messages: [ role: "user",
{ content:
role: "user", "Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " +
content: 'Balas hanya JSON object dengan format {"flags":[...]} dan gunakan hanya flag valid ini: ' +
"Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " + Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") +
'Balas hanya JSON object dengan format {"flags":[...]} dan gunakan hanya flag valid ini: ' + ". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " +
Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") + text,
". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " + },
text, ],
}, temperature: 0.1,
], top_p: 0.9,
temperature: 0.1, max_tokens: 200,
top_p: 0.9, stream: false,
max_tokens: 200, response_format: { type: "json_object" },
stream: false, } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming);
response_format: { type: "json_object" },
} as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming);
},
{
retries: 2,
minTimeout: 2000,
maxTimeout: 5000,
factor: 2,
logger: log,
},
);
const content = completion.choices[0]?.message?.content?.trim(); const content = completion.choices[0]?.message?.content?.trim();
if (!content) { if (!content) {
@@ -296,175 +233,18 @@ async function callPrimaryAiModeration(text: string): Promise<string[]> {
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Groq Llama Prompt Guard Moderation API (Fallback) // Two-tier cache + Primary AI pipeline
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
/** /**
* Call Groq Llama Prompt Guard 2-86M model for moderation scoring. * Detect badwords in text using a **two-tier cache + Primary AI**:
* Returns a probability score as a string (e.g. "0.9988824725151062").
* Scores above ~0.5 indicate moderation violations.
*/
async function callGrokModeration(text: string): Promise<string[]> {
const apiKey = config.GROQ_API_KEY;
if (!apiKey) {
return [];
}
const response = await retryWithBackoff(
async () => {
const res = await axios.post(
config.GROQ_MODERATION_BASE_URL,
{
model: config.GROQ_MODERATION_MODEL,
messages: [{ role: "user", content: text }],
},
{
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json",
"Content-Type": "application/json",
},
timeout: 15_000,
validateStatus: (status: number) => status < 500,
},
);
// 429 should abort retry immediately — no point hammering a rate limit
if (res.status === 429) {
throw new AbortError("Groq rate limited");
}
return res;
},
{
retries: 2,
minTimeout: 2000,
maxTimeout: 5000,
factor: 2,
logger: log,
},
);
const scoreStr = response.data?.choices?.[0]?.message?.content?.trim();
if (!scoreStr) {
return [];
}
// Parse the score (Llama Prompt Guard returns a single probability score)
const score = parseFloat(scoreStr);
if (isNaN(score) || score < 0.5) {
return [];
}
// Map score to moderation flags based on severity
const flags: string[] = [];
if (score >= 0.9) {
flags.push("vulgar_language", "harassment");
} else if (score >= 0.7) {
flags.push("vulgar_language");
} else {
flags.push("spam");
}
return flags;
}
// ---------------------------------------------------------------------------
// NVIDIA Nemotron-3 Content Safety API
// ---------------------------------------------------------------------------
/**
* Call NVIDIA Nemotron-3 Content Safety API to detect harmful content.
* Returns categories/flags from the API response.
*/
async function callNemotronContentSafety(text: string): Promise<string[]> {
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
if (!apiKey) {
return [];
}
const response = await retryWithBackoff(
async () => {
const res = await axios.post(
config.NVIDIA_NEMOTRON_BASE_URL,
{
model: config.NVIDIA_NEMOTRON_MODEL,
messages: [{ role: "user", content: text }],
max_tokens: 897,
temperature: 0.2,
top_p: 0.7,
stream: false,
},
{
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json",
},
timeout: 15_000,
validateStatus: (status: number) => status < 500,
},
);
// 429 should abort retry immediately — no point hammering a rate limit
if (res.status === 429) {
throw new AbortError("NVIDIA rate limited");
}
return res;
},
{
retries: 2,
minTimeout: 2000,
maxTimeout: 5000,
factor: 2,
logger: log,
},
);
const data = response.data;
const categories: string[] = [];
// Parse the LLM response for category flags
const content = data?.choices?.[0]?.message?.content ?? "";
if (content) {
const lowerContent = content.toLowerCase();
for (const category of NVIDIA_BAD_CATEGORIES) {
if (lowerContent.includes(category)) {
categories.push(CATEGORY_TO_BADWORD_LABEL[category] ?? category);
}
}
}
// Also check for structured response fields
const choice = data?.choices?.[0];
if (choice?.message?.content) {
try {
const parsed = JSON.parse(choice.message.content);
if (parsed.categories && Array.isArray(parsed.categories)) {
for (const cat of parsed.categories) {
if (NVIDIA_BAD_CATEGORIES.has(cat.name ?? cat)) {
categories.push(CATEGORY_TO_BADWORD_LABEL[cat.name ?? cat] ?? cat);
}
}
}
} catch {
// Not JSON — already handled via text search above
}
}
return Array.from(new Set(categories));
}
// ---------------------------------------------------------------------------
// Three-tier cache pipeline
// ---------------------------------------------------------------------------
/**
* Detect badwords in text using a **two-tier cache + API pipeline**:
* *
* 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path, * 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path,
* keyed by the full normalized text string. * keyed by the full normalized text string.
* 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted * 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted
* across restarts. Uses the FULL normalized text (not per-word) because * across restarts. Uses the FULL normalized text (not per-word) because
* context matters: "kau" alone is clean, but "awas kau" can be a threat. * context matters: "kau" alone is clean, but "awas kau" can be a threat.
* 3. **API pipeline** (NVIDIA → Primary AI → Groq) * 3. **Primary AI** (AI_LLM endpoint) — only runs when both cache layers miss.
* only runs when both cache layers miss.
* *
* No local hardcoded badword list — all detection goes through AI APIs * No local hardcoded badword list — all detection goes through AI APIs
* to eliminate false positives from substring matching. * to eliminate false positives from substring matching.
@@ -495,98 +275,24 @@ export async function detectIndonesianBadwords(
return flags; return flags;
} }
// ── Tier 3: API pipeline ── // ── Tier 3: Primary AI only ──
let finalHits: string[] = [];
const hits = new Set<string>(); try {
let sourceUsed: "nvidia" | "primary_ai" | "groq" = "primary_ai"; finalHits = await callPrimaryAiModeration(text);
} catch (error) {
// 3a. Try NVIDIA API if key is configured and not rate limited. log.warn(
const apiKey = config.NVIDIA_NEMOTRON_API_KEY; { error: error instanceof Error ? error.message : String(error) },
if (apiKey && Date.now() >= nemotronUnavailableUntil) { "Primary AI badword detection failed",
try { );
const apiCategories = await callNemotronContentSafety(text);
for (const hit of apiCategories) {
hits.add(hit);
}
if (apiCategories.length > 0) sourceUsed = "nvidia";
} catch (error) {
const status = axios.isAxiosError(error)
? error.response?.status
: null;
if (status === 429) {
nemotronUnavailableUntil =
Date.now() + NEMOTRON_RATE_LIMIT_COOLDOWN_MS;
} else {
// 502, timeout, or other transient error — cooldown briefly
nemotronUnavailableUntil = Date.now() + TRANSIENT_ERROR_COOLDOWN_MS;
}
log.warn(
{ error },
"NVIDIA Nemotron API call failed, falling back to primary AI",
);
}
} }
// 3b. Try the main AI model next.
if (hits.size === 0 && Date.now() >= primaryAiUnavailableUntil) {
try {
const primaryHits = await callPrimaryAiModeration(text);
for (const hit of primaryHits) {
hits.add(hit);
}
if (primaryHits.length > 0) sourceUsed = "primary_ai";
} catch (error) {
const status = axios.isAxiosError(error)
? error.response?.status
: null;
if (status === 429) {
primaryAiUnavailableUntil =
Date.now() + PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS;
} else {
// 502, timeout, or other transient error — cooldown briefly
primaryAiUnavailableUntil = Date.now() + TRANSIENT_ERROR_COOLDOWN_MS;
}
log.warn(
{ error },
"Primary AI badword detection failed, falling back to Groq",
);
}
}
// 3c. Try Groq Llama Prompt Guard as final API fallback.
if (hits.size === 0 && Date.now() >= groqUnavailableUntil) {
const groqKey = config.GROQ_API_KEY;
if (groqKey) {
try {
const groqHits = await callGrokModeration(text);
for (const hit of groqHits) {
hits.add(hit);
}
if (groqHits.length > 0) sourceUsed = "groq";
} catch (error) {
const status = axios.isAxiosError(error)
? error.response?.status
: null;
if (status === 429) {
groqUnavailableUntil = Date.now() + GROQ_RATE_LIMIT_COOLDOWN_MS;
} else {
// 502, timeout, or other transient error — cooldown briefly
groqUnavailableUntil = Date.now() + TRANSIENT_ERROR_COOLDOWN_MS;
}
log.warn({ error }, "Groq Llama Prompt Guard moderation failed");
}
}
}
const finalHits = Array.from(hits);
// Populate all cache tiers so the same text never triggers another API call // Populate all cache tiers so the same text never triggers another API call
// within the TTL window. // within the TTL window.
setCachedBadwords(cacheKey, finalHits); setCachedBadwords(cacheKey, finalHits);
await upsertCachedText( await upsertCachedText(
cacheKey, cacheKey,
finalHits, finalHits,
sourceUsed, "primary_ai",
Date.now() + DB_CACHE_TTL_MS, Date.now() + DB_CACHE_TTL_MS,
); );
services/discord-gateway/src/modules/ai-moderation/textCacheStore.ts
+2 -2
View File
@@ -7,7 +7,7 @@ const logger = createChildLogger("text-cache-store");
export interface TextCacheEntry { export interface TextCacheEntry {
text: string; text: string;
flags: string[]; flags: string[];
source: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm"; source: "local" | "primary_ai" | "vision_llm";
analyzed_at: number; analyzed_at: number;
expires_at: number; expires_at: number;
hit_count: number; hit_count: number;
@@ -53,7 +53,7 @@ export async function getCachedText(
export async function upsertCachedText( export async function upsertCachedText(
text: string, text: string,
flags: string[], flags: string[],
source: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm", source: "local" | "primary_ai" | "vision_llm",
expiresAt: number, expiresAt: number,
): Promise<void> { ): Promise<void> {
const now = Date.now(); const now = Date.now();
services/discord-gateway/src/shared/database/schema.ts
+2 -2
View File
@@ -369,9 +369,9 @@ export const pgTextAnalysisCacheTable = pgTable(
text: pgText("text").primaryKey(), text: pgText("text").primaryKey(),
/** JSON array of moderation flags detected for this text (e.g. ["vulgar_language","harassment"]). */ /** JSON array of moderation flags detected for this text (e.g. ["vulgar_language","harassment"]). */
flags: pgText("flags").notNull().default("[]"), flags: pgText("flags").notNull().default("[]"),
/** Which source produced this result: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm". */ /** Which source produced this result: "local" | "primary_ai" | "vision_llm". */
source: pgText("source", { source: pgText("source", {
enum: ["local", "nvidia", "primary_ai", "groq", "vision_llm"], enum: ["local", "primary_ai", "vision_llm"],
}) })
.notNull() .notNull()
.default("local"), .default("local"),
src/database/schema.ts
+2 -2
View File
@@ -369,9 +369,9 @@ export const pgTextAnalysisCacheTable = pgTable(
text: pgText("text").primaryKey(), text: pgText("text").primaryKey(),
/** JSON array of moderation flags detected for this text (e.g. ["vulgar_language","harassment"]). */ /** JSON array of moderation flags detected for this text (e.g. ["vulgar_language","harassment"]). */
flags: pgText("flags").notNull().default("[]"), flags: pgText("flags").notNull().default("[]"),
/** Which source produced this result: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm". */ /** Which source produced this result: "local" | "primary_ai" | "vision_llm". */
source: pgText("source", { source: pgText("source", {
enum: ["local", "nvidia", "primary_ai", "groq", "vision_llm"], enum: ["local", "primary_ai", "vision_llm"],
}) })
.notNull() .notNull()
.default("local"), .default("local"),
src/moderation/indonesianTextNormalizer.ts
+34 -281
View File
@@ -1,42 +1,12 @@
import axios from "axios";
import OpenAI from "openai"; import OpenAI from "openai";
import { config } from "../config.js"; import { config } from "../config.js";
import { createChildLogger } from "../logger.js"; import { createChildLogger } from "../logger.js";
import { retryWithBackoff } from "../retry.js";
import { getCachedText, upsertCachedText } from "./textCacheStore.js"; import { getCachedText, upsertCachedText } from "./textCacheStore.js";
const log = createChildLogger("indonesianTextNormalizer"); const log = createChildLogger("indonesianTextNormalizer");
const CUSTOM_EMOJI_PATTERN = /<a?:([a-zA-Z0-9_]+):(\d+)>/g; const CUSTOM_EMOJI_PATTERN = /<a?:([a-zA-Z0-9_]+):(\d+)>/g;
/** NVIDIA content safety categories that map to offensive/badword content. */
const NVIDIA_BAD_CATEGORIES = new Set([
"hate",
"harassment",
"sexual",
"violence",
"self-harm",
"illicit",
"profanity",
"vulgar",
"insult",
]);
/**
* Map NVIDIA Nemotron category labels to Indonesian badword-style labels.
*/
const CATEGORY_TO_BADWORD_LABEL: Record<string, string> = {
hate: "hate_speech",
harassment: "harassment",
sexual: "sexual_content",
violence: "violence",
"self-harm": "self_harm",
illicit: "illegal_content",
profanity: "vulgar_language",
vulgar: "vulgar_language",
insult: "harassment",
};
const VALID_PRIMARY_AI_FLAGS = new Set([ const VALID_PRIMARY_AI_FLAGS = new Set([
"spam", "spam",
"hate_speech", "hate_speech",
@@ -74,10 +44,6 @@ const BADWORD_CACHE_TTL_MS = 10 * 60 * 1000;
*/ */
const DB_CACHE_TTL_MS = 24 * 60 * 60 * 1000; const DB_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
const NEMOTRON_RATE_LIMIT_COOLDOWN_MS = 0;
const PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS = 0;
const GROQ_RATE_LIMIT_COOLDOWN_MS = 0;
interface BadwordCacheEntry { interface BadwordCacheEntry {
value: string[]; value: string[];
expiresAt: number; expiresAt: number;
@@ -85,9 +51,6 @@ interface BadwordCacheEntry {
const badwordCache = new Map<string, BadwordCacheEntry>(); const badwordCache = new Map<string, BadwordCacheEntry>();
const inFlightBadwordLookups = new Map<string, Promise<string[]>>(); const inFlightBadwordLookups = new Map<string, Promise<string[]>>();
let nemotronUnavailableUntil = 0;
let primaryAiUnavailableUntil = 0;
let groqUnavailableUntil = 0;
let primaryModerationClient: OpenAI | null = null; let primaryModerationClient: OpenAI | null = null;
export interface ModerationTextEvidence { export interface ModerationTextEvidence {
@@ -99,7 +62,7 @@ export interface ModerationTextEvidence {
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Sync helpers (unchanged) // Sync helpers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
export function normalizeDiscordCustomEmoji(text: string): { export function normalizeDiscordCustomEmoji(text: string): {
@@ -118,10 +81,6 @@ export function normalizeDiscordCustomEmoji(text: string): {
return { text: normalized, emojiNames }; return { text: normalized, emojiNames };
} }
// Local badword detection removed (lines 121-198).
// All detection now goes through the API pipeline (NVIDIA → Primary AI → Groq)
// to eliminate false positives from substring matching and hardcoded whitelists.
function normalizeBadwordCacheKey(text: string): string { function normalizeBadwordCacheKey(text: string): string {
return text.trim().replace(/\s+/g, " ").toLowerCase(); return text.trim().replace(/\s+/g, " ").toLowerCase();
} }
@@ -190,7 +149,7 @@ function normalizePrimaryAiFlag(value: string): string | null {
return lower; return lower;
} }
return CATEGORY_TO_BADWORD_LABEL[lower] ?? null; return null;
} }
function extractFlagsFromPrimaryAiContent(content: string): string[] { function extractFlagsFromPrimaryAiContent(content: string): string[] {
@@ -236,13 +195,6 @@ function extractFlagsFromPrimaryAiContent(content: string): string[] {
} }
} }
for (const category of Object.keys(CATEGORY_TO_BADWORD_LABEL)) {
if (lowerContent.includes(category)) {
const mapped = CATEGORY_TO_BADWORD_LABEL[category];
if (mapped) flags.add(mapped);
}
}
return Array.from(flags); return Array.from(flags);
} }
@@ -252,36 +204,25 @@ async function callPrimaryAiModeration(text: string): Promise<string[]> {
return []; return [];
} }
const completion = await retryWithBackoff( const completion = await client.chat.completions.create({
async () => { model: config.AI_LLM_MODEL,
return client.chat.completions.create({ messages: [
model: config.AI_LLM_MODEL, {
messages: [ role: "user",
{ content:
role: "user", "Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " +
content: 'Balas hanya JSON object dengan format {"flags":[...]} dan gunakan hanya flag valid ini: ' +
"Deteksi kata kasar / pelanggaran ringan dari teks Indonesia berikut. " + Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") +
'Balas hanya JSON object dengan format {"flags":[...]} dan gunakan hanya flag valid ini: ' + ". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " +
Array.from(VALID_PRIMARY_AI_FLAGS).join(", ") + text,
". Jika tidak ada pelanggaran, flags harus array kosong. Teks: " + },
text, ],
}, temperature: 0.1,
], top_p: 0.9,
temperature: 0.1, max_tokens: 200,
top_p: 0.9, stream: false,
max_tokens: 200, response_format: { type: "json_object" },
stream: false, } as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming);
response_format: { type: "json_object" },
} as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming);
},
{
retries: 0,
minTimeout: 0,
maxTimeout: 0,
factor: 2,
logger: log,
},
);
const content = completion.choices[0]?.message?.content?.trim(); const content = completion.choices[0]?.message?.content?.trim();
if (!content) { if (!content) {
@@ -292,141 +233,18 @@ async function callPrimaryAiModeration(text: string): Promise<string[]> {
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Groq Llama Prompt Guard Moderation API (Fallback) // Two-tier cache + Primary AI pipeline
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
/** /**
* Call Groq Llama Prompt Guard 2-86M model for moderation scoring. * Detect badwords in text using a **two-tier cache + Primary AI**:
* Returns a probability score as a string (e.g. "0.9988824725151062").
* Scores above ~0.5 indicate moderation violations.
*/
async function callGrokModeration(text: string): Promise<string[]> {
const apiKey = config.GROQ_API_KEY;
if (!apiKey) {
return [];
}
const response = await axios.post(
config.GROQ_MODERATION_BASE_URL,
{
model: config.GROQ_MODERATION_MODEL,
messages: [{ role: "user", content: text }],
},
{
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json",
"Content-Type": "application/json",
},
timeout: 10_000,
},
);
const scoreStr = response.data?.choices?.[0]?.message?.content?.trim();
if (!scoreStr) {
return [];
}
// Parse the score (Llama Prompt Guard returns a single probability score)
const score = parseFloat(scoreStr);
if (isNaN(score) || score < 0.5) {
return [];
}
// Map score to moderation flags based on severity
const flags: string[] = [];
if (score >= 0.9) {
flags.push("vulgar_language", "harassment");
} else if (score >= 0.7) {
flags.push("vulgar_language");
} else {
flags.push("spam");
}
return flags;
}
// ---------------------------------------------------------------------------
// NVIDIA Nemotron-3 Content Safety API
// ---------------------------------------------------------------------------
/**
* Call NVIDIA Nemotron-3 Content Safety API to detect harmful content.
* Returns categories/flags from the API response.
*/
async function callNemotronContentSafety(text: string): Promise<string[]> {
const apiKey = config.NVIDIA_NEMOTRON_API_KEY;
if (!apiKey) {
return [];
}
const response = await axios.post(
config.NVIDIA_NEMOTRON_BASE_URL,
{
model: config.NVIDIA_NEMOTRON_MODEL,
messages: [{ role: "user", content: text }],
max_tokens: 897,
temperature: 0.2,
top_p: 0.7,
stream: false,
},
{
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json",
},
timeout: 15_000,
},
);
const data = response.data;
const categories: string[] = [];
// Parse the LLM response for category flags
const content = data?.choices?.[0]?.message?.content ?? "";
if (content) {
const lowerContent = content.toLowerCase();
for (const category of NVIDIA_BAD_CATEGORIES) {
if (lowerContent.includes(category)) {
categories.push(CATEGORY_TO_BADWORD_LABEL[category] ?? category);
}
}
}
// Also check for structured response fields
const choice = data?.choices?.[0];
if (choice?.message?.content) {
try {
const parsed = JSON.parse(choice.message.content);
if (parsed.categories && Array.isArray(parsed.categories)) {
for (const cat of parsed.categories) {
if (NVIDIA_BAD_CATEGORIES.has(cat.name ?? cat)) {
categories.push(CATEGORY_TO_BADWORD_LABEL[cat.name ?? cat] ?? cat);
}
}
}
} catch {
// Not JSON — already handled via text search above
}
}
return Array.from(new Set(categories));
}
// ---------------------------------------------------------------------------
// Three-tier cache pipeline
// ---------------------------------------------------------------------------
/**
* Detect badwords in text using a **two-tier cache + API pipeline**:
* *
* 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path, * 1. **In-memory cache** (BADWORD_CACHE_TTL_MS, 10 min) — fastest path,
* keyed by the full normalized text string. * keyed by the full normalized text string.
* 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted * 2. **DB cache** (DB_CACHE_TTL_MS, 24 h) — same full-text key, persisted
* across restarts. Uses the FULL normalized text (not per-word) because * across restarts. Uses the FULL normalized text (not per-word) because
* context matters: "kau" alone is clean, but "awas kau" can be a threat. * context matters: "kau" alone is clean, but "awas kau" can be a threat.
* 3. **API pipeline** (NVIDIA → Primary AI → Groq) * 3. **Primary AI** (AI_LLM endpoint) — only runs when both cache layers miss.
* only runs when both cache layers miss.
* *
* No local hardcoded badword list — all detection goes through AI APIs * No local hardcoded badword list — all detection goes through AI APIs
* to eliminate false positives from substring matching. * to eliminate false positives from substring matching.
@@ -457,89 +275,24 @@ export async function detectIndonesianBadwords(
return flags; return flags;
} }
// ── Tier 3: API pipeline ── // ── Tier 3: Primary AI only ──
let finalHits: string[] = [];
const hits = new Set<string>(); try {
let sourceUsed: "nvidia" | "primary_ai" | "groq" = "primary_ai"; finalHits = await callPrimaryAiModeration(text);
} catch (error) {
// 3a. Try NVIDIA API if key is configured and not rate limited. log.warn(
const apiKey = config.NVIDIA_NEMOTRON_API_KEY; { error: error instanceof Error ? error.message : String(error) },
if (apiKey && Date.now() >= nemotronUnavailableUntil) { "Primary AI badword detection failed",
try { );
const apiCategories = await callNemotronContentSafety(text);
for (const hit of apiCategories) {
hits.add(hit);
}
if (apiCategories.length > 0) sourceUsed = "nvidia";
} catch (error) {
const status = axios.isAxiosError(error)
? error.response?.status
: null;
if (status === 429) {
nemotronUnavailableUntil =
Date.now() + NEMOTRON_RATE_LIMIT_COOLDOWN_MS;
}
log.warn(
{ error },
"NVIDIA Nemotron API call failed, falling back to primary AI",
);
}
} }
// 3b. Try the main AI model next.
if (hits.size === 0 && Date.now() >= primaryAiUnavailableUntil) {
try {
const primaryHits = await callPrimaryAiModeration(text);
for (const hit of primaryHits) {
hits.add(hit);
}
if (primaryHits.length > 0) sourceUsed = "primary_ai";
} catch (error) {
const status = axios.isAxiosError(error)
? error.response?.status
: null;
if (status === 429) {
primaryAiUnavailableUntil =
Date.now() + PRIMARY_AI_RATE_LIMIT_COOLDOWN_MS;
}
log.warn(
{ error },
"Primary AI badword detection failed, falling back to Groq",
);
}
}
// 3c. Try Groq Llama Prompt Guard as final API fallback.
if (hits.size === 0 && Date.now() >= groqUnavailableUntil) {
const groqKey = config.GROQ_API_KEY;
if (groqKey) {
try {
const groqHits = await callGrokModeration(text);
for (const hit of groqHits) {
hits.add(hit);
}
if (groqHits.length > 0) sourceUsed = "groq";
} catch (error) {
const status = axios.isAxiosError(error)
? error.response?.status
: null;
if (status === 429) {
groqUnavailableUntil = Date.now() + GROQ_RATE_LIMIT_COOLDOWN_MS;
}
log.warn({ error }, "Groq Llama Prompt Guard moderation failed");
}
}
}
const finalHits = Array.from(hits);
// Populate all cache tiers so the same text never triggers another API call // Populate all cache tiers so the same text never triggers another API call
// within the TTL window. // within the TTL window.
setCachedBadwords(cacheKey, finalHits); setCachedBadwords(cacheKey, finalHits);
await upsertCachedText( await upsertCachedText(
cacheKey, cacheKey,
finalHits, finalHits,
sourceUsed, "primary_ai",
Date.now() + DB_CACHE_TTL_MS, Date.now() + DB_CACHE_TTL_MS,
); );
src/moderation/textCacheStore.ts
+2 -2
View File
@@ -7,7 +7,7 @@ const logger = createChildLogger("text-cache-store");
export interface TextCacheEntry { export interface TextCacheEntry {
text: string; text: string;
flags: string[]; flags: string[];
source: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm"; source: "local" | "primary_ai" | "vision_llm";
analyzed_at: number; analyzed_at: number;
expires_at: number; expires_at: number;
hit_count: number; hit_count: number;
@@ -53,7 +53,7 @@ export async function getCachedText(
export async function upsertCachedText( export async function upsertCachedText(
text: string, text: string,
flags: string[], flags: string[],
source: "local" | "nvidia" | "primary_ai" | "groq" | "vision_llm", source: "local" | "primary_ai" | "vision_llm",
expiresAt: number, expiresAt: number,
): Promise<void> { ): Promise<void> {
const now = Date.now(); const now = Date.now();