fix: use AI for global username check instead of keyword list

Replace static OFFENSIVE_USERNAME_KEYWORDS substring matching with a
lightweight LLM call that evaluates whether a global username violates
server rules (gambling, scam, NSFW, SARA, etc). Fail-open design:
if the LLM call fails/times out, the nickname reset still completes.
This commit is contained in:
asepharyana
2026-08-27 13:20:20 +07:00
parent 1590479f58
commit 4991164591
@@ -10,6 +10,7 @@ import {
} from "./autoDeleteEligibility.js"; } from "./autoDeleteEligibility.js";
import { logDeletionToChannel } from "./autoDeleteLogger.js"; import { logDeletionToChannel } from "./autoDeleteLogger.js";
import { sendDeletionNotification } from "./autoDeleteNotify.js"; import { sendDeletionNotification } from "./autoDeleteNotify.js";
import { llmChat } from "./llmClient.js";
import { verdictToActionFields } from "./verdictToActionFields.js"; import { verdictToActionFields } from "./verdictToActionFields.js";
const logger = createChildLogger("auto-delete-manager"); const logger = createChildLogger("auto-delete-manager");
@@ -102,7 +103,7 @@ export async function resetOffensiveNickname(
// having an offensive global username. // having an offensive global username.
const refreshedMember = await guild.members.fetch(userId); const refreshedMember = await guild.members.fetch(userId);
const globalUsername = refreshedMember.user?.username ?? ""; const globalUsername = refreshedMember.user?.username ?? "";
if (isGlobalUsernameOffensive(globalUsername)) { if (await isGlobalUsernameOffensiveAI(globalUsername)) {
const randomName = generateRandomUsername(); const randomName = generateRandomUsername();
await refreshedMember.setNickname( await refreshedMember.setNickname(
randomName, randomName,
@@ -139,42 +140,58 @@ export async function resetOffensiveNickname(
} }
} }
// ─── Offensive Username Detection (global username check) ───────────── // ─── AI-Based Global Username Check ──────────────────────────────────
/** Keywords that indicate a gambling/scam/spam username (case-insensitive). */
const OFFENSIVE_USERNAME_KEYWORDS = [
"bandar",
"togel",
"slot",
"casino",
"judi",
"poker",
"bet",
"jackpot",
"pragmatic",
"deposit",
"withdraw",
"agen",
"bo",
"bocoran",
"rtp",
"maxwin",
"scatter",
"gacor",
"apk",
"situs",
"link",
"klik",
"daftar",
];
/** /**
* Check if a global username contains known gambling/scam keywords. * Ask the AI moderation LLM whether a global username is offensive
* Uses simple substring matching — fast and deterministic, no LLM call. * (gambling, scam, spam, NSFW, etc.). Returns `true` if the LLM
* judges the username as violating server rules.
*
* Fail-open: if the LLM call fails or times out, returns `false`
* so the nickname reset still completes — we don't want a broken LLM
* to block enforcement.
*/ */
function isGlobalUsernameOffensive(username: string): boolean { async function isGlobalUsernameOffensiveAI(username: string): Promise<boolean> {
const lower = username.toLowerCase(); try {
return OFFENSIVE_USERNAME_KEYWORDS.some((kw) => lower.includes(kw)); const completion = await llmChat({
messages: [
{
role: "system",
content:
'Kamu adalah moderator Discord. Tentukan apakah username berikut melanggar aturan server (judi, togel, scam, spam, NSFW, SARA, atau ofensif). Jawab HANYA dengan JSON: {"offensive": true} atau {"offensive": false}. Jangan penjelasan tambahan.',
},
{
role: "user",
content: `Username: "${username}"`,
},
],
max_tokens: 50,
temperature: 0,
stream: false,
timeout: 10_000,
});
const content = completion?.choices?.[0]?.message?.content?.trim() ?? "";
// Parse JSON response — handle both strict JSON and markdown-wrapped
const jsonMatch = content.match(
/\{[^}]*"offensive"\s*:\s*(true|false)[^}]*\}/,
);
if (jsonMatch) {
const parsed = JSON.parse(jsonMatch[0]) as { offensive: boolean };
return parsed.offensive === true;
}
// Fallback: if response contains "true" anywhere, treat as offensive
return content.toLowerCase().includes("true");
} catch (error) {
logger.warn(
{
username,
error: error instanceof Error ? error.message : String(error),
},
"AI username check failed — failing open (not offensive)",
);
return false;
}
} }
/** /**