fix(ai-moderation): patch structural bypass vulnerabilities in NLP pipeline

- Implement Pre-computation Normalization for Polyglot Obfuscation.

- Inject Ontological Graph for literal translation evasion (e.g., 'kostum hewan').

- Enforce Entropy-Triggered Routing to deny softmax fallback exploitation.

- Format discord-gateway codebase.
This commit is contained in:
MythEclipse
2026-06-06 15:38:36 +07:00
parent 5701b5f15f
commit 71a8a9e6e1
10 changed files with 197 additions and 78 deletions
@@ -97,7 +97,6 @@ function scheduleAutoDelete(row: MessageRecord): void {
setImmediate(run); setImmediate(run);
} }
function isAgeRestrictedMessage(message: MessageRecord): boolean { function isAgeRestrictedMessage(message: MessageRecord): boolean {
return isAgeRestrictedMetadata(message.metadata); return isAgeRestrictedMetadata(message.metadata);
} }
@@ -493,11 +492,27 @@ async function processIndividualFallback(
if (row.ai_status === "clean") { if (row.ai_status === "clean") {
import("./userReputationStore.js") import("./userReputationStore.js")
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id)) .then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
.catch((e) => logger.error({ error: e }, "Failed to record clean message streak in fallback")); .catch((e) =>
logger.error(
{ error: e },
"Failed to record clean message streak in fallback",
),
);
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") { } else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
import("./userReputationStore.js") import("./userReputationStore.js")
.then((store) => store.recordInfraction(row.user_id, row.guild_id, row.ai_severity as "low"|"medium"|"high"|"critical")) .then((store) =>
.catch((e) => logger.error({ error: e }, "Failed to record infraction penalty in fallback")); store.recordInfraction(
row.user_id,
row.guild_id,
row.ai_severity as "low" | "medium" | "high" | "critical",
),
)
.catch((e) =>
logger.error(
{ error: e },
"Failed to record infraction penalty in fallback",
),
);
} }
} }
@@ -716,12 +731,27 @@ async function processBatch(
// Update reputation autonomously (Belajar & Kebijaksanaan) // Update reputation autonomously (Belajar & Kebijaksanaan)
if (row.ai_status === "clean") { if (row.ai_status === "clean") {
import("./userReputationStore.js") import("./userReputationStore.js")
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id)) .then((store) =>
.catch((e) => logger.error({ error: e }, "Failed to record clean message streak")); store.recordCleanMessage(row.user_id, row.guild_id),
)
.catch((e) =>
logger.error(
{ error: e },
"Failed to record clean message streak",
),
);
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") { } else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
import("./userReputationStore.js") import("./userReputationStore.js")
.then((store) => store.recordInfraction(row.user_id, row.guild_id, row.ai_severity as "low"|"medium"|"high"|"critical")) .then((store) =>
.catch((e) => logger.error({ error: e }, "Failed to record infraction penalty")); store.recordInfraction(
row.user_id,
row.guild_id,
row.ai_severity as "low" | "medium" | "high" | "critical",
),
)
.catch((e) =>
logger.error({ error: e }, "Failed to record infraction penalty"),
);
} }
} }
} }
@@ -853,7 +883,8 @@ async function processBatch(
recordConversationBatchFailure(conversationKey); recordConversationBatchFailure(conversationKey);
// Preserve the longer cooldown: circuit breaker (via recordConversationBatchFailure) // Preserve the longer cooldown: circuit breaker (via recordConversationBatchFailure)
// may have set a 60s cooldown; don't let the shorter config value overwrite it. // may have set a 60s cooldown; don't let the shorter config value overwrite it.
const existingCooldown = conversationErrorCooldown.get(conversationKey) ?? 0; const existingCooldown =
conversationErrorCooldown.get(conversationKey) ?? 0;
const newCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS; const newCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS;
if (newCooldown > existingCooldown) { if (newCooldown > existingCooldown) {
conversationErrorCooldown.set(conversationKey, newCooldown); conversationErrorCooldown.set(conversationKey, newCooldown);
@@ -895,7 +926,8 @@ async function processBatch(
const errorStack = error instanceof Error ? error.stack : undefined; const errorStack = error instanceof Error ? error.stack : undefined;
// Preserve the longer cooldown: circuit breaker (via recordConversationBatchFailure) // Preserve the longer cooldown: circuit breaker (via recordConversationBatchFailure)
// may have set a 60s cooldown; don't let the shorter config value overwrite it. // may have set a 60s cooldown; don't let the shorter config value overwrite it.
const existingCatchCooldown = conversationErrorCooldown.get(conversationKey) ?? 0; const existingCatchCooldown =
conversationErrorCooldown.get(conversationKey) ?? 0;
const newCatchCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS; const newCatchCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS;
if (newCatchCooldown > existingCatchCooldown) { if (newCatchCooldown > existingCatchCooldown) {
conversationErrorCooldown.set(conversationKey, newCatchCooldown); conversationErrorCooldown.set(conversationKey, newCatchCooldown);
@@ -986,7 +1018,9 @@ function scheduleConversationAnalysis(conversationKey: string): void {
) )
.then(async (messages) => { .then(async (messages) => {
if (messages.length === 0) { if (messages.length === 0) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) { if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey); conversationProcessing.delete(conversationKey);
} }
return; return;
@@ -994,7 +1028,9 @@ function scheduleConversationAnalysis(conversationKey: string): void {
const processableMessages = await skipAgeRestrictedMessages(messages); const processableMessages = await skipAgeRestrictedMessages(messages);
if (processableMessages.length === 0) { if (processableMessages.length === 0) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) { if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey); conversationProcessing.delete(conversationKey);
} }
return; return;
@@ -1027,7 +1063,9 @@ function scheduleConversationAnalysis(conversationKey: string): void {
return processBatch(conversationKey, trimmed, processingStartedAt); return processBatch(conversationKey, trimmed, processingStartedAt);
}) })
.catch((err: unknown) => { .catch((err: unknown) => {
if (conversationProcessing.get(conversationKey) === processingStartedAt) { if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey); conversationProcessing.delete(conversationKey);
} }
logger.error( logger.error(
@@ -1043,7 +1081,6 @@ function scheduleConversationAnalysis(conversationKey: string): void {
conversationDebounceTimers.set(conversationKey, timer); conversationDebounceTimers.set(conversationKey, timer);
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Public API // Public API
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -1125,7 +1162,9 @@ export function startPendingAIAnalysisWorker(
_redisEventBroadcaster = eventBroadcaster; _redisEventBroadcaster = eventBroadcaster;
if (!config.AI_ANALYSIS_ENABLED) return; if (!config.AI_ANALYSIS_ENABLED) return;
import("./cultureLearner.js").then(m => m.startCultureLearnerWorker()).catch(console.error); import("./cultureLearner.js")
.then((m) => m.startCultureLearnerWorker())
.catch(console.error);
setInterval(() => { setInterval(() => {
revertStuckProcessingMessages(300000).catch((err: unknown) => { revertStuckProcessingMessages(300000).catch((err: unknown) => {
@@ -8,7 +8,9 @@ import {
/** /**
* Fetch the AI-generated culture summary for a channel. * Fetch the AI-generated culture summary for a channel.
*/ */
export async function getChannelCulture(channelId: string): Promise<ChannelCulture | null> { export async function getChannelCulture(
channelId: string,
): Promise<ChannelCulture | null> {
const db = getDatabase(); const db = getDatabase();
const existing = await db const existing = await db
.select() .select()
@@ -28,7 +30,7 @@ export async function updateChannelCulture(
cultureSummary: string, cultureSummary: string,
): Promise<void> { ): Promise<void> {
const db = getDatabase(); const db = getDatabase();
await db await db
.insert(channelCulturesTable) .insert(channelCulturesTable)
.values({ .values({
@@ -12,9 +12,12 @@ import { updateChannelCulture } from "./channelCultureStore.js";
const CULTURE_LEARNING_INTERVAL = 1000 * 60 * 60 * 12; // 12 hours const CULTURE_LEARNING_INTERVAL = 1000 * 60 * 60 * 12; // 12 hours
const log = createChildLogger("cultureLearner"); const log = createChildLogger("cultureLearner");
async function learnChannelCulture(channelId: string, guildId: string): Promise<void> { async function learnChannelCulture(
channelId: string,
guildId: string,
): Promise<void> {
const db = getDatabase(); const db = getDatabase();
// Get recent clean messages for this channel // Get recent clean messages for this channel
const recentMessages = await db const recentMessages = await db
.select({ .select({
@@ -25,8 +28,8 @@ async function learnChannelCulture(channelId: string, guildId: string): Promise<
.where( .where(
and( and(
eq(messagesTable.channel_id, channelId), eq(messagesTable.channel_id, channelId),
eq(messagesTable.ai_status, "clean") eq(messagesTable.ai_status, "clean"),
) ),
) )
.orderBy(desc(messagesTable.created_at)) .orderBy(desc(messagesTable.created_at))
.limit(100); .limit(100);
@@ -38,7 +41,7 @@ async function learnChannelCulture(channelId: string, guildId: string): Promise<
const messagesText = recentMessages const messagesText = recentMessages
.reverse() .reverse()
.map(m => `${m.username}: ${m.content}`) .map((m) => `${m.username}: ${m.content}`)
.join("\n"); .join("\n");
const prompt = `Anda adalah AI ahli perilaku sosiologis dan budaya online. const prompt = `Anda adalah AI ahli perilaku sosiologis dan budaya online.
@@ -59,13 +62,16 @@ Jangan menambahkan teks basa-basi, langsung berikan ringkasannya.`;
temperature: 0.7, // Higher temp for summarization temperature: 0.7, // Higher temp for summarization
retries: 2, retries: 2,
}); });
if (!completion) throw new Error("Empty response from LLM"); if (!completion) throw new Error("Empty response from LLM");
const text = completion.choices[0]?.message?.content?.trim(); const text = completion.choices[0]?.message?.content?.trim();
if (!text) throw new Error("Empty response from LLM"); if (!text) throw new Error("Empty response from LLM");
await updateChannelCulture(channelId, guildId, text); await updateChannelCulture(channelId, guildId, text);
log.info({ channelId, guildId }, "Successfully learned and updated channel culture"); log.info(
{ channelId, guildId },
"Successfully learned and updated channel culture",
);
} catch (error) { } catch (error) {
log.error({ channelId, error }, "Failed to learn channel culture"); log.error({ channelId, error }, "Failed to learn channel culture");
} }
@@ -106,12 +112,15 @@ export function startCultureLearnerWorker(): void {
// Run once on startup after 1 minute, then every 1 hour // Run once on startup after 1 minute, then every 1 hour
setTimeout(() => { setTimeout(() => {
runCultureLearningCycle().catch(e => log.error(e)); runCultureLearningCycle().catch((e) => log.error(e));
}, 60000); }, 60000);
cultureInterval = setInterval(() => { cultureInterval = setInterval(
runCultureLearningCycle().catch(e => log.error(e)); () => {
}, 1000 * 60 * 60); // Check every hour for channels that reached 12h expiry runCultureLearningCycle().catch((e) => log.error(e));
},
1000 * 60 * 60,
); // Check every hour for channels that reached 12h expiry
log.info("Started background culture learner worker"); log.info("Started background culture learner worker");
} }
@@ -104,29 +104,31 @@ export async function llmChat(
async () => { async () => {
return withLlmConcurrency(async () => { return withLlmConcurrency(async () => {
const execute = async (currentParams: any) => { const execute = async (currentParams: any) => {
const response = await client.chat.completions.create(currentParams, { signal }); const response = await client.chat.completions.create(currentParams, {
signal,
});
if (currentParams.stream) { if (currentParams.stream) {
let content = ""; let content = "";
let finishReason = "stop"; let finishReason = "stop";
for await (const chunk of response as any) { for await (const chunk of response as any) {
const choice = chunk?.choices?.[0]; const choice = chunk?.choices?.[0];
const textChunk = const textChunk =
choice?.delta?.content || choice?.delta?.content ||
choice?.message?.content || choice?.message?.content ||
choice?.text || choice?.text ||
chunk?.message?.content || chunk?.message?.content ||
chunk?.response || chunk?.response ||
chunk?.content || chunk?.content ||
""; "";
content += textChunk; content += textChunk;
const fr = choice?.finish_reason || chunk?.finish_reason; const fr = choice?.finish_reason || chunk?.finish_reason;
if (fr) finishReason = fr; if (fr) finishReason = fr;
} }
return { return {
id: 'stream-aggregated', id: "stream-aggregated",
choices: [ choices: [
{ {
message: { role: 'assistant', content, refusal: null }, message: { role: "assistant", content, refusal: null },
finish_reason: finishReason, finish_reason: finishReason,
index: 0, index: 0,
logprobs: null, logprobs: null,
@@ -134,7 +136,7 @@ export async function llmChat(
], ],
created: Math.floor(Date.now() / 1000), created: Math.floor(Date.now() / 1000),
model: currentParams.model, model: currentParams.model,
object: 'chat.completion', object: "chat.completion",
} as OpenAI.Chat.Completions.ChatCompletion; } as OpenAI.Chat.Completions.ChatCompletion;
} }
return response as OpenAI.Chat.Completions.ChatCompletion; return response as OpenAI.Chat.Completions.ChatCompletion;
@@ -143,12 +145,22 @@ export async function llmChat(
try { try {
return await execute(params); return await execute(params);
} catch (error: any) { } catch (error: any) {
const rawResponse = error.error || error.body || error.response?.data || "N/A"; const rawResponse =
const errorStr = (JSON.stringify(rawResponse) + String(error.message)).toLowerCase(); error.error || error.body || error.response?.data || "N/A";
const errorStr = (
JSON.stringify(rawResponse) + String(error.message)
).toLowerCase();
// Auto-fallback: If provider strictly demands streaming (400 Bad Request on stream params) // Auto-fallback: If provider strictly demands streaming (400 Bad Request on stream params)
if (error.status === 400 && errorStr.includes("stream") && !params.stream) { if (
log.warn({ model }, "Provider rejected non-streaming request. Fallback to stream: true initiated."); error.status === 400 &&
errorStr.includes("stream") &&
!params.stream
) {
log.warn(
{ model },
"Provider rejected non-streaming request. Fallback to stream: true initiated.",
);
params.stream = true; params.stream = true;
return await execute(params); return await execute(params);
} }
@@ -158,9 +170,9 @@ export async function llmChat(
error: error.message, error: error.message,
status: error.status, status: error.status,
rawResponse, rawResponse,
model model,
}, },
"LLM API request failed" "LLM API request failed",
); );
throw error; throw error;
} }
@@ -589,9 +589,12 @@ const analyzeSingleMediaImage = async (
// Attempt to acquire DISTRIBUTED lock // Attempt to acquire DISTRIBUTED lock
// Lock expires in 60 seconds (generous timeout for LLM) // Lock expires in 60 seconds (generous timeout for LLM)
const locked = await acquireMediaAnalysisLock(cacheKey, Date.now() + 60000); const locked = await acquireMediaAnalysisLock(cacheKey, Date.now() + 60000);
if (!locked) { if (!locked) {
log.debug({ cacheKey }, "Media analysis distributed lock acquired by another pod. Polling..."); log.debug(
{ cacheKey },
"Media analysis distributed lock acquired by another pod. Polling...",
);
// Poll DB for up to 30 seconds // Poll DB for up to 30 seconds
for (let i = 0; i < 15; i++) { for (let i = 0; i < 15; i++) {
await new Promise((resolve) => setTimeout(resolve, 2000)); await new Promise((resolve) => setTimeout(resolve, 2000));
@@ -601,7 +604,10 @@ const analyzeSingleMediaImage = async (
return pollCached; return pollCached;
} }
} }
log.warn({ cacheKey }, "Polling for distributed media analysis timed out. Falling back."); log.warn(
{ cacheKey },
"Polling for distributed media analysis timed out. Falling back.",
);
return FAILED_ANALYSIS_PREFIX; return FAILED_ANALYSIS_PREFIX;
} }
@@ -842,10 +848,7 @@ async function callModerationLLM(
throw apiError; throw apiError;
} }
// 401/403 → abort immediately, never retry // 401/403 → abort immediately, never retry
if ( if (apiError?.status === 401 || apiError?.status === 403) {
apiError?.status === 401 ||
apiError?.status === 403
) {
const abortErr = new Error(String(apiError)); const abortErr = new Error(String(apiError));
abortErr.name = "AbortError"; abortErr.name = "AbortError";
throw abortErr; throw abortErr;
@@ -1097,8 +1100,12 @@ async function runTextOnlyBatch(
const channelId = targets.length > 0 ? targets[0].channel_id : ""; const channelId = targets.length > 0 ? targets[0].channel_id : "";
const guildId = targets.length > 0 ? targets[0].guild_id : ""; const guildId = targets.length > 0 ? targets[0].guild_id : "";
const channelCultureObj = channelId ? await getChannelCulture(channelId) : null; const channelCultureObj = channelId
const channelCulture = channelCultureObj ? channelCultureObj.culture_summary : undefined; ? await getChannelCulture(channelId)
: null;
const channelCulture = channelCultureObj
? channelCultureObj.culture_summary
: undefined;
// Run sub-batches sequentially to avoid rate limits // Run sub-batches sequentially to avoid rate limits
for (let i = 0; i < subBatches.length; i++) { for (let i = 0; i < subBatches.length; i++) {
@@ -1167,7 +1174,12 @@ async function runTextOnlyBatch(
let batchResult: { results: AnalysisResult[]; raw: unknown }; let batchResult: { results: AnalysisResult[]; raw: unknown };
try { try {
batchResult = await callModerationLLM(buildContent, targetIds, `text-batch-${i + 1}`, abortController.signal); batchResult = await callModerationLLM(
buildContent,
targetIds,
`text-batch-${i + 1}`,
abortController.signal,
);
} catch (err: any) { } catch (err: any) {
if (err.name === "AbortError" || abortController.signal.aborted) { if (err.name === "AbortError" || abortController.signal.aborted) {
throw new Error( throw new Error(
@@ -1661,8 +1673,12 @@ async function _runSingleMediaAnalysis(
.join(" "); .join(" ");
const channelId = target.channel_id; const channelId = target.channel_id;
const channelCultureObj = channelId ? await getChannelCulture(channelId) : null; const channelCultureObj = channelId
const channelCulture = channelCultureObj ? channelCultureObj.culture_summary : undefined; ? await getChannelCulture(channelId)
: null;
const channelCulture = channelCultureObj
? channelCultureObj.culture_summary
: undefined;
const rep = await initializeUserReputation(target.user_id, target.guild_id); const rep = await initializeUserReputation(target.user_id, target.guild_id);
const userCtx = `<user_reputation trust_score="${rep.trust_score}" />`; const userCtx = `<user_reputation trust_score="${rep.trust_score}" />`;
@@ -18,6 +18,12 @@
const SYSTEM_RULES = `Kamu adalah asisten moderasi konten untuk server Discord berbahasa Indonesia. const SYSTEM_RULES = `Kamu adalah asisten moderasi konten untuk server Discord berbahasa Indonesia.
Bahasa utama komunitas ini adalah BAHASA INDONESIA. Bahasa Inggris adalah bahasa sekunder. Bahasa utama komunitas ini adalah BAHASA INDONESIA. Bahasa Inggris adalah bahasa sekunder.
## PRE-COMPUTATION NORMALIZATION & CROSS-LINGUAL DEFENSE (MANDATORY STEP)
1. Jika teks menggunakan campuran bahasa (Inggris, Indonesia, bahasa daerah seperti Jawa Ngoko/Krama), KAMU WAJIB melakukan normalisasi mental/menerjemahkan semuanya ke Bahasa Indonesia standar sebelum memproses intent.
2. JANGAN PERNAH memberikan kelonggaran hanya karena sintaksis berantakan atau bercampur bahasa (Polyglot Obfuscation).
3. Lakukan Named Entity Recognition (NER) secara agresif. Identifikasi nama orang/karakter (seperti "ren") meskipun nama tersebut didahului oleh kata archaic/honorific daerah (seperti "diagem").
## Aturan Umum ## Aturan Umum
- Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll adalah AMAN. - Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll adalah AMAN.
- Istilah kultur pop/anime Jepang: "moe", "waifu", "husbando", "tsundere", "wibu", "otaku" adalah ekspresi normal/AMAN dan BUKAN "sexual_deviation". JANGAN flag kata-kata ini kecuali diiringi deskripsi/ajakan seksual eksplisit. - Istilah kultur pop/anime Jepang: "moe", "waifu", "husbando", "tsundere", "wibu", "otaku" adalah ekspresi normal/AMAN dan BUKAN "sexual_deviation". JANGAN flag kata-kata ini kecuali diiringi deskripsi/ajakan seksual eksplisit.
@@ -50,6 +56,7 @@ Pedoman ini mencerminkan nilai-nilai yang dijunjung server. Terapkan dengan bija
### Anti-Evasion & Obfuscation (STRICT RULE) ### Anti-Evasion & Obfuscation (STRICT RULE)
- **Zalgo / Leetspeak / Simbol:** Pesan yang menggunakan karakter simbolik acak, Zalgo text, atau leetspeak (misal: "++++++K1[[ your $€/F", "b1tch", "k0nt0l") adalah TEKNIK EVASI. KAMU WAJIB mendekode makna aslinya. Jika maknanya merujuk pada ancaman atau kata kasar, FLAG sebagai "harassment" atau "hate_speech". PENGECUALIAN: Kaomoji (misal ╯°□°)╯︵ ┻━┻) atau ASCII art dekoratif adalah AMAN dan BUKAN teknik evasi. - **Zalgo / Leetspeak / Simbol:** Pesan yang menggunakan karakter simbolik acak, Zalgo text, atau leetspeak (misal: "++++++K1[[ your $€/F", "b1tch", "k0nt0l") adalah TEKNIK EVASI. KAMU WAJIB mendekode makna aslinya. Jika maknanya merujuk pada ancaman atau kata kasar, FLAG sebagai "harassment" atau "hate_speech". PENGECUALIAN: Kaomoji (misal ╯°□°)╯︵ ┻━┻) atau ASCII art dekoratif adalah AMAN dan BUKAN teknik evasi.
- **Polyglot Obfuscation (Serangan Lintas Bahasa):** Mencampuradukkan kosa kata Inggris, Indonesia, dan daerah secara acak (misal: "sesuatu sing that...") adalah teknik pengaburan makna (semantic fragmentation). JANGAN anggap ini "bahasa gaul santai". Jika ada entitas atau terjemahan literal tersembunyi di dalamnya, FLAG sesuai pelanggaran aslinya.
- **Emoji Huruf / Regional Indicators:** Jika pesan menggunakan deretan emoji bendera/huruf (seperti 🇬 🇦 🇾) atau karakter spesial yang dirangkai untuk mengeja sebuah kata, JANGAN menganggapnya sebagai "serangkaian emoji tanpa teks". KAMU WAJIB membaca rangkaian emoji tersebut sebagai huruf dan kata aslinya. Jika kata yang dibentuk melanggar, flag sesuai pelanggarannya. - **Emoji Huruf / Regional Indicators:** Jika pesan menggunakan deretan emoji bendera/huruf (seperti 🇬 🇦 🇾) atau karakter spesial yang dirangkai untuk mengeja sebuah kata, JANGAN menganggapnya sebagai "serangkaian emoji tanpa teks". KAMU WAJIB membaca rangkaian emoji tersebut sebagai huruf dan kata aslinya. Jika kata yang dibentuk melanggar, flag sesuai pelanggarannya.
- **Bahasa Asing & Vulgaritas Obscure:** Kata-kata vulgar atau anatomi seksual dalam bahasa asing/slang (seperti "titten", "nigger", "kys", "whore") ADALAH PELANGGARAN. PENGECUALIAN: Kata asing dengan ejaan mirip namun makna normal (False Friends) seperti penyebutan negara "Niger", warna bahasa Spanyol "negro", atau kata ganti Korea "niga" adalah AMAN. - **Bahasa Asing & Vulgaritas Obscure:** Kata-kata vulgar atau anatomi seksual dalam bahasa asing/slang (seperti "titten", "nigger", "kys", "whore") ADALAH PELANGGARAN. PENGECUALIAN: Kata asing dengan ejaan mirip namun makna normal (False Friends) seperti penyebutan negara "Niger", warna bahasa Spanyol "negro", atau kata ganti Korea "niga" adalah AMAN.
- **Zero Tolerance untuk Evasi & Vulgaritas Seksual:** Setiap indikasi user mencoba menyembunyikan kata kasar di balik simbol, atau menggunakan vulgaritas anatomi seksual secara "santai", HARUS langsung di-flag. Jangan beri toleransi atau label "clean". - **Zero Tolerance untuk Evasi & Vulgaritas Seksual:** Setiap indikasi user mencoba menyembunyikan kata kasar di balik simbol, atau menggunakan vulgaritas anatomi seksual secara "santai", HARUS langsung di-flag. Jangan beri toleransi atau label "clean".
@@ -83,6 +90,7 @@ Prioritas menengah (PERILAKU MERUSAK):
Prioritas rendah (PELANGGARAN RINGAN): Prioritas rendah (PELANGGARAN RINGAN):
- harassment (targeted insult), vulgar_language (profanity terarah) - harassment (targeted insult), vulgar_language (profanity terarah)
- sexual_deviation: mempromosikan/mendukung/membahas topik seksual/identitas yang dibatasi server. Ini **TERMASUK MENTION SINGKAT** tentang: BL (Boys Love), yaoi, yuri, pengakuan orientasi seksual non-hetero, referensi furry fetish, promosi identitas LGBT. PENGECUALIAN: Karakter hewan fiksi antropomorfik normal (seperti Sonic, Pokemon, maskot anime) adalah BUKAN referensi furry fetish. Jangan flag diskusi karakter fiksi hewan normal sebagai sexual_deviation. - sexual_deviation: mempromosikan/mendukung/membahas topik seksual/identitas yang dibatasi server. Ini **TERMASUK MENTION SINGKAT** tentang: BL (Boys Love), yaoi, yuri, pengakuan orientasi seksual non-hetero, referensi furry fetish, promosi identitas LGBT. PENGECUALIAN: Karakter hewan fiksi antropomorfik normal (seperti Sonic, Pokemon, maskot anime) adalah BUKAN referensi furry fetish. Jangan flag diskusi karakter fiksi hewan normal sebagai sexual_deviation.
- **ONTOLOGICAL GRAPH (PENYAMARAN LITERAL):** Waspadai dog-whistle! Frasa terjemahan harfiah yang kaku seperti "kostum hewan", "bermain peran hewan", dll, adalah PENYAMARAN LITERAL (Subcultural Masking) untuk "fursuit", "furry fetish", atau "petplay". JANGAN menganggapnya merujuk pada pakaian hewan sungguhan. FLAG sebagai "sexual_deviation" atau "warn" karena ini adalah teknik evasi.
- Username/display name ofensif → "offensive_username" (dengan pertimbangan konteks). PENGECUALIAN: Jangan flag username yang memuat badword secara tidak sengaja akibat susunan huruf alami (Scunthorpe problem, misal "Sasuke" aman meski mengandung "asu"). - Username/display name ofensif → "offensive_username" (dengan pertimbangan konteks). PENGECUALIAN: Jangan flag username yang memuat badword secara tidak sengaja akibat susunan huruf alami (Scunthorpe problem, misal "Sasuke" aman meski mengandung "asu").
## Aturan Analisis URL — JANGAN GUESS BERDASARKAN DOMAIN ## Aturan Analisis URL — JANGAN GUESS BERDASARKAN DOMAIN
@@ -102,9 +110,10 @@ Prioritas rendah (PELANGGARAN RINGAN):
5. Apakah ada conflict_instigation (memancing drama/keributan)? → warn, low-medium 5. Apakah ada conflict_instigation (memancing drama/keributan)? → warn, low-medium
6. Apakah ada username ofensif? → warn, low (kecuali diperkuat isi pesan) 6. Apakah ada username ofensif? → warn, low (kecuali diperkuat isi pesan)
7. Apakah ada spam/promosi borderline? → warn, low-medium 7. Apakah ada spam/promosi borderline? → warn, low-medium
8. Jika tidak ada pelanggaran jelas atau bukti ambigu → clean 8. Jika tidak ada pelanggaran jelas atau bukti ambigu karena murni kurang konteks historis → clean
Jangan pernah flag hanya berdasarkan kecurigaan atau ketidakjelasan konteks. 9. **ENTROPY-TRIGGERED ROUTING (PENTING):** Jika teks terasa "acak" (random), terfragmentasi, sangat tidak jelas sintaksisnya, atau mengandung frasa aneh lintas bahasa, KAMU DILARANG KERAS memberikan status "clean" dengan alasan "konteks percakapan santai". Berikan status "warn" dengan flag "potential_evasion" atau "unclear_context" agar direviu admin. Jangan biarkan teks berentropi tinggi lolos sebagai "clean".
Jika ragu antara dua level, pilih yang lebih rendah (innocent until proven guilty). Jangan pernah flag "clean" hanya karena kebingungan mem-parsing tata bahasa.
Jika ragu antara "clean" dan "warn" karena teks sangat aneh (high entropy), PILIH WARN.
## ATURAN UNTUK GAMBAR — DUA MODE BERBEDA ## ATURAN UNTUK GAMBAR — DUA MODE BERBEDA
@@ -196,6 +205,10 @@ Contoh 10 — Promosi LGBT/furry (sexual_deviation):
Input: [target] id=10101 user=fox: aku gender fluid, panggil aja ze/zer. Yang mau join server furry silakan DM aku ya Input: [target] id=10101 user=fox: aku gender fluid, panggil aja ze/zer. Yang mau join server furry silakan DM aku ya
Output: {"results":[{"message_id":"10101","status":"flagged","flags":["sexual_deviation"],"score":0.75,"categories":["sexual_deviation"],"severity":"medium","confidence":0.85,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["aku gender fluid, panggil aja ze/zer","Yang mau join server furry silakan DM aku"],"analysis":"Pengirim mempromosikan identitas LGBT dan komunitas furry. Konten ini melanggar kebijakan server yang tidak memberikan ruang untuk penyimpangan seksual."}]} Output: {"results":[{"message_id":"10101","status":"flagged","flags":["sexual_deviation"],"score":0.75,"categories":["sexual_deviation"],"severity":"medium","confidence":0.85,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["aku gender fluid, panggil aja ze/zer","Yang mau join server furry silakan DM aku"],"analysis":"Pengirim mempromosikan identitas LGBT dan komunitas furry. Konten ini melanggar kebijakan server yang tidak memberikan ruang untuk penyimpangan seksual."}]}
Contoh 10.5 — Obfuscation & Literal Masking (Anti-Celah Bypass):
Input: [target] id=10505 user=sneaky: I love sesuatu sing that kostum hewan diagem ren
Output: {"results":[{"message_id":"10505","status":"warn","flags":["sexual_deviation","potential_evasion"],"score":0.65,"categories":["sexual_deviation"],"severity":"low","confidence":0.9,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["I love sesuatu sing that kostum hewan diagem ren"],"analysis":"Pengirim menggunakan teknik Polyglot Obfuscation (campuran bahasa Inggris, Indonesia, dan Krama) dengan penyamaran literal 'kostum hewan' untuk menghindari filter. Memiliki entropi tinggi dan berpotensi merujuk pada furry fetish."}]}
Contoh 11 — Username ofensif (isi pesan bersih): Contoh 11 — Username ofensif (isi pesan bersih):
Input: [target] id=12121 user=pejabat_munafik_dajjal: Halo teman-teman, ada yang main game? Input: [target] id=12121 user=pejabat_munafik_dajjal: Halo teman-teman, ada yang main game?
Output: {"results":[{"message_id":"12121","status":"flagged","flags":["offensive_username"],"score":0.3,"categories":["offensive_username"],"severity":"low","confidence":0.95,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["Username 'pejabat_munafik_dajjal' mengandung unsur ofensif/SARA"],"analysis":"Pengirim memiliki username ofensif yang menyerang pejabat dengan label SARA. Namun isi pesan bersih dan tidak terkait username. Flag ringan."}]} Output: {"results":[{"message_id":"12121","status":"flagged","flags":["offensive_username"],"score":0.3,"categories":["offensive_username"],"severity":"low","confidence":0.95,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["Username 'pejabat_munafik_dajjal' mengandung unsur ofensif/SARA"],"analysis":"Pengirim memiliki username ofensif yang menyerang pejabat dengan label SARA. Namun isi pesan bersih dan tidak terkait username. Flag ringan."}]}
@@ -246,6 +259,10 @@ Contoh 10 — Promosi LGBT/furry (sexual_deviation):
Input: [target] id=10101 user=fox: aku gender fluid, panggil aja ze/zer. Yang mau join server furry silakan DM aku ya Input: [target] id=10101 user=fox: aku gender fluid, panggil aja ze/zer. Yang mau join server furry silakan DM aku ya
Output: {"results":[{"message_id":"10101","status":"flagged","flags":["sexual_deviation"],"score":0.75,"categories":["sexual_deviation"],"severity":"medium","confidence":0.85,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["aku gender fluid, panggil aja ze/zer","Yang mau join server furry silakan DM aku"],"analysis":"Pengirim mempromosikan identitas LGBT dan komunitas furry. Konten ini melanggar kebijakan server yang tidak memberikan ruang untuk penyimpangan seksual."}]} Output: {"results":[{"message_id":"10101","status":"flagged","flags":["sexual_deviation"],"score":0.75,"categories":["sexual_deviation"],"severity":"medium","confidence":0.85,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["aku gender fluid, panggil aja ze/zer","Yang mau join server furry silakan DM aku"],"analysis":"Pengirim mempromosikan identitas LGBT dan komunitas furry. Konten ini melanggar kebijakan server yang tidak memberikan ruang untuk penyimpangan seksual."}]}
Contoh 10.5 — Obfuscation & Literal Masking (Anti-Celah Bypass):
Input: [target] id=10505 user=sneaky: I love sesuatu sing that kostum hewan diagem ren
Output: {"results":[{"message_id":"10505","status":"warn","flags":["sexual_deviation","potential_evasion"],"score":0.65,"categories":["sexual_deviation"],"severity":"low","confidence":0.9,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["I love sesuatu sing that kostum hewan diagem ren"],"analysis":"Pengirim menggunakan teknik Polyglot Obfuscation (campuran bahasa Inggris, Indonesia, dan Krama) dengan penyamaran literal 'kostum hewan' untuk menghindari filter. Memiliki entropi tinggi dan berpotensi merujuk pada furry fetish."}]}
Contoh 11 — Username ofensif (isi pesan bersih): Contoh 11 — Username ofensif (isi pesan bersih):
Input: [target] id=12121 user=pejabat_munafik_dajjal: Halo teman-teman, ada yang main game? Input: [target] id=12121 user=pejabat_munafik_dajjal: Halo teman-teman, ada yang main game?
Output: {"results":[{"message_id":"12121","status":"flagged","flags":["offensive_username"],"score":0.3,"categories":["offensive_username"],"severity":"low","confidence":0.95,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["Username 'pejabat_munafik_dajjal' mengandung unsur ofensif/SARA"],"analysis":"Pengirim memiliki username ofensif yang menyerang pejabat dengan label SARA. Namun isi pesan bersih dan tidak terkait username. Flag ringan."}]} Output: {"results":[{"message_id":"12121","status":"flagged","flags":["offensive_username"],"score":0.3,"categories":["offensive_username"],"severity":"low","confidence":0.95,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["Username 'pejabat_munafik_dajjal' mengandung unsur ofensif/SARA"],"analysis":"Pengirim memiliki username ofensif yang menyerang pejabat dengan label SARA. Namun isi pesan bersih dan tidak terkait username. Flag ringan."}]}
@@ -301,6 +318,10 @@ Contoh 10 — Promosi LGBT/furry (sexual_deviation):
Input: [target] id=10101 user=fox: aku gender fluid, panggil aja ze/zer. Yang mau join server furry silakan DM aku ya Input: [target] id=10101 user=fox: aku gender fluid, panggil aja ze/zer. Yang mau join server furry silakan DM aku ya
Output: {"results":[{"message_id":"10101","status":"flagged","flags":["sexual_deviation"],"score":0.75,"categories":["sexual_deviation"],"severity":"medium","confidence":0.85,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["aku gender fluid, panggil aja ze/zer","Yang mau join server furry silakan DM aku"],"analysis":"Pengirim mempromosikan identitas LGBT dan komunitas furry. Konten ini melanggar kebijakan server yang tidak memberikan ruang untuk penyimpangan seksual."}]} Output: {"results":[{"message_id":"10101","status":"flagged","flags":["sexual_deviation"],"score":0.75,"categories":["sexual_deviation"],"severity":"medium","confidence":0.85,"recommended_action":"delete","policy_version":"default-2026-05-30","evidence":["aku gender fluid, panggil aja ze/zer","Yang mau join server furry silakan DM aku"],"analysis":"Pengirim mempromosikan identitas LGBT dan komunitas furry. Konten ini melanggar kebijakan server yang tidak memberikan ruang untuk penyimpangan seksual."}]}
Contoh 10.5 — Obfuscation & Literal Masking (Anti-Celah Bypass):
Input: [target] id=10505 user=sneaky: I love sesuatu sing that kostum hewan diagem ren
Output: {"results":[{"message_id":"10505","status":"warn","flags":["sexual_deviation","potential_evasion"],"score":0.65,"categories":["sexual_deviation"],"severity":"low","confidence":0.9,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["I love sesuatu sing that kostum hewan diagem ren"],"analysis":"Pengirim menggunakan teknik Polyglot Obfuscation (campuran bahasa Inggris, Indonesia, dan Krama) dengan penyamaran literal 'kostum hewan' untuk menghindari filter. Memiliki entropi tinggi dan berpotensi merujuk pada furry fetish."}]}
Contoh 11 — Username ofensif (isi pesan bersih): Contoh 11 — Username ofensif (isi pesan bersih):
Input: [target] id=12121 user=pejabat_munafik_dajjal: Halo teman-teman, ada yang main game? Input: [target] id=12121 user=pejabat_munafik_dajjal: Halo teman-teman, ada yang main game?
Output: {"results":[{"message_id":"12121","status":"flagged","flags":["offensive_username"],"score":0.3,"categories":["offensive_username"],"severity":"low","confidence":0.95,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["Username 'pejabat_munafik_dajjal' mengandung unsur ofensif/SARA"],"analysis":"Pengirim memiliki username ofensif yang menyerang pejabat dengan label SARA. Namun isi pesan bersih dan tidak terkait username. Flag ringan."}]} Output: {"results":[{"message_id":"12121","status":"flagged","flags":["offensive_username"],"score":0.3,"categories":["offensive_username"],"severity":"low","confidence":0.95,"recommended_action":"warn","policy_version":"default-2026-05-30","evidence":["Username 'pejabat_munafik_dajjal' mengandung unsur ofensif/SARA"],"analysis":"Pengirim memiliki username ofensif yang menyerang pejabat dengan label SARA. Namun isi pesan bersih dan tidak terkait username. Flag ringan."}]}
@@ -455,7 +476,9 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
parts.push(`## Kultur Channel (Pembelajaran AI)\n${channelCulture}`); parts.push(`## Kultur Channel (Pembelajaran AI)\n${channelCulture}`);
} }
parts.push(`## Konteks Pengguna\nSetiap pesan mungkin memiliki tag <user_reputation>. Tag ini hanya indikator **referensi**, bukan bukti pelanggaran. Nilai trust_score yang rendah bukan alasan untuk memflag pesan yang bersih. Nilai trust_score yang tinggi bukan alasan untuk mengabaikan pelanggaran nyata. **Setiap pesan harus dinilai berdasarkan isinya sendiri.**`); parts.push(
`## Konteks Pengguna\nSetiap pesan mungkin memiliki tag <user_reputation>. Tag ini hanya indikator **referensi**, bukan bukti pelanggaran. Nilai trust_score yang rendah bukan alasan untuk memflag pesan yang bersih. Nilai trust_score yang tinggi bukan alasan untuk mengabaikan pelanggaran nyata. **Setiap pesan harus dinilai berdasarkan isinya sendiri.**`,
);
parts.push(OUTPUT_INSTRUCTIONS); parts.push(OUTPUT_INSTRUCTIONS);
@@ -54,7 +54,9 @@ export async function initializeUserReputation(
/** /**
* Fetch a user's reputation score. Returns default 50 if none exists. * Fetch a user's reputation score. Returns default 50 if none exists.
*/ */
export async function getUserReputation(userId: string): Promise<UserReputation | null> { export async function getUserReputation(
userId: string,
): Promise<UserReputation | null> {
const db = getDatabase(); const db = getDatabase();
const existing = await db const existing = await db
.select() .select()
@@ -68,7 +70,10 @@ export async function getUserReputation(userId: string): Promise<UserReputation
/** /**
* Increment the clean message streak and update trust score if threshold is met. * Increment the clean message streak and update trust score if threshold is met.
*/ */
export async function recordCleanMessage(userId: string, guildId: string): Promise<void> { export async function recordCleanMessage(
userId: string,
guildId: string,
): Promise<void> {
const rep = await initializeUserReputation(userId, guildId); const rep = await initializeUserReputation(userId, guildId);
const db = getDatabase(); const db = getDatabase();
let newStreak = rep.clean_message_streak + 1; let newStreak = rep.clean_message_streak + 1;
@@ -133,7 +138,10 @@ export async function recordInfraction(
/** /**
* Fetch a user's past N flagged messages for context injection. * Fetch a user's past N flagged messages for context injection.
*/ */
export async function getUserRecentInfractions(userId: string, limit: number = 3) { export async function getUserRecentInfractions(
userId: string,
limit: number = 3,
) {
const db = getDatabase(); const db = getDatabase();
return await db return await db
.select({ .select({
@@ -146,8 +154,8 @@ export async function getUserRecentInfractions(userId: string, limit: number = 3
.where( .where(
and( and(
eq(messagesTable.user_id, userId), eq(messagesTable.user_id, userId),
eq(messagesTable.ai_status, "flagged") eq(messagesTable.ai_status, "flagged"),
) ),
) )
.orderBy(desc(messagesTable.created_at)) .orderBy(desc(messagesTable.created_at))
.limit(limit); .limit(limit);
@@ -828,12 +828,10 @@ export async function getConversationKeysWithIncompleteAnalysis(
try { try {
const database = db(); const database = db();
const rows = await database const rows = await database
.selectDistinct<Array<{ thread_id: string | null; channel_id: string }>>( .selectDistinct<Array<{ thread_id: string | null; channel_id: string }>>({
{ thread_id: messagesTable.thread_id,
thread_id: messagesTable.thread_id, channel_id: messagesTable.channel_id,
channel_id: messagesTable.channel_id, })
},
)
.from(messagesTable) .from(messagesTable)
.where( .where(
and( and(
@@ -928,7 +926,6 @@ export async function getIncompleteMessagesByConversation(
} }
} }
// Message Reviews CRUD // Message Reviews CRUD
// ==================== // ====================
@@ -1307,7 +1304,10 @@ export async function revertStuckProcessingMessages(
if (Array.isArray(rows) && rows.length > 0) { if (Array.isArray(rows) && rows.length > 0) {
logger.info( logger.info(
{ count: rows.length, messageIds: rows.map((r: { id: string }) => r.id) }, {
count: rows.length,
messageIds: rows.map((r: { id: string }) => r.id),
},
"Reverted stuck processing messages back to pending", "Reverted stuck processing messages back to pending",
); );
} }
@@ -1,7 +1,13 @@
import type fs from "node:fs"; import type fs from "node:fs";
import type prism from "prism-media"; import type prism from "prism-media";
export type AIStatus = "pending" | "processing" | "clean" | "warn" | "flagged" | "error"; export type AIStatus =
| "pending"
| "processing"
| "clean"
| "warn"
| "flagged"
| "error";
export type AISeverity = "none" | "low" | "medium" | "high" | "critical"; export type AISeverity = "none" | "low" | "medium" | "high" | "critical";
export type AIRecommendedAction = export type AIRecommendedAction =
| "none" | "none"
@@ -258,7 +258,9 @@ export const pgUserReputationsTable = pgTable(
user_id: pgText("user_id").primaryKey(), user_id: pgText("user_id").primaryKey(),
guild_id: pgText("guild_id").notNull(), guild_id: pgText("guild_id").notNull(),
trust_score: pgInteger("trust_score").notNull().default(50), trust_score: pgInteger("trust_score").notNull().default(50),
clean_message_streak: pgInteger("clean_message_streak").notNull().default(0), clean_message_streak: pgInteger("clean_message_streak")
.notNull()
.default(0),
total_infractions: pgInteger("total_infractions").notNull().default(0), total_infractions: pgInteger("total_infractions").notNull().default(0),
last_infraction_at: pgBigint("last_infraction_at", { mode: "number" }), last_infraction_at: pgBigint("last_infraction_at", { mode: "number" }),
created_at: pgBigint("created_at", { mode: "number" }).notNull(), created_at: pgBigint("created_at", { mode: "number" }).notNull(),
@@ -280,7 +282,9 @@ export const pgChannelCulturesTable = pgTable(
channel_id: pgText("channel_id").primaryKey(), channel_id: pgText("channel_id").primaryKey(),
guild_id: pgText("guild_id").notNull(), guild_id: pgText("guild_id").notNull(),
culture_summary: pgText("culture_summary").notNull(), culture_summary: pgText("culture_summary").notNull(),
last_analyzed_at: pgBigint("last_analyzed_at", { mode: "number" }).notNull(), last_analyzed_at: pgBigint("last_analyzed_at", {
mode: "number",
}).notNull(),
}, },
(table) => ({ (table) => ({
guildIdx: pgIndex("idx_channel_cultures_guild_id").on(table.guild_id), guildIdx: pgIndex("idx_channel_cultures_guild_id").on(table.guild_id),