feat(ai-moderation): enrich analysis context with recency, repetition, user history and channel topic

- <message> targets now carry time (ISO), repetitions (N identical short texts = spam signal), bot and edited flags; escape id/user XML
- rich <user_reputation>: total_infractions, clean_streak, last_offense_days_ago, repeat_offender (7-day window)
- <user_history> with last flagged messages for repeat offenders (wires dead getUserRecentInfractions)
- <user_profile as_of> staleness signal; <location_context topic> from captured channel topic
- prompt framing + output instructions teach the LLM to use the new signals without treating history as proof
- tests: contextEnrichment.test.ts (13) + topic cases in conversationContext.test.ts
This commit is contained in:
asepharyana
2026-08-10 17:15:33 +07:00
parent 0a5254bf20
commit 65c9c2cd9e
11 changed files with 488 additions and 26 deletions
@@ -165,6 +165,7 @@ export function buildLocationContext(targets: MessageRecord[]): string {
channel?: { channel?: {
channelName?: string | null; channelName?: string | null;
threadName?: string | null; threadName?: string | null;
topic?: string | null;
nsfw?: boolean; nsfw?: boolean;
ageRestricted?: boolean; ageRestricted?: boolean;
nsfwLevel?: string | null; nsfwLevel?: string | null;
@@ -181,6 +182,13 @@ export function buildLocationContext(targets: MessageRecord[]): string {
if (ch.threadName) if (ch.threadName)
attrs.push(`thread_name="${escapeXml(ch.threadName)}"`); attrs.push(`thread_name="${escapeXml(ch.threadName)}"`);
} }
if (typeof ch.topic === "string" && ch.topic.trim().length > 0) {
const topic =
ch.topic.length > 200
? `${ch.topic.slice(0, 200).trimEnd()}`
: ch.topic;
attrs.push(`topic="${escapeXml(topic)}"`);
}
if (typeof ch.nsfw === "boolean") attrs.push(`nsfw="${ch.nsfw}"`); if (typeof ch.nsfw === "boolean") attrs.push(`nsfw="${ch.nsfw}"`);
if (typeof ch.ageRestricted === "boolean") { if (typeof ch.ageRestricted === "boolean") {
attrs.push(`age_restricted="${ch.ageRestricted}"`); attrs.push(`age_restricted="${ch.ageRestricted}"`);
@@ -66,13 +66,22 @@ export async function runMediaBatch(
}); });
// Gather user profiles ONCE for the whole batch and emit a deduplicated // Gather user profiles ONCE for the whole batch and emit a deduplicated
// <user_profiles> map; per-message blocks (from prepareMediaMessage) // <user_profiles> map (with last-generated timestamp); per-message blocks
// reference it via <user_profile_ref>. // (from prepareMediaMessage) reference it via <user_profile_ref>.
const profileByUser = new Map<string, string>(); const profileByUser = new Map<
string,
{
text: string;
asOf?: number | null;
}
>();
for (const t of targets) { for (const t of targets) {
if (profileByUser.has(t.user_id)) continue; if (profileByUser.has(t.user_id)) continue;
const profile = await getUserProfile(t.user_id); const profile = await getUserProfile(t.user_id);
profileByUser.set(t.user_id, profile?.profile_summary ?? ""); profileByUser.set(t.user_id, {
text: profile?.profile_summary ?? "",
asOf: profile?.last_analyzed_at ?? null,
});
} }
const userProfilesBlock = buildUserProfilesBlock(profileByUser); const userProfilesBlock = buildUserProfilesBlock(profileByUser);
@@ -95,18 +95,29 @@ export function truncateForAi(content: string): string {
// entries per message with <user_profile_ref user_id="..."/>. // entries per message with <user_profile_ref user_id="..."/>.
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
export interface UserProfileEntry {
/** Profile summary text (from user_profiles.profile_summary). */
text: string;
/** Epoch ms when the profile was last generated — staleness signal for
* the LLM (a profile from months ago may not reflect current behavior). */
asOf?: number | null;
}
/** Build a deduplicated `<user_profiles>` map block, keyed by Discord user id. */ /** Build a deduplicated `<user_profiles>` map block, keyed by Discord user id. */
export function buildUserProfilesBlock( export function buildUserProfilesBlock(
profiles: ReadonlyMap<string, string>, profiles: ReadonlyMap<string, UserProfileEntry>,
): string { ): string {
const entries = Array.from(profiles.entries()).filter( const entries = Array.from(profiles.entries()).filter(
([, text]) => text.trim().length > 0, ([, entry]) => entry.text.trim().length > 0,
); );
if (entries.length === 0) return ""; if (entries.length === 0) return "";
const lines = entries.map( const lines = entries.map(([userId, entry]) => {
([userId, text]) => const asOfAttr =
` <user_profile user_id="${escapeXml(userId)}">${sanitizeAiContent(text)}</user_profile>`, typeof entry.asOf === "number" && entry.asOf > 0
); ? ` as_of="${new Date(entry.asOf).toISOString()}"`
: "";
return ` <user_profile user_id="${escapeXml(userId)}"${asOfAttr}>${sanitizeAiContent(entry.text)}</user_profile>`;
});
return `<user_profiles>\n${lines.join("\n")}\n</user_profiles>`; return `<user_profiles>\n${lines.join("\n")}\n</user_profiles>`;
} }
@@ -115,6 +126,110 @@ export function buildUserProfileRef(userId: string): string {
return `<user_profile_ref user_id="${escapeXml(userId)}"/>`; return `<user_profile_ref user_id="${escapeXml(userId)}"/>`;
} }
// ---------------------------------------------------------------------------
// User reputation — richer than a bare trust score.
//
// The trust model tracks total_infractions, a clean-message streak and the
// last infraction timestamp. Feeding all of it to the LLM lets it tell a
// first-timer (same score, 1 infraction) from a repeat offender (score 50,
// 3 infractions, last one yesterday) — the same score means very different
// things in those two contexts.
// ---------------------------------------------------------------------------
export interface ReputationAttrsSource {
trust_score: number;
total_infractions: number;
clean_message_streak: number;
last_infraction_at: number | null;
}
const DAY_MS = 24 * 60 * 60 * 1000;
const REPEAT_OFFENSE_WINDOW_MS = 7 * DAY_MS;
/**
* Formats reputation fields into XML attributes for `<user_reputation .../>`.
* Derived signals: last_offense_days_ago (0 = today) and repeat_offender
* (infraction within the last 7 days) are computed here so both the text and
* media paths emit the exact same shape.
*/
export function formatReputationAttrs(
rep: ReputationAttrsSource,
now: number = Date.now(),
): string {
const attrs = [
`trust_score="${rep.trust_score}"`,
`total_infractions="${rep.total_infractions}"`,
`clean_streak="${rep.clean_message_streak}"`,
];
if (
typeof rep.last_infraction_at === "number" &&
rep.last_infraction_at > 0
) {
const daysAgo = Math.max(
0,
Math.floor((now - rep.last_infraction_at) / DAY_MS),
);
attrs.push(`last_offense_days_ago="${daysAgo}"`);
const isRepeat =
rep.total_infractions > 0 &&
now - rep.last_infraction_at <= REPEAT_OFFENSE_WINDOW_MS;
if (isRepeat) attrs.push(`repeat_offender="true"`);
}
return attrs.join(" ");
}
/**
* Builds an optional `<user_history>` block (last flagged messages) from
* getUserRecentInfractions rows. Only emitted when there is real history —
* lets the LLM see the PATTERN (e.g. the same scam link posted repeatedly)
* without treating old flags as proof for the current message.
*/
export function buildUserHistoryXml(
history: Array<{
content: string;
severity: string | null;
created_at: number;
}>,
now: number = Date.now(),
): string {
const filtered = history.filter((h) => h.content?.trim());
if (filtered.length === 0) return "";
const lines = filtered.map((h) => {
const daysAgo = Math.max(0, Math.floor((now - h.created_at) / DAY_MS));
const severityAttr = h.severity
? ` severity="${escapeXml(h.severity)}"`
: "";
const snippet =
h.content.length > 100
? `${h.content.slice(0, 100).trimEnd()}`
: h.content;
return ` <infraction${severityAttr} time_ago_days="${daysAgo}">${escapeXml(snippet)}</infraction>`;
});
return `<user_history>\n${lines.join("\n")}\n</user_history>`;
}
/**
* Whether the message author was a bot (captured in metadata.author.bot).
* Bot posts (logging bots, webhook-style automation) deserve different
* scrutiny than user posts — expose the flag instead of hiding it.
*/
export function resolveIsBot(msg: MessageRecord): boolean {
if (!msg.metadata) return false;
try {
const meta = JSON.parse(msg.metadata) as {
author?: { bot?: boolean } | null;
};
return Boolean(meta?.author?.bot);
} catch {
return false;
}
}
/** Whether the shown content is an EDIT of the original post (evasion signal). */
export function resolveIsEdited(msg: MessageRecord): boolean {
return Boolean(msg.edited_content);
}
/** /**
* Returns the real text content for AI analysis, stripping fallback text * Returns the real text content for AI analysis, stripping fallback text
* that getDisplayContent() synthesized ("[Attachment: ...]", "[Sticker: ...]", * that getDisplayContent() synthesized ("[Attachment: ...]", "[Sticker: ...]",
@@ -40,6 +40,7 @@ Data konteks tersedia: <user_profiles> (peta ringkasan kepribadian, di pesan USE
Gunakan untuk personalisasi analysis, tapi: Gunakan untuk personalisasi analysis, tapi:
- Profil adalah KONTEKS, bukan bukti. Profil mencurigakan ≠ flag; profil bersih ≠ loloskan pelanggaran. - Profil adalah KONTEKS, bukan bukti. Profil mencurigakan ≠ flag; profil bersih ≠ loloskan pelanggaran.
- Perubahan perilaku mencolok (biasanya teknis tiba-tiba provokatif) layak dicatat di analysis. - Perubahan perilaku mencolok (biasanya teknis tiba-tiba provokatif) layak dicatat di analysis.
- <user_history> (kutipan pesan yang pernah di-flag) = pola pelanggaran lama. Gunakan untuk mendeteksi PENGULANGAN (mis. spam link yang sama, provokasi berulang), tapi JANGAN memflag pesan bersih hanya karena riwayat.
- JANGAN paksa referensi profil jika tidak relevan — analysis natural lebih baik. - JANGAN paksa referensi profil jika tidak relevan — analysis natural lebih baik.
- Channel culture coding/teknis → pesan teknis lebih wajar; channel santai → slang lebih wajar. Jangan dipakai mengabaikan pelanggaran nyata. - Channel culture coding/teknis → pesan teknis lebih wajar; channel santai → slang lebih wajar. Jangan dipakai mengabaikan pelanggaran nyata.
@@ -58,6 +59,7 @@ Contoh buruk: "Pesan berisi teks dan gambar tanpa pelanggaran." (mengabaikan buk
- **conflict_instigation:** "Pengirim <ajakan memicu konflik>. <konteks>. Diberi peringatan karena berpotensi memicu drama." - **conflict_instigation:** "Pengirim <ajakan memicu konflik>. <konteks>. Diberi peringatan karena berpotensi memicu drama."
- **Username ofensif (pesan bersih):** "Pengirim memiliki username yang <alasan ofensif>. Isi pesan hanya <isi>. Diberi warning ringan." — (pesan memperkuat): "<username SARA> + isi pesan memperkuat tone kebencian. Pelanggaran berat." - **Username ofensif (pesan bersih):** "Pengirim memiliki username yang <alasan ofensif>. Isi pesan hanya <isi>. Diberi warning ringan." — (pesan memperkuat): "<username SARA> + isi pesan memperkuat tone kebencian. Pelanggaran berat."
- **Evasi (zalgo/leetspeak):** "Pengirim menggunakan teknik obfuscation untuk menyembunyikan <makna asli>. <dampak>. <kesimpulan>." - **Evasi (zalgo/leetspeak):** "Pengirim menggunakan teknik obfuscation untuk menyembunyikan <makna asli>. <dampak>. <kesimpulan>."
- **Spam (repetitions > 1):** "Pengirim mengirim teks yang sama sebanyak N kali dalam waktu singkat. <isi pesan>. Diberi peringatan karena spam berulang." — nilai tetap dari isi; pengulangan saja (mis. "ok" x5 dalam obrolan aktif) bukan pelanggaran.
- **sexual_deviation:** "Pengirim <konten penyimpangan>. <konteks>. Melanggar kebijakan server." - **sexual_deviation:** "Pengirim <konten penyimpangan>. <konteks>. Melanggar kebijakan server."
- **SARA/penistaan agama:** "Pengirim <jenis penistaan spesifik: parodi ayat, mengaku Tuhan, mockery ritual, istilah agama sebagai joke, provokasi antar-agama>. <bukti>. Melanggar kebijakan SARA." — JANGAN gunakan kata "bercanda" untuk SARA. - **SARA/penistaan agama:** "Pengirim <jenis penistaan spesifik: parodi ayat, mengaku Tuhan, mockery ritual, istilah agama sebagai joke, provokasi antar-agama>. <bukti>. Melanggar kebijakan SARA." — JANGAN gunakan kata "bercanda" untuk SARA.
@@ -105,17 +105,18 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
parts.push( parts.push(
`## Blok Data di Pesan USER\n` + `## Blok Data di Pesan USER\n` +
`Semua data dinamis per-batch dikirim di pesan USER — system prompt ini TIDAK memuat data batch:\n` + `Semua data dinamis per-batch dikirim di pesan USER — system prompt ini TIDAK memuat data batch:\n` +
`- <location_context .../> = metadata channel/thread (channel_id, channel_name, thread_name, nsfw, age_restricted).\n` + `- <location_context .../> = metadata channel/thread (channel_id, channel_name, thread_name, topic, nsfw, age_restricted). topic = deskripsi resmi channel — pakai untuk menilai kesesuaian pesan dengan tujuan channel.\n` +
`- <conversation_context> = obrolan SEBELUM pesan target. Baris "[context]" di dalamnya BUKAN yang dinilai.\n` + `- <conversation_context> = obrolan SEBELUM pesan target. Baris "[context]" di dalamnya BUKAN yang dinilai.\n` +
`- <user_profiles> = peta ringkasan kepribadian per user_id; setiap <message> merujuk lewat <user_profile_ref user_id="..."/>.\n` + `- <user_profiles> = peta ringkasan kepribadian per user_id (attr as_of = kapan profil terakhir dibuat — profil lama mungkin tidak mencerminkan perilaku terkini); setiap <message> merujuk lewat <user_profile_ref user_id="..."/>.\n` +
`- <web_searches> / <web_content> = bukti web (lihat "Web Sebagai Bukti Utama").\n` + `- <web_searches> / <web_content> = bukti web (lihat "Web Sebagai Bukti Utama").\n` +
`- <messages_to_analyze> = pesan-pesan TARGET yang WAJIB dinilai.`, `- <messages_to_analyze> = pesan-pesan TARGET yang WAJIB dinilai. Atribut <message>: id, user (nama server), time (ISO — kapan pesan dikirim), repetitions (N = teks pendek sama muncul N kali di batch — sinyal spam), bot (true jika dari bot), edited (true jika konten adalah hasil edit setelah posting).`,
); );
parts.push( parts.push(
`## Konteks Pengguna (Referensi, Bukan Bukti)\n` + `## Konteks Pengguna (Referensi, Bukan Bukti)\n` +
`Konteks per pengguna hanya indikator **referensi** untuk personalisasi analisis, BUKAN bukti pelanggaran:\n` + `Konteks per pengguna hanya indikator **referensi** untuk personalisasi analisis, BUKAN bukti pelanggaran:\n` +
`- <user_reputation trust_score="..."> = histori moderasi pengguna. Skor rendah BUKAN alasan memflag pesan bersih; skor tinggi BUKAN alasan mengabaikan pelanggaran nyata.\n` + `- <user_reputation trust_score="..." total_infractions="..." clean_streak="..." last_offense_days_ago="..." repeat_offender="..."> = histori moderasi pengguna. Skor rendah BUKAN alasan memflag pesan bersih; skor tinggi BUKAN alasan mengabaikan pelanggaran nyata. repeat_offender="true" = ada pelanggaran dalam 7 hari terakhir.\n` +
`- <user_history> (di dalam <user_reputation>) = kutipan pesan-pesan pengguna yang PERNAH di-flag. Gunakan untuk mengenali POLA berulang (spam link sama, provokasi), tapi JANGAN memflag pesan bersih hanya karena riwayat.\n` +
`- <user_profiles> (di pesan USER) = peta ringkasan kepribadian per user_id. <user_profile_ref user_id="..."/> dalam sebuah pesan menunjuk ke peta itu. Tanpa ref = tidak ada profil untuk pengguna tersebut.\n` + `- <user_profiles> (di pesan USER) = peta ringkasan kepribadian per user_id. <user_profile_ref user_id="..."/> dalam sebuah pesan menunjuk ke peta itu. Tanpa ref = tidak ada profil untuk pengguna tersebut.\n` +
`- Profil berguna untuk mengenali penyimpangan perilaku mencolok (mis. pengguna teknis tiba-tiba provokatif), tapi JANGAN memflag atau meloloskan hanya karena profil.\n` + `- Profil berguna untuk mengenali penyimpangan perilaku mencolok (mis. pengguna teknis tiba-tiba provokatif), tapi JANGAN memflag atau meloloskan hanya karena profil.\n` +
`**Setiap pesan dinilai berdasarkan isinya sendiri.**`, `**Setiap pesan dinilai berdasarkan isinya sendiri.**`,
@@ -126,7 +127,11 @@ export function buildSystemPrompt(options: BuildSystemPromptOptions): string {
`- Baris dalam <conversation_context> berformat "[context] id=... time=<ISO> user=<nama>: isi", diurutkan paling lama → paling baru. Baris pertama biasanya "[conversation_flow] status=... context_msgs=... dropped=..." — metadata sistem tentang status percakapan (ongoing/sparse/cold_start), BUKAN pesan yang dinilai.\n` + `- Baris dalam <conversation_context> berformat "[context] id=... time=<ISO> user=<nama>: isi", diurutkan paling lama → paling baru. Baris pertama biasanya "[conversation_flow] status=... context_msgs=... dropped=..." — metadata sistem tentang status percakapan (ongoing/sparse/cold_start), BUKAN pesan yang dinilai.\n` +
`- <messages_to_analyze> berisi pesan-pesan TARGET yang WAJIB dinilai. Hasilkan SATU hasil per message_id — jangan menggabungkan beberapa pesan, jangan melewati, jangan mengarang id.\n` + `- <messages_to_analyze> berisi pesan-pesan TARGET yang WAJIB dinilai. Hasilkan SATU hasil per message_id — jangan menggabungkan beberapa pesan, jangan melewati, jangan mengarang id.\n` +
`- Setiap target dinilai berdasarkan isinya sendiri; konteks percakapan memengaruhi interpretasi, bukan menggantikan isi pesan.\n` + `- Setiap target dinilai berdasarkan isinya sendiri; konteks percakapan memengaruhi interpretasi, bukan menggantikan isi pesan.\n` +
`- Marker "…[pesan dipotong: terlalu panjang]" = konten TARGET sengaja dipotong; marker "…[konteks dipotong: terlalu panjang]" = konten pesan KONTEKS dipotong. Nilai dari bagian yang terlihat; pemotongan BUKAN pelanggaran dan BUKAN teknik evasi.`, `- Marker "…[pesan dipotong: terlalu panjang]" = konten TARGET sengaja dipotong; marker "…[konteks dipotong: terlalu panjang]" = konten pesan KONTEKS dipotong. Nilai dari bagian yang terlihat; pemotongan BUKAN pelanggaran dan BUKAN teknik evasi.\n` +
`- Atribut time= pada <message> target = kapan pesan dikirim (ISO). Pakai untuk menilai kerelevanan waktu (mis. pesan lama di-bump, spam beruntun dalam menit yang sama).\n` +
`- repetitions="N" pada <message> = teks pendek yang sama muncul N kali dalam batch — pertimbangkan sebagai sinyal spam, tapi nilai tetap dari isi pesan.\n` +
`- bot="true" = pengirim adalah bot (otomatisasi), bukan pengguna manusia — jangan perlakukan sebagai pelanggaran personal, tapi kontennya tetap dinilai.\n` +
`- edited="true" = konten yang ditampilkan adalah hasil edit setelah posting (sinyal potensi evasi), nilai konten saat ini apa adanya.`,
); );
parts.push(OUTPUT_INSTRUCTIONS); parts.push(OUTPUT_INSTRUCTIONS);
@@ -19,11 +19,15 @@ import { callModerationLLM } from "./llmCaller.js";
import { analyzeSingleMediaImage } from "./mediaAnalysisClient.js"; import { analyzeSingleMediaImage } from "./mediaAnalysisClient.js";
import { import {
buildReferenceXml, buildReferenceXml,
buildUserHistoryXml,
buildUserProfileRef, buildUserProfileRef,
buildUserProfilesBlock, buildUserProfilesBlock,
escapeXml, escapeXml,
formatReputationAttrs,
getAnalysisContent, getAnalysisContent,
resolveDisplayName, resolveDisplayName,
resolveIsBot,
resolveIsEdited,
truncateForAi, truncateForAi,
} from "./moderationBuilders.js"; } from "./moderationBuilders.js";
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js"; import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
@@ -36,7 +40,10 @@ import {
import { getRecentCorrectedModerations } from "./textCacheStore.js"; import { getRecentCorrectedModerations } from "./textCacheStore.js";
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js"; import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
import { getUserProfile } from "./userProfileStore.js"; import { getUserProfile } from "./userProfileStore.js";
import { initializeUserReputation } from "./userReputationStore.js"; import {
getUserRecentInfractions,
initializeUserReputation,
} from "./userReputationStore.js";
import type { MessageImagePart } from "./visionAnalyzer.js"; import type { MessageImagePart } from "./visionAnalyzer.js";
const log = createChildLogger("textBatchProcessor"); const log = createChildLogger("textBatchProcessor");
@@ -191,18 +198,46 @@ export async function runTextOnlyBatch(
// User reputation + profiles (raw summary text — deduplicated into a // User reputation + profiles (raw summary text — deduplicated into a
// single <user_profiles> map per batch; messages only reference it). // single <user_profiles> map per batch; messages only reference it).
const userContexts = new Map<string, string>(); const userContexts = new Map<string, string>();
const userProfiles = new Map<string, string>(); const userProfiles = new Map<
string,
{
text: string;
asOf?: number | null;
}
>();
for (const msg of batch) { for (const msg of batch) {
if (!userContexts.has(msg.user_id)) { if (!userContexts.has(msg.user_id)) {
const rep = await initializeUserReputation(msg.user_id, msg.guild_id); const rep = await initializeUserReputation(msg.user_id, msg.guild_id);
userContexts.set( const repAttrs = formatReputationAttrs(rep);
msg.user_id, let repXml = `<user_reputation ${repAttrs}/>`;
`<user_reputation trust_score="${rep.trust_score}" />`, // Repeat offenders get their last flagged messages as <user_history>
); // so the LLM can recognize PATTERNS (same scam link, repeated
// provocation) — history is reference, never proof. Best-effort.
if (rep.total_infractions > 0) {
try {
const history = await getUserRecentInfractions(msg.user_id, 2);
const historyXml = buildUserHistoryXml(
history.map((h) => ({
content: h.content ?? "",
severity: h.severity,
created_at: h.created_at,
})),
);
if (historyXml) {
repXml = `<user_reputation ${repAttrs}>\n${historyXml}\n</user_reputation>`;
}
} catch {
// history is a bonus — fall back to attrs-only reputation
}
}
userContexts.set(msg.user_id, repXml);
} }
if (!userProfiles.has(msg.user_id)) { if (!userProfiles.has(msg.user_id)) {
const profile = await getUserProfile(msg.user_id); const profile = await getUserProfile(msg.user_id);
userProfiles.set(msg.user_id, profile?.profile_summary ?? ""); userProfiles.set(msg.user_id, {
text: profile?.profile_summary ?? "",
asOf: profile?.last_analyzed_at ?? null,
});
} }
} }
const userProfilesBlock = buildUserProfilesBlock(userProfiles); const userProfilesBlock = buildUserProfilesBlock(userProfiles);
@@ -303,11 +338,16 @@ export async function runTextOnlyBatch(
.map((line) => `\n${line}`) .map((line) => `\n${line}`)
.join(""); .join("");
const userCtx = userContexts.get(msg.user_id) ?? ""; const userCtx = userContexts.get(msg.user_id) ?? "";
const userProfileRef = (userProfiles.get(msg.user_id) ?? "").trim() const userProfileRef = (
userProfiles.get(msg.user_id)?.text ?? ""
).trim()
? buildUserProfileRef(msg.user_id) ? buildUserProfileRef(msg.user_id)
: ""; : "";
const refXml = await buildReferenceXml(msg); const refXml = await buildReferenceXml(msg);
return `<message id="${msg.id}" user="${resolveDisplayName(msg)}">\n ${userCtx}${userProfileRef ? `\n ${userProfileRef}` : ""}${refXml ? `\n ${refXml}` : ""}\n <content>${escapeXml(content)}</content>${webContext}${mediaEvidenceCtx}\n</message>`; const repetitionCount = groupMapping.get(msg.id)?.length ?? 1;
const isBot = resolveIsBot(msg);
const isEdited = resolveIsEdited(msg);
return `<message id="${escapeXml(msg.id)}" user="${escapeXml(resolveDisplayName(msg))}" time="${new Date(msg.created_at).toISOString()}"${repetitionCount > 1 ? ` repetitions="${repetitionCount}"` : ""}${isBot ? ` bot="true"` : ""}${isEdited ? ` edited="true"` : ""}>\n ${userCtx}${userProfileRef ? `\n ${userProfileRef}` : ""}${refXml ? `\n ${refXml}` : ""}\n <content>${escapeXml(content)}</content>${webContext}${mediaEvidenceCtx}\n</message>`;
}), }),
) )
).join("\n"); ).join("\n");
@@ -37,10 +37,14 @@ import {
} from "./mediaDownloader.js"; } from "./mediaDownloader.js";
import { import {
buildReferenceXml, buildReferenceXml,
buildUserHistoryXml,
buildUserProfileRef, buildUserProfileRef,
escapeXml, escapeXml,
formatReputationAttrs,
getAnalysisContent, getAnalysisContent,
resolveDisplayName, resolveDisplayName,
resolveIsBot,
resolveIsEdited,
truncateForAi, truncateForAi,
} from "./moderationBuilders.js"; } from "./moderationBuilders.js";
import { import {
@@ -56,7 +60,10 @@ import {
} from "./searxngSearch.js"; } from "./searxngSearch.js";
import { extractUrlsFromText } from "./urlFetcher.js"; import { extractUrlsFromText } from "./urlFetcher.js";
import { getUserProfile } from "./userProfileStore.js"; import { getUserProfile } from "./userProfileStore.js";
import { initializeUserReputation } from "./userReputationStore.js"; import {
getUserRecentInfractions,
initializeUserReputation,
} from "./userReputationStore.js";
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Types // Types
@@ -376,6 +383,30 @@ export async function prepareMediaMessage(
? buildUserProfileRef(target.user_id) ? buildUserProfileRef(target.user_id)
: ""; : "";
const messageBlock = `<message id="${escapeXml(target.id)}" user="${escapeXml(resolveDisplayName(target))}">\n <user_reputation trust_score="${rep.trust_score}" />${profileRef ? `\n ${profileRef}` : ""}${refXml ? `\n ${refXml}` : ""}\n <content>${escapeXml(truncateForAi(content))}</content>${mediaContext ? ` ${escapeXml(mediaContext)}` : ""}${webContext}${mediaAnalysisContext}${searxngXml}\n</message>`; // Rich reputation — same shape as the text path: attrs + optional
// <user_history> with the last flagged messages for repeat offenders.
const repAttrs = formatReputationAttrs(rep);
let repXml = `<user_reputation ${repAttrs}/>`;
if (rep.total_infractions > 0) {
try {
const history = await getUserRecentInfractions(target.user_id, 2);
const historyXml = buildUserHistoryXml(
history.map((h) => ({
content: h.content ?? "",
severity: h.severity,
created_at: h.created_at,
})),
);
if (historyXml) {
repXml = `<user_reputation ${repAttrs}>\n${historyXml}\n</user_reputation>`;
}
} catch {
// history is a bonus — fall back to attrs-only reputation
}
}
const isBot = resolveIsBot(target);
const isEdited = resolveIsEdited(target);
const messageBlock = `<message id="${escapeXml(target.id)}" user="${escapeXml(resolveDisplayName(target))}" time="${new Date(target.created_at).toISOString()}"${isBot ? ` bot="true"` : ""}${isEdited ? ` edited="true"` : ""}>\n ${repXml}${profileRef ? `\n ${profileRef}` : ""}${refXml ? `\n ${refXml}` : ""}\n <content>${escapeXml(truncateForAi(content))}</content>${mediaContext ? ` ${escapeXml(mediaContext)}` : ""}${webContext}${mediaAnalysisContext}${searxngXml}\n</message>`;
return { targetId, messageBlock }; return { targetId, messageBlock };
} }
@@ -9,6 +9,10 @@ export interface MessageLocation {
threadId: string | null; threadId: string | null;
threadName: string | null; threadName: string | null;
channelName: string | null; channelName: string | null;
/** Channel topic (resmi/deskripsi channel) — strong context for judging
* whether a message fits the channel's purpose. Guarded: some channel
* types (threads on older API builds) expose no topic. */
topic?: string | null;
nsfw?: boolean; nsfw?: boolean;
nsfwLevel?: string | null; nsfwLevel?: string | null;
ageRestricted?: boolean; ageRestricted?: boolean;
@@ -107,12 +111,17 @@ export function getMessageLocation(message: Message): MessageLocation {
nsfw?: boolean; nsfw?: boolean;
nsfwLevel?: string | null; nsfwLevel?: string | null;
}; };
const topic =
"topic" in channel && typeof channel.topic === "string"
? channel.topic
: null;
if (!channel.isThread?.()) { if (!channel.isThread?.()) {
return { return {
channelId: message.channelId, channelId: message.channelId,
threadId: null, threadId: null,
threadName: null, threadName: null,
channelName: "name" in channel ? channel.name : null, channelName: "name" in channel ? channel.name : null,
topic,
nsfw: nsfw:
typeof safetyChannel.nsfw === "boolean" typeof safetyChannel.nsfw === "boolean"
? safetyChannel.nsfw ? safetyChannel.nsfw
@@ -133,6 +142,7 @@ export function getMessageLocation(message: Message): MessageLocation {
threadId: channel.id, threadId: channel.id,
threadName: channel.name, threadName: channel.name,
channelName: channel.parent?.name ?? null, channelName: channel.parent?.name ?? null,
topic,
nsfw: nsfw:
typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined, typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
nsfwLevel: nsfwLevel:
@@ -0,0 +1,209 @@
// ═══════════════════════════════════════════════════════════════════════════
// Context enrichment builders — rich <user_reputation> attrs, <user_history>,
// <user_profiles> as_of, bot/edited detection (pure, no DB)
// ═══════════════════════════════════════════════════════════════════════════
import { describe, expect, it } from "vitest";
import {
buildUserHistoryXml,
buildUserProfilesBlock,
formatReputationAttrs,
resolveIsBot,
resolveIsEdited,
} from "../src/modules/ai-moderation/moderationBuilders.js";
import type { MessageRecord } from "../src/modules/message-capture/types.js";
const NOW = 1_800_000_000_000;
function msg(overrides: Partial<MessageRecord> = {}): MessageRecord {
return {
id: "m1",
guild_id: "g1",
channel_id: "c1",
thread_id: null,
user_id: "u1",
username: "user1",
avatar_url: null,
content: "hai",
edited_content: null,
created_at: NOW,
edited_at: null,
deleted_at: null,
type: "text",
is_reply: null,
is_forward: null,
is_crosspost: null,
reference_message_id: null,
reference_channel_id: null,
reference_guild_id: null,
metadata: null,
...overrides,
};
}
const DAY_MS = 24 * 60 * 60 * 1000;
describe("formatReputationAttrs — rich reputation signal", () => {
it("emits trust, infraction count and clean streak", () => {
const attrs = formatReputationAttrs({
trust_score: 62,
total_infractions: 3,
clean_message_streak: 45,
last_infraction_at: null,
});
expect(attrs).toContain('trust_score="62"');
expect(attrs).toContain('total_infractions="3"');
expect(attrs).toContain('clean_streak="45"');
});
it("derives last_offense_days_ago and marks repeat offenders (7-day window)", () => {
const attrs = formatReputationAttrs(
{
trust_score: 50,
total_infractions: 2,
clean_message_streak: 0,
last_infraction_at: NOW - 2 * DAY_MS,
},
NOW,
);
expect(attrs).toContain('last_offense_days_ago="2"');
expect(attrs).toContain('repeat_offender="true"');
});
it("does NOT mark repeat offender when the last offense is older than 7 days", () => {
const attrs = formatReputationAttrs(
{
trust_score: 50,
total_infractions: 2,
clean_message_streak: 10,
last_infraction_at: NOW - 30 * DAY_MS,
},
NOW,
);
expect(attrs).toContain('last_offense_days_ago="30"');
expect(attrs).not.toContain("repeat_offender");
});
it("omits offense-derived attrs when the user has no recorded infraction date", () => {
const attrs = formatReputationAttrs({
trust_score: 85,
total_infractions: 0,
clean_message_streak: 120,
last_infraction_at: null,
});
expect(attrs).not.toContain("last_offense_days_ago");
expect(attrs).not.toContain("repeat_offender");
});
it("clamps a future/skewed timestamp to days_ago=0", () => {
const attrs = formatReputationAttrs(
{
trust_score: 50,
total_infractions: 1,
clean_message_streak: 0,
last_infraction_at: NOW + 5 * DAY_MS,
},
NOW,
);
expect(attrs).toContain('last_offense_days_ago="0"');
});
});
describe("buildUserHistoryXml — last flagged messages for repeat offenders", () => {
it("returns empty when there is no real history", () => {
expect(buildUserHistoryXml([])).toBe("");
expect(
buildUserHistoryXml([{ content: " ", severity: "low", created_at: 1 }]),
).toBe("");
});
it("renders <infraction> rows with severity and recency", () => {
const xml = buildUserHistoryXml(
[
{
content: "beli barang murah disini https://scam.example",
severity: "high",
created_at: NOW - 3 * DAY_MS,
},
],
NOW,
);
expect(xml).toContain("<user_history>");
expect(xml).toContain('severity="high"');
expect(xml).toContain('time_ago_days="3"');
expect(xml).toContain("beli barang murah disini");
});
it("caps long snippets and XML-escapes content", () => {
const xml = buildUserHistoryXml(
[
{
content: "x".repeat(300),
severity: "low",
created_at: NOW - DAY_MS,
},
],
NOW,
);
expect(xml.length).toBeLessThan(250);
});
});
describe("buildUserProfilesBlock — deduplicated map with staleness", () => {
it("emits as_of when the profile has a last-generated timestamp", () => {
const block = buildUserProfilesBlock(
new Map([
[
"u1",
{
text: "Developer teknis, bahasa Indonesia",
asOf: NOW - 3 * DAY_MS,
},
],
]),
);
expect(block).toContain('<user_profile user_id="u1"');
expect(block).toContain(
`as_of="${new Date(NOW - 3 * DAY_MS).toISOString()}"`,
);
expect(block).toContain("Developer teknis");
});
it("omits as_of when absent, and drops empty profiles", () => {
const block = buildUserProfilesBlock(
new Map([
["u1", { text: "profil aktif", asOf: null }],
["u2", { text: " " }],
]),
);
expect(block).toContain('user_id="u1"');
expect(block).not.toContain("as_of");
expect(block).not.toContain("u2");
});
it("returns empty for no profiles", () => {
expect(buildUserProfilesBlock(new Map())).toBe("");
});
});
describe("resolveIsBot / resolveIsEdited — message flags", () => {
it("reads author.bot from captured metadata", () => {
const bot = msg({
metadata: JSON.stringify({
author: { id: "x", username: "bot", bot: true },
}),
});
const human = msg({
metadata: JSON.stringify({
author: { id: "y", username: "user", bot: false },
}),
});
expect(resolveIsBot(bot)).toBe(true);
expect(resolveIsBot(human)).toBe(false);
expect(resolveIsBot(msg())).toBe(false);
});
it("flags edited content only when edited_content is present (the edit path)", () => {
expect(resolveIsEdited(msg({ edited_content: "versi baru" }))).toBe(true);
expect(resolveIsEdited(msg())).toBe(false);
});
});
@@ -197,6 +197,36 @@ describe("buildLocationContext — channel/thread/nsfw enrichment", () => {
expect(line).toContain('age_restricted="false"'); expect(line).toContain('age_restricted="false"');
}); });
it("includes the channel topic (escaped) when captured", () => {
const t = target();
t.metadata = JSON.stringify({
channel: {
channelName: "rules",
topic: "Diskusi coding & programming — no self-promo",
nsfw: false,
},
});
const line = buildLocationContext([t]);
expect(line).toContain(
'topic="Diskusi coding &amp; programming — no self-promo"',
);
});
it("caps an oversized topic and omits empty/absent topic", () => {
const t = target();
t.metadata = JSON.stringify({
channel: { channelName: "general", topic: "x".repeat(500), nsfw: false },
});
const line = buildLocationContext([t]);
const match = line.match(/topic="([^"]*)"/);
expect(match).not.toBeNull();
expect(match?.[1].length).toBeLessThanOrEqual(201);
const t2 = target();
t2.metadata = JSON.stringify({ channel: { channelName: "general" } });
expect(buildLocationContext([t2])).not.toContain("topic=");
});
it("returns empty when no metadata", () => { it("returns empty when no metadata", () => {
expect(buildLocationContext([target()])).toBe(""); expect(buildLocationContext([target()])).toBe("");
}); });
@@ -71,6 +71,9 @@ export interface ChannelRef {
channelName?: string | null; channelName?: string | null;
threadId?: string | null; threadId?: string | null;
threadName?: string | null; threadName?: string | null;
/** Channel topic (captured in gateway metadata.channel.topic). */
topic?: string | null;
nsfw?: boolean;
} }
export interface ReferenceInfo { export interface ReferenceInfo {