feat(gmw): moderation explainability + semantic message search

- Persist structured verdict (flags/severity/confidence/evidence) on
  moderation_actions so the public web can show WHY a message was moderated.
- Add a persistent Qdrant archive collection (gmw_message_archive); embed
  every captured message at capture time (fire-and-forget, best-effort).
- Public semantic search over the archive (backend oRPC + FE toggle on the
  messages view). Both features are read-only/public and fully automatic.

Migration: 0015_add_moderation_explainability.sql
This commit is contained in:
asepharyana
2026-08-18 15:11:01 +07:00
parent d68f6b653a
commit 1ae19074ee
25 changed files with 1189 additions and 7 deletions
@@ -10,6 +10,7 @@ import {
} from "./autoDeleteEligibility.js";
import { logDeletionToChannel } from "./autoDeleteLogger.js";
import { sendDeletionNotification } from "./autoDeleteNotify.js";
import { verdictToActionFields } from "./verdictToActionFields.js";
const logger = createChildLogger("auto-delete-manager");
@@ -174,6 +175,7 @@ async function logAutoDeleteAttempt(
guild_id: message.guild_id,
action_type: "delete_message",
reason: result.reason,
...verdictToActionFields(message),
executed_by: "auto-delete-manager",
status: result.deleted
? "executed"
@@ -238,6 +240,7 @@ export async function attemptAutoDeleteFlaggedMessage(
action_type: "reset_nickname",
reason:
"nickname melanggar aturan server (offensive_username); pesan dibiarkan",
...verdictToActionFields(message),
executed_by: "auto-delete-manager",
status: resetOk ? "executed" : "failed",
error: resetOk ? null : "nickname_reset_failed",
@@ -50,6 +50,10 @@ function collectionName(): string {
return config.QDRANT_COLLECTION ?? "gmw_text_moderation";
}
/** Persistent archive collection for semantic message search (no TTL). */
export const ARCHIVE_COLLECTION =
config.QDRANT_ARCHIVE_COLLECTION ?? "gmw_message_archive";
function headers(): Record<string, string> {
const h: Record<string, string> = {
"Content-Type": "application/json",
@@ -390,3 +394,131 @@ export async function deleteQdrantPointsByContentHash(
export function isQdrantConfigured(): boolean {
return Boolean(config.QDRANT_URL);
}
// ─── Archive variants (collection-aware, for persistent message search) ───
// These mirror the cache functions but take an explicit collection name so the
// semantic-search archive (gmw_message_archive) can live alongside the
// TTL-bounded automod cache without disturbing it.
/** Ensure an arbitrary collection exists with the right vector size. */
export async function ensureQdrantCollectionV2(
name: string,
vectorSize: number,
): Promise<boolean> {
try {
let existing: {
result?: { config?: { params?: { vectors?: { size?: number } } } };
} | null = null;
try {
existing = (await request("GET", `/collections/${name}`)) as {
result?: { config?: { params?: { vectors?: { size?: number } } } };
} | null;
} catch (error) {
if (!(error instanceof Error) || !error.message.includes("-> 404")) {
throw error;
}
}
const size = existing?.result?.config?.params?.vectors?.size;
if (size === vectorSize) return true;
if (size !== undefined && size !== vectorSize) {
log.warn(
{ collection: name, oldSize: size, newSize: vectorSize },
"Qdrant archive collection vector size changed — recreating collection",
);
await request("DELETE", `/collections/${name}`);
}
await request("PUT", `/collections/${name}`, {
vectors: { size: vectorSize, distance: "Cosine" },
});
return true;
} catch (error) {
log.error(
{
error: error instanceof Error ? error.message : String(error),
collection: name,
},
"Failed to ensure Qdrant archive collection",
);
return false;
}
}
/** Upsert one embedding + payload point into a named collection. */
export async function upsertQdrantPointV2(
name: string,
pointId: number,
vector: number[],
payload: QdrantVerdictPayload,
): Promise<boolean> {
try {
if (!(await ensureQdrantCollectionV2(name, vector.length))) return false;
await request(
"PUT",
`/collections/${name}/points`,
{
points: [{ id: pointId, vector, payload }],
wait: true,
},
30_000,
);
return true;
} catch (error) {
log.warn(
{
error: error instanceof Error ? error.message : String(error),
collection: name,
} as Record<string, unknown>,
"Qdrant archive upsert failed — entry skipped",
);
return false;
}
}
export interface QdrantArchiveHit {
pointId: number;
score: number;
payload: QdrantVerdictPayload;
}
/** Search a named collection for the nearest stored vector. */
export async function searchQdrantV2(
name: string,
vector: number[],
limit: number,
scoreThreshold: number,
): Promise<QdrantArchiveHit[]> {
try {
const json = (await request("POST", `/collections/${name}/points/search`, {
vector,
limit,
score_threshold: scoreThreshold,
with_payload: true,
})) as {
result?: Array<{
id?: number;
score?: number;
payload?: QdrantVerdictPayload;
}>;
};
return (json.result ?? [])
.filter((hit) => hit.payload?.text)
.map((hit) => ({
pointId: hit.id ?? 0,
score: hit.score ?? 0,
payload: hit.payload as QdrantVerdictPayload,
}));
} catch (error) {
log.warn(
{
error: error instanceof Error ? error.message : String(error),
collection: name,
} as Record<string, unknown>,
"Qdrant archive search failed — semantic search skipped",
);
return [];
}
}
@@ -0,0 +1,48 @@
import type { MessageRecord } from "../message-capture/types.js";
/**
* Map a captured message's persisted AI verdict (the `ai_*` columns on
* MessageRecord) into the explainability columns of a moderation action.
*
* This is READ-ONLY structured data — it never changes any enforcement
* decision. It exists so the public web view can show *why* a message was
* moderated, making GMW's automod transparent instead of a black box.
*
* All fields are null-safe: manual actions (e.g. command-handler bans) carry
* no AI verdict, so they simply store nulls and the UI falls back to the
* free-text `reason`.
*/
export function verdictToActionFields(message?: MessageRecord | null): {
flags: string | null;
categories: string | null;
severity: string | null;
confidence: number | null;
score: number | null;
evidence: string | null;
policy_version: string | null;
} {
if (!message) {
return {
flags: null,
categories: null,
severity: null,
confidence: null,
score: null,
evidence: null,
policy_version: null,
};
}
// ai_moderation_flags / ai_categories are stored as JSON-stringified TEXT
// (see messagesAnalysis.buildAIAnalysisSet → stringifyAIList). Pass them
// through verbatim so the backend can JSON.parse them back into arrays.
return {
flags: message.ai_moderation_flags ?? null,
categories: message.ai_categories ?? null,
severity: message.ai_severity ?? null,
confidence: message.ai_confidence ?? null,
score: message.ai_moderation_score ?? null,
evidence: null, // not persisted on MessageRecord; reserved for future use
policy_version: null, // set by caller if a policy version is available
};
}