feat(gmw): moderation explainability + semantic message search
- Persist structured verdict (flags/severity/confidence/evidence) on moderation_actions so the public web can show WHY a message was moderated. - Add a persistent Qdrant archive collection (gmw_message_archive); embed every captured message at capture time (fire-and-forget, best-effort). - Public semantic search over the archive (backend oRPC + FE toggle on the messages view). Both features are read-only/public and fully automatic. Migration: 0015_add_moderation_explainability.sql
This commit is contained in:
@@ -10,6 +10,7 @@ import {
|
||||
} from "./autoDeleteEligibility.js";
|
||||
import { logDeletionToChannel } from "./autoDeleteLogger.js";
|
||||
import { sendDeletionNotification } from "./autoDeleteNotify.js";
|
||||
import { verdictToActionFields } from "./verdictToActionFields.js";
|
||||
|
||||
const logger = createChildLogger("auto-delete-manager");
|
||||
|
||||
@@ -174,6 +175,7 @@ async function logAutoDeleteAttempt(
|
||||
guild_id: message.guild_id,
|
||||
action_type: "delete_message",
|
||||
reason: result.reason,
|
||||
...verdictToActionFields(message),
|
||||
executed_by: "auto-delete-manager",
|
||||
status: result.deleted
|
||||
? "executed"
|
||||
@@ -238,6 +240,7 @@ export async function attemptAutoDeleteFlaggedMessage(
|
||||
action_type: "reset_nickname",
|
||||
reason:
|
||||
"nickname melanggar aturan server (offensive_username); pesan dibiarkan",
|
||||
...verdictToActionFields(message),
|
||||
executed_by: "auto-delete-manager",
|
||||
status: resetOk ? "executed" : "failed",
|
||||
error: resetOk ? null : "nickname_reset_failed",
|
||||
|
||||
@@ -50,6 +50,10 @@ function collectionName(): string {
|
||||
return config.QDRANT_COLLECTION ?? "gmw_text_moderation";
|
||||
}
|
||||
|
||||
/** Persistent archive collection for semantic message search (no TTL). */
|
||||
export const ARCHIVE_COLLECTION =
|
||||
config.QDRANT_ARCHIVE_COLLECTION ?? "gmw_message_archive";
|
||||
|
||||
function headers(): Record<string, string> {
|
||||
const h: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
@@ -390,3 +394,131 @@ export async function deleteQdrantPointsByContentHash(
|
||||
export function isQdrantConfigured(): boolean {
|
||||
return Boolean(config.QDRANT_URL);
|
||||
}
|
||||
|
||||
// ─── Archive variants (collection-aware, for persistent message search) ───
|
||||
// These mirror the cache functions but take an explicit collection name so the
|
||||
// semantic-search archive (gmw_message_archive) can live alongside the
|
||||
// TTL-bounded automod cache without disturbing it.
|
||||
|
||||
/** Ensure an arbitrary collection exists with the right vector size. */
|
||||
export async function ensureQdrantCollectionV2(
|
||||
name: string,
|
||||
vectorSize: number,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
let existing: {
|
||||
result?: { config?: { params?: { vectors?: { size?: number } } } };
|
||||
} | null = null;
|
||||
try {
|
||||
existing = (await request("GET", `/collections/${name}`)) as {
|
||||
result?: { config?: { params?: { vectors?: { size?: number } } } };
|
||||
} | null;
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error) || !error.message.includes("-> 404")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const size = existing?.result?.config?.params?.vectors?.size;
|
||||
if (size === vectorSize) return true;
|
||||
|
||||
if (size !== undefined && size !== vectorSize) {
|
||||
log.warn(
|
||||
{ collection: name, oldSize: size, newSize: vectorSize },
|
||||
"Qdrant archive collection vector size changed — recreating collection",
|
||||
);
|
||||
await request("DELETE", `/collections/${name}`);
|
||||
}
|
||||
|
||||
await request("PUT", `/collections/${name}`, {
|
||||
vectors: { size: vectorSize, distance: "Cosine" },
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
log.error(
|
||||
{
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
collection: name,
|
||||
},
|
||||
"Failed to ensure Qdrant archive collection",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Upsert one embedding + payload point into a named collection. */
|
||||
export async function upsertQdrantPointV2(
|
||||
name: string,
|
||||
pointId: number,
|
||||
vector: number[],
|
||||
payload: QdrantVerdictPayload,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
if (!(await ensureQdrantCollectionV2(name, vector.length))) return false;
|
||||
await request(
|
||||
"PUT",
|
||||
`/collections/${name}/points`,
|
||||
{
|
||||
points: [{ id: pointId, vector, payload }],
|
||||
wait: true,
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
return true;
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
{
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
collection: name,
|
||||
} as Record<string, unknown>,
|
||||
"Qdrant archive upsert failed — entry skipped",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface QdrantArchiveHit {
|
||||
pointId: number;
|
||||
score: number;
|
||||
payload: QdrantVerdictPayload;
|
||||
}
|
||||
|
||||
/** Search a named collection for the nearest stored vector. */
|
||||
export async function searchQdrantV2(
|
||||
name: string,
|
||||
vector: number[],
|
||||
limit: number,
|
||||
scoreThreshold: number,
|
||||
): Promise<QdrantArchiveHit[]> {
|
||||
try {
|
||||
const json = (await request("POST", `/collections/${name}/points/search`, {
|
||||
vector,
|
||||
limit,
|
||||
score_threshold: scoreThreshold,
|
||||
with_payload: true,
|
||||
})) as {
|
||||
result?: Array<{
|
||||
id?: number;
|
||||
score?: number;
|
||||
payload?: QdrantVerdictPayload;
|
||||
}>;
|
||||
};
|
||||
|
||||
return (json.result ?? [])
|
||||
.filter((hit) => hit.payload?.text)
|
||||
.map((hit) => ({
|
||||
pointId: hit.id ?? 0,
|
||||
score: hit.score ?? 0,
|
||||
payload: hit.payload as QdrantVerdictPayload,
|
||||
}));
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
{
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
collection: name,
|
||||
} as Record<string, unknown>,
|
||||
"Qdrant archive search failed — semantic search skipped",
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
|
||||
/**
|
||||
* Map a captured message's persisted AI verdict (the `ai_*` columns on
|
||||
* MessageRecord) into the explainability columns of a moderation action.
|
||||
*
|
||||
* This is READ-ONLY structured data — it never changes any enforcement
|
||||
* decision. It exists so the public web view can show *why* a message was
|
||||
* moderated, making GMW's automod transparent instead of a black box.
|
||||
*
|
||||
* All fields are null-safe: manual actions (e.g. command-handler bans) carry
|
||||
* no AI verdict, so they simply store nulls and the UI falls back to the
|
||||
* free-text `reason`.
|
||||
*/
|
||||
export function verdictToActionFields(message?: MessageRecord | null): {
|
||||
flags: string | null;
|
||||
categories: string | null;
|
||||
severity: string | null;
|
||||
confidence: number | null;
|
||||
score: number | null;
|
||||
evidence: string | null;
|
||||
policy_version: string | null;
|
||||
} {
|
||||
if (!message) {
|
||||
return {
|
||||
flags: null,
|
||||
categories: null,
|
||||
severity: null,
|
||||
confidence: null,
|
||||
score: null,
|
||||
evidence: null,
|
||||
policy_version: null,
|
||||
};
|
||||
}
|
||||
|
||||
// ai_moderation_flags / ai_categories are stored as JSON-stringified TEXT
|
||||
// (see messagesAnalysis.buildAIAnalysisSet → stringifyAIList). Pass them
|
||||
// through verbatim so the backend can JSON.parse them back into arrays.
|
||||
return {
|
||||
flags: message.ai_moderation_flags ?? null,
|
||||
categories: message.ai_categories ?? null,
|
||||
severity: message.ai_severity ?? null,
|
||||
confidence: message.ai_confidence ?? null,
|
||||
score: message.ai_moderation_score ?? null,
|
||||
evidence: null, // not persisted on MessageRecord; reserved for future use
|
||||
policy_version: null, // set by caller if a policy version is available
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user