feat(gmw): moderation explainability + semantic message search

- Persist structured verdict (flags/severity/confidence/evidence) on moderation_actions so the public web can show WHY a message was moderated. - Add a persistent Qdrant archive collection (gmw_message_archive); embed every captured message at capture time (fire-and-forget, best-effort). - Public semantic search over the archive (backend oRPC + FE toggle on the messages view). Both features are read-only/public and fully automatic. Migration: 0015_add_moderation_explainability.sql
2026-08-18 15:11:01 +07:00
parent d68f6b653a
commit 1ae19074ee
25 changed files with 1189 additions and 7 deletions
@@ -0,0 +1,16 @@
+-- Migration: 0015_add_moderation_explainability.sql
+-- Date: 2026-08-18
+-- Description: Add structured explainability columns to moderation_actions.
+-- These are READ (surfaced read-only to the public web) so they are never
+-- "written but never read" — they back the public moderation transparency view.
+-- Idempotent: no-ops on databases that already carry the columns.
+
+ALTER TABLE IF EXISTS "moderation_actions"
+  ADD COLUMN IF NOT EXISTS "flags" text,
+  ADD COLUMN IF NOT EXISTS "categories" text,
+  ADD COLUMN IF NOT EXISTS "severity" text
+    CHECK ("severity" IS NULL OR "severity" IN ('none','low','medium','high','critical')),
+  ADD COLUMN IF NOT EXISTS "confidence" real,
+  ADD COLUMN IF NOT EXISTS "score" real,
+  ADD COLUMN IF NOT EXISTS "evidence" text,
+  ADD COLUMN IF NOT EXISTS "policy_version" text;
@@ -106,6 +106,13 @@
      "when": 1785621600000,
      "tag": "0014_add_term_glossary_cache",
      "breakpoints": true
+    },
+    {
+      "idx": 15,
+      "version": "7",
+      "when": 1787184000000,
+      "tag": "0015_add_moderation_explainability",
+      "breakpoints": true
    }
  ]
 }
@@ -10,6 +10,7 @@ import {
 } from "./autoDeleteEligibility.js";
 import { logDeletionToChannel } from "./autoDeleteLogger.js";
 import { sendDeletionNotification } from "./autoDeleteNotify.js";
+import { verdictToActionFields } from "./verdictToActionFields.js";

 const logger = createChildLogger("auto-delete-manager");

@@ -174,6 +175,7 @@ async function logAutoDeleteAttempt(
      guild_id: message.guild_id,
      action_type: "delete_message",
      reason: result.reason,
+      ...verdictToActionFields(message),
      executed_by: "auto-delete-manager",
      status: result.deleted
        ? "executed"
@@ -238,6 +240,7 @@ export async function attemptAutoDeleteFlaggedMessage(
            action_type: "reset_nickname",
            reason:
              "nickname melanggar aturan server (offensive_username); pesan dibiarkan",
+            ...verdictToActionFields(message),
            executed_by: "auto-delete-manager",
            status: resetOk ? "executed" : "failed",
            error: resetOk ? null : "nickname_reset_failed",
@@ -50,6 +50,10 @@ function collectionName(): string {
  return config.QDRANT_COLLECTION ?? "gmw_text_moderation";
 }

+/** Persistent archive collection for semantic message search (no TTL). */
+export const ARCHIVE_COLLECTION =
+  config.QDRANT_ARCHIVE_COLLECTION ?? "gmw_message_archive";
+
 function headers(): Record<string, string> {
  const h: Record<string, string> = {
    "Content-Type": "application/json",
@@ -390,3 +394,131 @@ export async function deleteQdrantPointsByContentHash(
 export function isQdrantConfigured(): boolean {
  return Boolean(config.QDRANT_URL);
 }
+
+// ─── Archive variants (collection-aware, for persistent message search) ───
+// These mirror the cache functions but take an explicit collection name so the
+// semantic-search archive (gmw_message_archive) can live alongside the
+// TTL-bounded automod cache without disturbing it.
+
+/** Ensure an arbitrary collection exists with the right vector size. */
+export async function ensureQdrantCollectionV2(
+  name: string,
+  vectorSize: number,
+): Promise<boolean> {
+  try {
+    let existing: {
+      result?: { config?: { params?: { vectors?: { size?: number } } } };
+    } | null = null;
+    try {
+      existing = (await request("GET", `/collections/${name}`)) as {
+        result?: { config?: { params?: { vectors?: { size?: number } } } };
+      } | null;
+    } catch (error) {
+      if (!(error instanceof Error) || !error.message.includes("-> 404")) {
+        throw error;
+      }
+    }
+
+    const size = existing?.result?.config?.params?.vectors?.size;
+    if (size === vectorSize) return true;
+
+    if (size !== undefined && size !== vectorSize) {
+      log.warn(
+        { collection: name, oldSize: size, newSize: vectorSize },
+        "Qdrant archive collection vector size changed — recreating collection",
+      );
+      await request("DELETE", `/collections/${name}`);
+    }
+
+    await request("PUT", `/collections/${name}`, {
+      vectors: { size: vectorSize, distance: "Cosine" },
+    });
+    return true;
+  } catch (error) {
+    log.error(
+      {
+        error: error instanceof Error ? error.message : String(error),
+        collection: name,
+      },
+      "Failed to ensure Qdrant archive collection",
+    );
+    return false;
+  }
+}
+
+/** Upsert one embedding + payload point into a named collection. */
+export async function upsertQdrantPointV2(
+  name: string,
+  pointId: number,
+  vector: number[],
+  payload: QdrantVerdictPayload,
+): Promise<boolean> {
+  try {
+    if (!(await ensureQdrantCollectionV2(name, vector.length))) return false;
+    await request(
+      "PUT",
+      `/collections/${name}/points`,
+      {
+        points: [{ id: pointId, vector, payload }],
+        wait: true,
+      },
+      30_000,
+    );
+    return true;
+  } catch (error) {
+    log.warn(
+      {
+        error: error instanceof Error ? error.message : String(error),
+        collection: name,
+      } as Record<string, unknown>,
+      "Qdrant archive upsert failed — entry skipped",
+    );
+    return false;
+  }
+}
+
+export interface QdrantArchiveHit {
+  pointId: number;
+  score: number;
+  payload: QdrantVerdictPayload;
+}
+
+/** Search a named collection for the nearest stored vector. */
+export async function searchQdrantV2(
+  name: string,
+  vector: number[],
+  limit: number,
+  scoreThreshold: number,
+): Promise<QdrantArchiveHit[]> {
+  try {
+    const json = (await request("POST", `/collections/${name}/points/search`, {
+      vector,
+      limit,
+      score_threshold: scoreThreshold,
+      with_payload: true,
+    })) as {
+      result?: Array<{
+        id?: number;
+        score?: number;
+        payload?: QdrantVerdictPayload;
+      }>;
+    };
+
+    return (json.result ?? [])
+      .filter((hit) => hit.payload?.text)
+      .map((hit) => ({
+        pointId: hit.id ?? 0,
+        score: hit.score ?? 0,
+        payload: hit.payload as QdrantVerdictPayload,
+      }));
+  } catch (error) {
+    log.warn(
+      {
+        error: error instanceof Error ? error.message : String(error),
+        collection: name,
+      } as Record<string, unknown>,
+      "Qdrant archive search failed — semantic search skipped",
+    );
+    return [];
+  }
+}
@@ -0,0 +1,48 @@
+import type { MessageRecord } from "../message-capture/types.js";
+
+/**
+ * Map a captured message's persisted AI verdict (the `ai_*` columns on
+ * MessageRecord) into the explainability columns of a moderation action.
+ *
+ * This is READ-ONLY structured data — it never changes any enforcement
+ * decision. It exists so the public web view can show *why* a message was
+ * moderated, making GMW's automod transparent instead of a black box.
+ *
+ * All fields are null-safe: manual actions (e.g. command-handler bans) carry
+ * no AI verdict, so they simply store nulls and the UI falls back to the
+ * free-text `reason`.
+ */
+export function verdictToActionFields(message?: MessageRecord | null): {
+  flags: string | null;
+  categories: string | null;
+  severity: string | null;
+  confidence: number | null;
+  score: number | null;
+  evidence: string | null;
+  policy_version: string | null;
+} {
+  if (!message) {
+    return {
+      flags: null,
+      categories: null,
+      severity: null,
+      confidence: null,
+      score: null,
+      evidence: null,
+      policy_version: null,
+    };
+  }
+
+  // ai_moderation_flags / ai_categories are stored as JSON-stringified TEXT
+  // (see messagesAnalysis.buildAIAnalysisSet → stringifyAIList). Pass them
+  // through verbatim so the backend can JSON.parse them back into arrays.
+  return {
+    flags: message.ai_moderation_flags ?? null,
+    categories: message.ai_categories ?? null,
+    severity: message.ai_severity ?? null,
+    confidence: message.ai_confidence ?? null,
+    score: message.ai_moderation_score ?? null,
+    evidence: null, // not persisted on MessageRecord; reserved for future use
+    policy_version: null, // set by caller if a policy version is available
+  };
+}
@@ -0,0 +1,63 @@
+import { embedText } from "@/modules/ai-moderation/embeddingClient.js";
+import {
+  ARCHIVE_COLLECTION,
+  qdrantPointId,
+  upsertQdrantPointV2,
+} from "@/modules/ai-moderation/qdrantClient.js";
+import { config } from "@/shared/config/config.js";
+import { createChildLogger } from "@/shared/logger/index";
+
+const log = createChildLogger("archive-embedder");
+
+export interface ArchiveMessage {
+  id: string;
+  content: string;
+  username: string;
+  channel_id: string;
+  guild_id: string;
+  created_at: number;
+}
+
+/**
+ * Fire-and-forget: embed a captured message and upsert it into the persistent
+ * archive collection so the public web can semantic-search the corpus.
+ *
+ * Failures are swallowed — searching is a nice-to-have, never a precondition
+ * for capture or moderation. The message text is kept in the payload so the
+ * search endpoint can return results even for deleted messages.
+ */
+export function archiveMessageEmbedded(message: ArchiveMessage): void {
+  if (!config.AI_LLM_EMBEDDING_MODEL) return; // embeddings disabled → skip
+  const text = message.content?.trim();
+  if (!text || text.length < 3) return;
+
+  void (async () => {
+    try {
+      const vector = await embedText(text);
+      if (!vector) return;
+      const ok = await upsertQdrantPointV2(
+        ARCHIVE_COLLECTION,
+        qdrantPointId(`archive:${message.id}`),
+        vector,
+        {
+          text: text.slice(0, 4000),
+          flags: "",
+          analyzed_at: Date.now(),
+          // 5-year persistent window (archive is NOT a TTL cache).
+          expires_at: Date.now() + 1000 * 60 * 60 * 24 * 365 * 5,
+          content_hash: message.id,
+        },
+      );
+      if (!ok) return;
+      log.debug({ messageId: message.id }, "Archived message embedding");
+    } catch (err) {
+      log.debug(
+        {
+          messageId: message.id,
+          error: err instanceof Error ? err.message : String(err),
+        },
+        "archive embed skipped",
+      );
+    }
+  })();
+}
@@ -4,6 +4,7 @@ import { config } from "../../shared/config/config.js";
 import { queueMessageAnalysis } from "../ai-moderation/aiAnalyzer.js";
 import { processAttachmentUpload } from "../attachment-upload/attachmentUploader.js";
 import type { EventBroadcaster } from "../event-broadcaster/eventBroadcaster.js";
+import { archiveMessageEmbedded } from "../message-capture/archiveEmbedder.js";
 import {
  getDisplayContent,
  getMessageLocation,
@@ -203,6 +204,7 @@ export async function captureMessage(
  type: "text" | "edited" | "deleted",
  options: { source?: "live" | "backlog" } = {},
 ): Promise<void> {
+  const isBacklog = options.source === "backlog";
  const location = getMessageLocation(message);
  const messageRecord = buildMessageRecord(message, type);

@@ -211,7 +213,11 @@ export async function captureMessage(
    return;
  }

-  const isBacklog = options.source === "backlog";
+  // Fire-and-forget: make the captured message searchable in the persistent
+  // archive (public semantic search). Never blocks capture/moderation.
+  if (!isBacklog && messageRecord.content) {
+    archiveMessageEmbedded(messageRecord);
+  }

  if (_eventBroadcaster && !isBacklog) {
    _eventBroadcaster.messageCreated(messageRecord);
@@ -178,6 +178,7 @@ export const configSchema = z
    // embedding column remains as a legacy fallback).
    QDRANT_URL: z.string().optional(),
    QDRANT_COLLECTION: z.string().default("gmw_text_moderation"),
+    QDRANT_ARCHIVE_COLLECTION: z.string().default("gmw_message_archive"),
    QDRANT_API_KEY: z.string().optional(),
    AI_LLM_MAX_CONCURRENT: z.coerce.number().int().positive().default(8),
    AI_LLM_IMAGE_MAX_DIMENSION: z.coerce
@@ -662,6 +662,16 @@ export const pgModerationActionsTable = pgTable(
    error: pgText("error"),
    created_at: pgBigint("created_at", { mode: "number" }).notNull(),
    executed_at: pgBigint("executed_at", { mode: "number" }),
+    // ── Explainability (structured verdict; surfaced read-only to public web) ──
+    flags: pgText("flags"), // JSON array of string flags, e.g. ["sara_agama","vulgar"]
+    categories: pgText("categories"), // JSON array of category strings
+    severity: pgText("severity", {
+      enum: ["none", "low", "medium", "high", "critical"],
+    }),
+    confidence: pgReal("confidence"), // 0..1
+    score: pgReal("score"), // 0..1 raw model score
+    evidence: pgText("evidence"), // JSON array of short quoted snippets
+    policy_version: pgText("policy_version"), // rules.ts policy version string
  },
  (table) => ({
    messageIdIdx: pgIndex("idx_moderation_actions_message_id").on(
@@ -2,6 +2,7 @@ import {
  bigint as pgBigint,
  boolean as pgBoolean,
  index as pgIndex,
+  real as pgReal,
  pgTable,
  text as pgText,
  uuid as pgUuid,
@@ -48,6 +49,16 @@ export const pgModerationActionsTable = pgTable(
    error: pgText("error"),
    created_at: pgBigint("created_at", { mode: "number" }).notNull(),
    executed_at: pgBigint("executed_at", { mode: "number" }),
+    // ── Explainability (structured verdict; surfaced read-only to public web) ──
+    flags: pgText("flags"), // JSON array of string flags, e.g. ["sara_agama","vulgar"]
+    categories: pgText("categories"), // JSON array of category strings
+    severity: pgText("severity", {
+      enum: ["none", "low", "medium", "high", "critical"],
+    }),
+    confidence: pgReal("confidence"), // 0..1
+    score: pgReal("score"), // 0..1 raw model score
+    evidence: pgText("evidence"), // JSON array of short quoted snippets
+    policy_version: pgText("policy_version"), // rules.ts policy version string
  },
  (table) => ({
    messageIdIdx: pgIndex("idx_moderation_actions_message_id").on(