feat(gmw): moderation explainability + semantic message search

- Persist structured verdict (flags/severity/confidence/evidence) on
  moderation_actions so the public web can show WHY a message was moderated.
- Add a persistent Qdrant archive collection (gmw_message_archive); embed
  every captured message at capture time (fire-and-forget, best-effort).
- Public semantic search over the archive (backend oRPC + FE toggle on the
  messages view). Both features are read-only/public and fully automatic.

Migration: 0015_add_moderation_explainability.sql
This commit is contained in:
asepharyana
2026-08-18 15:11:01 +07:00
parent d68f6b653a
commit 1ae19074ee
25 changed files with 1189 additions and 7 deletions
@@ -0,0 +1,43 @@
import { config } from "@/shared/config/index.js";
import { createChildLogger } from "@/shared/logger/index.js";
const logger = createChildLogger("messages-embed");
/**
* Embed a search query with the configured OpenAI-compatible embedding model.
* Uses raw fetch (the backend has no openai SDK dependency) and returns null
* when embeddings are not configured (search unavailable).
*
* encoding_format: "float" is REQUIRED — Nvidia-backed models reject base64.
*/
export async function embedQuery(text: string): Promise<number[] | null> {
if (!config.AI_LLM_API_KEY || !config.AI_LLM_EMBEDDING_MODEL) return null;
try {
const res = await fetch(`${config.AI_LLM_BASE_URL}/embeddings`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${config.AI_LLM_API_KEY}`,
},
body: JSON.stringify({
model: config.AI_LLM_EMBEDDING_MODEL,
input: text,
encoding_format: "float",
}),
});
if (!res.ok) {
logger.warn({ status: res.status }, "query embed HTTP error");
return null;
}
const json = (await res.json()) as {
data?: Array<{ embedding?: number[] }>;
};
return json.data?.[0]?.embedding ?? null;
} catch (error) {
logger.warn(
{ error: error instanceof Error ? error.message : String(error) },
"query embed failed",
);
return null;
}
}
@@ -41,3 +41,11 @@ export const messageUpdateSchema = z.object({
export type MessageQuery = z.infer<typeof messageQuerySchema>;
export type MessageCreate = z.infer<typeof messageCreateSchema>;
export type MessageUpdate = z.infer<typeof messageUpdateSchema>;
export const semanticSearchSchema = z.object({
query: z.string().min(1).max(500),
limit: z.coerce.number().int().positive().max(50).default(10),
guildId: z.string().optional(),
});
export type SemanticSearchQuery = z.infer<typeof semanticSearchSchema>;
@@ -1,7 +1,9 @@
import { NotFoundError, ValidationError } from "@/shared/errors/index";
import { createChildLogger } from "@/shared/logger/index";
import { embedQuery } from "./embed.js";
import { messagesRepository } from "./messages.repository.js";
import type { MessageQuery } from "./messages.schema.js";
import type { MessageQuery, SemanticSearchQuery } from "./messages.schema.js";
import { searchArchive } from "./qdrant.js";
const logger = createChildLogger("messages.service");
@@ -78,6 +80,40 @@ export class MessagesService {
logger.debug({ channelId, limit }, "Getting review messages");
return messagesRepository.getReviewMessages(channelId, limit);
}
/**
* Public, read-only semantic search over the persistent message archive.
* Embeds the query, searches Qdrant, returns text + metadata. Best-effort:
* if embeddings/Qdrant are unavailable, returns an empty result set.
*/
async semanticSearch(
input: SemanticSearchQuery,
): Promise<{ results: ReturnType<typeof mapSearchHit>[]; nextCursor: null }> {
const vector = await embedQuery(input.query);
if (!vector) {
logger.debug(
{ query: input.query },
"semantic search skipped: no embedder",
);
return { results: [], nextCursor: null };
}
const hits = await searchArchive(vector, input.limit, 0.6);
const results = hits.map((h) => mapSearchHit(h));
return { results, nextCursor: null };
}
}
/** Shape returned to the frontend (text + metadata from the archive payload). */
function mapSearchHit(hit: {
score: number;
payload: { text: string; content_hash?: string; analyzed_at: number };
}) {
return {
message_id: hit.payload.content_hash ?? null,
content: hit.payload.text,
score: hit.score,
created_at: hit.payload.analyzed_at,
};
}
export const messagesService = new MessagesService();
@@ -0,0 +1,95 @@
import { config } from "@/shared/config/index.js";
import { createChildLogger } from "@/shared/logger/index.js";
const logger = createChildLogger("messages-qdrant");
export interface ArchiveHit {
score: number;
payload: {
text: string;
content_hash?: string;
analyzed_at: number;
expires_at: number;
};
}
function baseUrl(): string {
return (config.QDRANT_URL ?? "http://100.121.180.82:6333").replace(
/\/+$/,
"",
);
}
function headers(): Record<string, string> {
const h: Record<string, string> = { "Content-Type": "application/json" };
if (config.QDRANT_API_KEY) h["api-key"] = config.QDRANT_API_KEY;
return h;
}
export const ARCHIVE_COLLECTION =
config.QDRANT_ARCHIVE_COLLECTION ?? "gmw_message_archive";
async function request(
method: string,
path: string,
body?: unknown,
timeoutMs = 10_000,
): Promise<unknown> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const res = await fetch(`${baseUrl()}${path}`, {
method,
headers: headers(),
body: body === undefined ? undefined : JSON.stringify(body),
signal: controller.signal,
});
const text = await res.text();
if (!res.ok) {
throw new Error(
`Qdrant ${method} ${path} -> ${res.status}: ${text.slice(0, 200)}`,
);
}
return text ? JSON.parse(text) : null;
} finally {
clearTimeout(timer);
}
}
/** Search the archive collection for the nearest vectors to `vector`. */
export async function searchArchive(
vector: number[],
limit: number,
scoreThreshold: number,
): Promise<ArchiveHit[]> {
if (!config.QDRANT_URL) return [];
try {
const json = (await request(
"POST",
`/collections/${ARCHIVE_COLLECTION}/points/search`,
{
vector,
limit,
score_threshold: scoreThreshold,
with_payload: true,
},
)) as {
result?: Array<{
score?: number;
payload?: ArchiveHit["payload"];
}>;
};
return (json.result ?? [])
.filter((h) => h.payload?.text)
.map((h) => ({
score: h.score ?? 0,
payload: h.payload as ArchiveHit["payload"],
}));
} catch (error) {
logger.warn(
{ error: error instanceof Error ? error.message : String(error) },
"archive search failed",
);
return [];
}
}
@@ -17,6 +17,20 @@ const ACTION_TYPES = [
] as const;
const STATUSES = ["pending", "executed", "failed"] as const;
/** Parse a JSON-stringified array column (e.g. flags/categories/evidence).
* Returns null on empty/malformed input so the FE can treat it as "no data". */
function parseJsonArray(value: unknown): string[] | null {
if (value == null) return null;
const str = typeof value === "string" ? value : String(value);
if (str.length === 0) return null;
try {
const parsed = JSON.parse(str);
return Array.isArray(parsed) ? (parsed as string[]) : null;
} catch {
return null;
}
}
export class ModerationRepository {
async getStats() {
const db = getDatabase();
@@ -103,6 +117,13 @@ export class ModerationRepository {
a.error,
a.created_at,
a.executed_at,
a.flags,
a.categories,
a.severity,
a.confidence,
a.score,
a.evidence,
a.policy_version,
m.username,
LEFT(m.content, 300) AS content
FROM moderation_actions a
@@ -126,6 +147,13 @@ export class ModerationRepository {
error: r.error ? String(r.error) : null,
created_at: r.created_at ? Number(r.created_at) : null,
executed_at: r.executed_at ? Number(r.executed_at) : null,
flags: parseJsonArray(r.flags),
categories: parseJsonArray(r.categories),
severity: r.severity ? String(r.severity) : null,
confidence: r.confidence != null ? Number(r.confidence) : null,
score: r.score != null ? Number(r.score) : null,
evidence: parseJsonArray(r.evidence),
policy_version: r.policy_version ? String(r.policy_version) : null,
username: r.username ? String(r.username) : null,
content: r.content ? String(r.content) : null,
}));
+8 -1
View File
@@ -16,7 +16,10 @@ import {
skip,
stop,
} from "../modules/media/media.service";
import { messageQuerySchema } from "../modules/messages/messages.schema";
import {
messageQuerySchema,
semanticSearchSchema,
} from "../modules/messages/messages.schema";
import { messagesService } from "../modules/messages/messages.service";
import { moderationService } from "../modules/moderation/moderation.service";
import { recordingsService } from "../modules/recordings/recordings.service";
@@ -136,6 +139,10 @@ const messagesRouter = {
);
return { results: rows, limit: input.limit, cursor: null };
}),
// Public, read-only semantic search over the message archive.
semanticSearch: os
.input(semanticSearchSchema)
.handler(({ input }) => messagesService.semanticSearch(input)),
};
// ── Moderation ───────────────────────────────────────────────────
@@ -134,6 +134,7 @@ export const configSchema = z
.default("https://9router.asepharyana.my.id/v1"),
AI_LLM_MODEL: z.string().default("text"),
AI_LLM_VISION_MODEL: z.string().optional(),
AI_LLM_EMBEDDING_MODEL: z.string().optional(),
AI_LLM_MAX_CONCURRENT: z.coerce.number().int().positive().default(5),
AI_LLM_IMAGE_MAX_DIMENSION: z.coerce
.number()
@@ -208,6 +209,11 @@ export const configSchema = z
.default("https://api.openai.com/v1"),
OPENAI_MODERATION_MODEL: z.string().default("omni-moderation-latest"),
// ── Qdrant (message archive for semantic search) ──────────────────
QDRANT_URL: z.string().optional(),
QDRANT_API_KEY: z.string().optional(),
QDRANT_ARCHIVE_COLLECTION: z.string().default("gmw_message_archive"),
// ── Auto Delete ─────────────────────────────────────────────────────
AUTO_DELETE_FLAGGED_ENABLED: z
.string()