feat(gmw): moderation explainability + semantic message search
- Persist structured verdict (flags/severity/confidence/evidence) on moderation_actions so the public web can show WHY a message was moderated. - Add a persistent Qdrant archive collection (gmw_message_archive); embed every captured message at capture time (fire-and-forget, best-effort). - Public semantic search over the archive (backend oRPC + FE toggle on the messages view). Both features are read-only/public and fully automatic. Migration: 0015_add_moderation_explainability.sql
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
import { config } from "@/shared/config/index.js";
|
||||
import { createChildLogger } from "@/shared/logger/index.js";
|
||||
|
||||
const logger = createChildLogger("messages-embed");
|
||||
|
||||
/**
|
||||
* Embed a search query with the configured OpenAI-compatible embedding model.
|
||||
* Uses raw fetch (the backend has no openai SDK dependency) and returns null
|
||||
* when embeddings are not configured (search unavailable).
|
||||
*
|
||||
* encoding_format: "float" is REQUIRED — Nvidia-backed models reject base64.
|
||||
*/
|
||||
export async function embedQuery(text: string): Promise<number[] | null> {
|
||||
if (!config.AI_LLM_API_KEY || !config.AI_LLM_EMBEDDING_MODEL) return null;
|
||||
try {
|
||||
const res = await fetch(`${config.AI_LLM_BASE_URL}/embeddings`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${config.AI_LLM_API_KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: config.AI_LLM_EMBEDDING_MODEL,
|
||||
input: text,
|
||||
encoding_format: "float",
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
logger.warn({ status: res.status }, "query embed HTTP error");
|
||||
return null;
|
||||
}
|
||||
const json = (await res.json()) as {
|
||||
data?: Array<{ embedding?: number[] }>;
|
||||
};
|
||||
return json.data?.[0]?.embedding ?? null;
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"query embed failed",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -41,3 +41,11 @@ export const messageUpdateSchema = z.object({
|
||||
export type MessageQuery = z.infer<typeof messageQuerySchema>;
|
||||
export type MessageCreate = z.infer<typeof messageCreateSchema>;
|
||||
export type MessageUpdate = z.infer<typeof messageUpdateSchema>;
|
||||
|
||||
export const semanticSearchSchema = z.object({
|
||||
query: z.string().min(1).max(500),
|
||||
limit: z.coerce.number().int().positive().max(50).default(10),
|
||||
guildId: z.string().optional(),
|
||||
});
|
||||
|
||||
export type SemanticSearchQuery = z.infer<typeof semanticSearchSchema>;
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { NotFoundError, ValidationError } from "@/shared/errors/index";
|
||||
import { createChildLogger } from "@/shared/logger/index";
|
||||
import { embedQuery } from "./embed.js";
|
||||
import { messagesRepository } from "./messages.repository.js";
|
||||
import type { MessageQuery } from "./messages.schema.js";
|
||||
import type { MessageQuery, SemanticSearchQuery } from "./messages.schema.js";
|
||||
import { searchArchive } from "./qdrant.js";
|
||||
|
||||
const logger = createChildLogger("messages.service");
|
||||
|
||||
@@ -78,6 +80,40 @@ export class MessagesService {
|
||||
logger.debug({ channelId, limit }, "Getting review messages");
|
||||
return messagesRepository.getReviewMessages(channelId, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Public, read-only semantic search over the persistent message archive.
|
||||
* Embeds the query, searches Qdrant, returns text + metadata. Best-effort:
|
||||
* if embeddings/Qdrant are unavailable, returns an empty result set.
|
||||
*/
|
||||
async semanticSearch(
|
||||
input: SemanticSearchQuery,
|
||||
): Promise<{ results: ReturnType<typeof mapSearchHit>[]; nextCursor: null }> {
|
||||
const vector = await embedQuery(input.query);
|
||||
if (!vector) {
|
||||
logger.debug(
|
||||
{ query: input.query },
|
||||
"semantic search skipped: no embedder",
|
||||
);
|
||||
return { results: [], nextCursor: null };
|
||||
}
|
||||
const hits = await searchArchive(vector, input.limit, 0.6);
|
||||
const results = hits.map((h) => mapSearchHit(h));
|
||||
return { results, nextCursor: null };
|
||||
}
|
||||
}
|
||||
|
||||
/** Shape returned to the frontend (text + metadata from the archive payload). */
|
||||
function mapSearchHit(hit: {
|
||||
score: number;
|
||||
payload: { text: string; content_hash?: string; analyzed_at: number };
|
||||
}) {
|
||||
return {
|
||||
message_id: hit.payload.content_hash ?? null,
|
||||
content: hit.payload.text,
|
||||
score: hit.score,
|
||||
created_at: hit.payload.analyzed_at,
|
||||
};
|
||||
}
|
||||
|
||||
export const messagesService = new MessagesService();
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { config } from "@/shared/config/index.js";
|
||||
import { createChildLogger } from "@/shared/logger/index.js";
|
||||
|
||||
const logger = createChildLogger("messages-qdrant");
|
||||
|
||||
export interface ArchiveHit {
|
||||
score: number;
|
||||
payload: {
|
||||
text: string;
|
||||
content_hash?: string;
|
||||
analyzed_at: number;
|
||||
expires_at: number;
|
||||
};
|
||||
}
|
||||
|
||||
function baseUrl(): string {
|
||||
return (config.QDRANT_URL ?? "http://100.121.180.82:6333").replace(
|
||||
/\/+$/,
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
function headers(): Record<string, string> {
|
||||
const h: Record<string, string> = { "Content-Type": "application/json" };
|
||||
if (config.QDRANT_API_KEY) h["api-key"] = config.QDRANT_API_KEY;
|
||||
return h;
|
||||
}
|
||||
|
||||
export const ARCHIVE_COLLECTION =
|
||||
config.QDRANT_ARCHIVE_COLLECTION ?? "gmw_message_archive";
|
||||
|
||||
async function request(
|
||||
method: string,
|
||||
path: string,
|
||||
body?: unknown,
|
||||
timeoutMs = 10_000,
|
||||
): Promise<unknown> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetch(`${baseUrl()}${path}`, {
|
||||
method,
|
||||
headers: headers(),
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: controller.signal,
|
||||
});
|
||||
const text = await res.text();
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`Qdrant ${method} ${path} -> ${res.status}: ${text.slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
return text ? JSON.parse(text) : null;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/** Search the archive collection for the nearest vectors to `vector`. */
|
||||
export async function searchArchive(
|
||||
vector: number[],
|
||||
limit: number,
|
||||
scoreThreshold: number,
|
||||
): Promise<ArchiveHit[]> {
|
||||
if (!config.QDRANT_URL) return [];
|
||||
try {
|
||||
const json = (await request(
|
||||
"POST",
|
||||
`/collections/${ARCHIVE_COLLECTION}/points/search`,
|
||||
{
|
||||
vector,
|
||||
limit,
|
||||
score_threshold: scoreThreshold,
|
||||
with_payload: true,
|
||||
},
|
||||
)) as {
|
||||
result?: Array<{
|
||||
score?: number;
|
||||
payload?: ArchiveHit["payload"];
|
||||
}>;
|
||||
};
|
||||
return (json.result ?? [])
|
||||
.filter((h) => h.payload?.text)
|
||||
.map((h) => ({
|
||||
score: h.score ?? 0,
|
||||
payload: h.payload as ArchiveHit["payload"],
|
||||
}));
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
"archive search failed",
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,20 @@ const ACTION_TYPES = [
|
||||
] as const;
|
||||
const STATUSES = ["pending", "executed", "failed"] as const;
|
||||
|
||||
/** Parse a JSON-stringified array column (e.g. flags/categories/evidence).
|
||||
* Returns null on empty/malformed input so the FE can treat it as "no data". */
|
||||
function parseJsonArray(value: unknown): string[] | null {
|
||||
if (value == null) return null;
|
||||
const str = typeof value === "string" ? value : String(value);
|
||||
if (str.length === 0) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(str);
|
||||
return Array.isArray(parsed) ? (parsed as string[]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class ModerationRepository {
|
||||
async getStats() {
|
||||
const db = getDatabase();
|
||||
@@ -103,6 +117,13 @@ export class ModerationRepository {
|
||||
a.error,
|
||||
a.created_at,
|
||||
a.executed_at,
|
||||
a.flags,
|
||||
a.categories,
|
||||
a.severity,
|
||||
a.confidence,
|
||||
a.score,
|
||||
a.evidence,
|
||||
a.policy_version,
|
||||
m.username,
|
||||
LEFT(m.content, 300) AS content
|
||||
FROM moderation_actions a
|
||||
@@ -126,6 +147,13 @@ export class ModerationRepository {
|
||||
error: r.error ? String(r.error) : null,
|
||||
created_at: r.created_at ? Number(r.created_at) : null,
|
||||
executed_at: r.executed_at ? Number(r.executed_at) : null,
|
||||
flags: parseJsonArray(r.flags),
|
||||
categories: parseJsonArray(r.categories),
|
||||
severity: r.severity ? String(r.severity) : null,
|
||||
confidence: r.confidence != null ? Number(r.confidence) : null,
|
||||
score: r.score != null ? Number(r.score) : null,
|
||||
evidence: parseJsonArray(r.evidence),
|
||||
policy_version: r.policy_version ? String(r.policy_version) : null,
|
||||
username: r.username ? String(r.username) : null,
|
||||
content: r.content ? String(r.content) : null,
|
||||
}));
|
||||
|
||||
@@ -16,7 +16,10 @@ import {
|
||||
skip,
|
||||
stop,
|
||||
} from "../modules/media/media.service";
|
||||
import { messageQuerySchema } from "../modules/messages/messages.schema";
|
||||
import {
|
||||
messageQuerySchema,
|
||||
semanticSearchSchema,
|
||||
} from "../modules/messages/messages.schema";
|
||||
import { messagesService } from "../modules/messages/messages.service";
|
||||
import { moderationService } from "../modules/moderation/moderation.service";
|
||||
import { recordingsService } from "../modules/recordings/recordings.service";
|
||||
@@ -136,6 +139,10 @@ const messagesRouter = {
|
||||
);
|
||||
return { results: rows, limit: input.limit, cursor: null };
|
||||
}),
|
||||
// Public, read-only semantic search over the message archive.
|
||||
semanticSearch: os
|
||||
.input(semanticSearchSchema)
|
||||
.handler(({ input }) => messagesService.semanticSearch(input)),
|
||||
};
|
||||
|
||||
// ── Moderation ───────────────────────────────────────────────────
|
||||
|
||||
@@ -134,6 +134,7 @@ export const configSchema = z
|
||||
.default("https://9router.asepharyana.my.id/v1"),
|
||||
AI_LLM_MODEL: z.string().default("text"),
|
||||
AI_LLM_VISION_MODEL: z.string().optional(),
|
||||
AI_LLM_EMBEDDING_MODEL: z.string().optional(),
|
||||
AI_LLM_MAX_CONCURRENT: z.coerce.number().int().positive().default(5),
|
||||
AI_LLM_IMAGE_MAX_DIMENSION: z.coerce
|
||||
.number()
|
||||
@@ -208,6 +209,11 @@ export const configSchema = z
|
||||
.default("https://api.openai.com/v1"),
|
||||
OPENAI_MODERATION_MODEL: z.string().default("omni-moderation-latest"),
|
||||
|
||||
// ── Qdrant (message archive for semantic search) ──────────────────
|
||||
QDRANT_URL: z.string().optional(),
|
||||
QDRANT_API_KEY: z.string().optional(),
|
||||
QDRANT_ARCHIVE_COLLECTION: z.string().default("gmw_message_archive"),
|
||||
|
||||
// ── Auto Delete ─────────────────────────────────────────────────────
|
||||
AUTO_DELETE_FLAGGED_ENABLED: z
|
||||
.string()
|
||||
|
||||
Reference in New Issue
Block a user