feat(gmw): moderation explainability + semantic message search
- Persist structured verdict (flags/severity/confidence/evidence) on moderation_actions so the public web can show WHY a message was moderated. - Add a persistent Qdrant archive collection (gmw_message_archive); embed every captured message at capture time (fire-and-forget, best-effort). - Public semantic search over the archive (backend oRPC + FE toggle on the messages view). Both features are read-only/public and fully automatic. Migration: 0015_add_moderation_explainability.sql
This commit is contained in:
@@ -178,6 +178,7 @@ export const configSchema = z
|
||||
// embedding column remains as a legacy fallback).
|
||||
QDRANT_URL: z.string().optional(),
|
||||
QDRANT_COLLECTION: z.string().default("gmw_text_moderation"),
|
||||
QDRANT_ARCHIVE_COLLECTION: z.string().default("gmw_message_archive"),
|
||||
QDRANT_API_KEY: z.string().optional(),
|
||||
AI_LLM_MAX_CONCURRENT: z.coerce.number().int().positive().default(8),
|
||||
AI_LLM_IMAGE_MAX_DIMENSION: z.coerce
|
||||
|
||||
@@ -662,6 +662,16 @@ export const pgModerationActionsTable = pgTable(
|
||||
error: pgText("error"),
|
||||
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
|
||||
executed_at: pgBigint("executed_at", { mode: "number" }),
|
||||
// ── Explainability (structured verdict; surfaced read-only to public web) ──
|
||||
flags: pgText("flags"), // JSON array of string flags, e.g. ["sara_agama","vulgar"]
|
||||
categories: pgText("categories"), // JSON array of category strings
|
||||
severity: pgText("severity", {
|
||||
enum: ["none", "low", "medium", "high", "critical"],
|
||||
}),
|
||||
confidence: pgReal("confidence"), // 0..1
|
||||
score: pgReal("score"), // 0..1 raw model score
|
||||
evidence: pgText("evidence"), // JSON array of short quoted snippets
|
||||
policy_version: pgText("policy_version"), // rules.ts policy version string
|
||||
},
|
||||
(table) => ({
|
||||
messageIdIdx: pgIndex("idx_moderation_actions_message_id").on(
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
bigint as pgBigint,
|
||||
boolean as pgBoolean,
|
||||
index as pgIndex,
|
||||
real as pgReal,
|
||||
pgTable,
|
||||
text as pgText,
|
||||
uuid as pgUuid,
|
||||
@@ -48,6 +49,16 @@ export const pgModerationActionsTable = pgTable(
|
||||
error: pgText("error"),
|
||||
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
|
||||
executed_at: pgBigint("executed_at", { mode: "number" }),
|
||||
// ── Explainability (structured verdict; surfaced read-only to public web) ──
|
||||
flags: pgText("flags"), // JSON array of string flags, e.g. ["sara_agama","vulgar"]
|
||||
categories: pgText("categories"), // JSON array of category strings
|
||||
severity: pgText("severity", {
|
||||
enum: ["none", "low", "medium", "high", "critical"],
|
||||
}),
|
||||
confidence: pgReal("confidence"), // 0..1
|
||||
score: pgReal("score"), // 0..1 raw model score
|
||||
evidence: pgText("evidence"), // JSON array of short quoted snippets
|
||||
policy_version: pgText("policy_version"), // rules.ts policy version string
|
||||
},
|
||||
(table) => ({
|
||||
messageIdIdx: pgIndex("idx_moderation_actions_message_id").on(
|
||||
|
||||
Reference in New Issue
Block a user