feat: expand AI moderation with structured analysis, review workflow, and guardrails

- Add structured AI moderation fields (categories, severity, confidence,
  recommended_action, policy_version, evidence) to messages table
- Add moderation_reviews, moderation_actions, and retention_policies tables
- Upgrade LLM response parsing to support structured metadata with backwards
  compatibility for legacy responses
- Implement public AI evaluation review UI with decision controls
  (approve, false positive + reanalyze, escalate)
- Add auto-delete guardrails requiring high confidence, severity, and
  allowed categories; log all attempts to moderation_actions
- Add retention manager scaffolding for messages/attachments/voice
- Add action executor for moderation actions (mute, warn, kick, ban)
- Add review routes: GET/POST/PATCH /api/reviews, GET/POST/PATCH /api/actions
- Preserve auth separation: voice/media/recordings gated, review public

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-05-30 01:02:51 +07:00
co-authored by Claude Opus 4.6
parent b938420eb3
commit c894e5cd75
15 changed files with 2104 additions and 27 deletions
+84 -2
View File
@@ -13,6 +13,16 @@ import type {
} from "./types.js";
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]);
const RecommendedActionSchema = z.enum([
"none",
"monitor",
"warn",
"review",
"delete",
"escalate",
]);
const ModerationResponseSchema = z.object({
results: z.array(
z.object({
@@ -21,6 +31,12 @@ const ModerationResponseSchema = z.object({
flags: z.array(z.string()).catch([]),
score: z.number().catch(0),
analysis: z.string().catch(""),
categories: z.array(z.string()).optional().catch(undefined),
severity: SeveritySchema.optional().catch(undefined),
confidence: z.number().optional().catch(undefined),
recommended_action: RecommendedActionSchema.optional().catch(undefined),
policy_version: z.string().optional().catch(undefined),
evidence: z.array(z.string()).optional().catch(undefined),
}),
),
});
@@ -33,6 +49,31 @@ function hasDeferralAnalysis(analysis: string): boolean {
return DEFERRAL_ANALYSIS_PATTERN.test(analysis);
}
function clampScore(value: number | undefined, fallback = 0): number {
return Math.max(0, Math.min(1, Number.isFinite(value) ? (value as number) : fallback));
}
function deriveSeverity(
status: "clean" | "warn" | "flagged",
score: number,
): z.infer<typeof SeveritySchema> {
if (status === "clean") return "none";
if (status === "warn") return score >= 0.65 ? "medium" : "low";
if (score >= 0.9) return "critical";
return score >= 0.75 ? "high" : "medium";
}
function deriveRecommendedAction(
status: "clean" | "warn" | "flagged",
severity: z.infer<typeof SeveritySchema>,
): z.infer<typeof RecommendedActionSchema> {
if (status === "clean") return "none";
if (status === "warn") return severity === "medium" ? "review" : "warn";
if (severity === "critical") return "escalate";
if (severity === "high") return "delete";
return "review";
}
const openai = new OpenAI({
apiKey: config.AI_LLM_API_KEY,
baseURL: config.AI_LLM_BASE_URL,
@@ -198,7 +239,19 @@ export function parseModerationResponse(
const targetIdSet = new Set(targetIds);
const results: (AnalysisResult | null)[] = response.results.map((result) => {
const { message_id, status, flags, score, analysis } = result;
const {
message_id,
status,
flags,
score,
analysis,
categories,
severity,
confidence,
recommended_action,
policy_version,
evidence,
} = result;
const finalId = message_id.trim();
if (!targetIdSet.has(finalId)) {
@@ -217,12 +270,23 @@ export function parseModerationResponse(
);
}
const normalizedScore = clampScore(score);
const normalizedConfidence = clampScore(confidence, normalizedScore);
const normalizedSeverity = severity ?? deriveSeverity(status, normalizedScore);
return {
messageId: finalId,
status: status as "clean" | "warn" | "flagged",
flags,
score: Math.max(0, Math.min(1, score)),
score: normalizedScore,
analysis,
categories: categories ?? flags,
severity: normalizedSeverity,
confidence: normalizedConfidence,
recommendedAction:
recommended_action ?? deriveRecommendedAction(status, normalizedSeverity),
policyVersion: policy_version ?? "default-2026-05-30",
evidence: evidence ?? [],
};
});
@@ -243,6 +307,12 @@ export function parseModerationResponse(
flags: ["analysis_incomplete"],
score: 0,
analysis: "Analysis incomplete - LLM did not process this message",
categories: ["analysis_incomplete"],
severity: "none",
confidence: 0,
recommendedAction: "review",
policyVersion: "default-2026-05-30",
evidence: [],
});
}
}
@@ -696,6 +766,12 @@ Struktur wajib:
"status": "clean" | "warn" | "flagged",
"flags": [<string array, kosong jika clean>],
"score": <float 0.01.0>,
"categories": [<kategori kebijakan, kosong jika clean>],
"severity": "none" | "low" | "medium" | "high" | "critical",
"confidence": <float 0.01.0>,
"recommended_action": "none" | "monitor" | "warn" | "review" | "delete" | "escalate",
"policy_version": "default-2026-05-30",
"evidence": [<kutipan/evidence singkat dari teks/media/konteks>],
"analysis": "<penjelasan singkat dalam Bahasa Indonesia, maks 2 kalimat>"
}
]
@@ -880,6 +956,12 @@ CRITICAL: "message_id" HARUS berupa STRING (dibungkus tanda kutip ganda). Jangan
flags: ["analysis_parse_failed"],
score: 0,
analysis: `Parsing failed: ${errorMsg}.`,
categories: ["analysis_parse_failed"],
severity: "none",
confidence: 0,
recommendedAction: "review",
policyVersion: "default-2026-05-30",
evidence: [],
}));
}