feat: expand AI moderation with structured analysis, review workflow, and guardrails

- Add structured AI moderation fields (categories, severity, confidence,
  recommended_action, policy_version, evidence) to messages table
- Add moderation_reviews, moderation_actions, and retention_policies tables
- Upgrade LLM response parsing to support structured metadata with backwards
  compatibility for legacy responses
- Implement public AI evaluation review UI with decision controls
  (approve, false positive + reanalyze, escalate)
- Add auto-delete guardrails requiring high confidence, severity, and
  allowed categories; log all attempts to moderation_actions
- Add retention manager scaffolding for messages/attachments/voice
- Add action executor for moderation actions (mute, warn, kick, ban)
- Add review routes: GET/POST/PATCH /api/reviews, GET/POST/PATCH /api/actions
- Preserve auth separation: voice/media/recordings gated, review public

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-05-30 01:02:51 +07:00
co-authored by Claude Opus 4.6
parent b938420eb3
commit c894e5cd75
15 changed files with 2104 additions and 27 deletions
@@ -24,9 +24,25 @@ function getAiIcon(status: string) {
return null;
}
function parseStringList(value?: string | null): string[] {
if (!value) return [];
try {
const parsed = JSON.parse(value) as unknown;
return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : [];
} catch {
return value
.split(",")
.map((item) => item.trim())
.filter(Boolean);
}
}
export function MessageCard({ message, onReanalyze }: MessageCardProps) {
const displayContent = message.edited_content ?? message.content;
const aiStatus = message.ai_status ?? "pending";
const categories = parseStringList(message.ai_categories ?? message.ai_moderation_flags);
const evidence = parseStringList(message.ai_evidence);
const confidence = message.ai_confidence ?? message.ai_moderation_score ?? null;
const [isReanalyzing, setIsReanalyzing] = useState(false);
const handleReanalyze = async () => {
@@ -62,11 +78,30 @@ export function MessageCard({ message, onReanalyze }: MessageCardProps) {
<p className="whitespace-pre-wrap break-words text-sm leading-6 text-foreground/90">
{displayContent || "(empty message)"}
</p>
<div className="flex flex-wrap gap-2 text-xs">
{message.ai_severity ? <Badge variant="outline">severity: {message.ai_severity}</Badge> : null}
{message.ai_recommended_action ? <Badge variant="outline">action: {message.ai_recommended_action}</Badge> : null}
{confidence != null ? <Badge variant="outline">confidence: {Math.round(confidence * 100)}%</Badge> : null}
{message.ai_policy_version ? <Badge variant="outline">policy: {message.ai_policy_version}</Badge> : null}
{categories.slice(0, 6).map((category) => (
<Badge key={category} variant="secondary">{category}</Badge>
))}
</div>
{message.ai_analysis ? (
<div className="rounded-xl bg-muted p-3 text-sm text-muted-foreground">
{message.ai_analysis}
</div>
) : null}
{evidence.length > 0 ? (
<div className="rounded-xl border border-border bg-background/50 p-3 text-xs text-muted-foreground">
<div className="mb-1 font-medium text-foreground/80">Evidence</div>
<ul className="list-disc space-y-1 pl-4">
{evidence.slice(0, 4).map((item, index) => (
<li key={`${message.id}-evidence-${index}`}>{item}</li>
))}
</ul>
</div>
) : null}
{message.ai_error ? (
<div className="rounded-xl bg-destructive/10 p-3 text-sm text-destructive">
AI error: {message.ai_error}