From c894e5cd75f6fb2440787beefd0566ea2fdf73bc Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Sat, 30 May 2026 01:02:51 +0700 Subject: [PATCH] feat: expand AI moderation with structured analysis, review workflow, and guardrails - Add structured AI moderation fields (categories, severity, confidence, recommended_action, policy_version, evidence) to messages table - Add moderation_reviews, moderation_actions, and retention_policies tables - Upgrade LLM response parsing to support structured metadata with backwards compatibility for legacy responses - Implement public AI evaluation review UI with decision controls (approve, false positive + reanalyze, escalate) - Add auto-delete guardrails requiring high confidence, severity, and allowed categories; log all attempts to moderation_actions - Add retention manager scaffolding for messages/attachments/voice - Add action executor for moderation actions (mute, warn, kick, ban) - Add review routes: GET/POST/PATCH /api/reviews, GET/POST/PATCH /api/actions - Preserve auth separation: voice/media/recordings gated, review public Co-Authored-By: Claude Opus 4.6 --- frontend/src/api/client.ts | 14 + .../src/components/messages/MessageCard.tsx | 35 ++ .../src/components/review/ReviewPanel.tsx | 177 ++++++++- frontend/src/hooks/useReview.ts | 221 +++++++++++ src/config.ts | 19 +- src/database/schema.ts | 249 ++++++++++++ src/http/app.ts | 2 + src/moderation/actionExecutor.ts | 303 ++++++++++++++ src/moderation/aiAnalyzer.ts | 12 + src/moderation/autoDeleteManager.ts | 130 +++++- src/moderation/llmModerationClient.ts | 86 +++- src/moderation/messageStore.ts | 369 +++++++++++++++++- src/moderation/retentionManager.ts | 179 +++++++++ src/moderation/types.ts | 67 ++++ src/routes/reviewRoutes.ts | 268 +++++++++++++ 15 files changed, 2104 insertions(+), 27 deletions(-) create mode 100644 frontend/src/hooks/useReview.ts create mode 100644 src/moderation/actionExecutor.ts create mode 100644 src/moderation/retentionManager.ts create mode 100644 src/routes/reviewRoutes.ts diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 7b18e69..443d667 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,4 +1,12 @@ export type AIStatus = "pending" | "clean" | "warn" | "flagged" | "error"; +export type AISeverity = "none" | "low" | "medium" | "high" | "critical"; +export type AIRecommendedAction = + | "none" + | "monitor" + | "warn" + | "review" + | "delete" + | "escalate"; export interface MessageRecord { id: string; @@ -20,6 +28,12 @@ export interface MessageRecord { ai_moderation_score?: number | null; ai_moderation_raw?: string | null; ai_analysis?: string | null; + ai_categories?: string | null; + ai_severity?: AISeverity | null; + ai_confidence?: number | null; + ai_recommended_action?: AIRecommendedAction | null; + ai_policy_version?: string | null; + ai_evidence?: string | null; ai_analyzed_at?: number | null; ai_error?: string | null; } diff --git a/frontend/src/components/messages/MessageCard.tsx b/frontend/src/components/messages/MessageCard.tsx index f74627f..1599eff 100644 --- a/frontend/src/components/messages/MessageCard.tsx +++ b/frontend/src/components/messages/MessageCard.tsx @@ -24,9 +24,25 @@ function getAiIcon(status: string) { return null; } +function parseStringList(value?: string | null): string[] { + if (!value) return []; + try { + const parsed = JSON.parse(value) as unknown; + return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : []; + } catch { + return value + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + } +} + export function MessageCard({ message, onReanalyze }: MessageCardProps) { const displayContent = message.edited_content ?? message.content; const aiStatus = message.ai_status ?? "pending"; + const categories = parseStringList(message.ai_categories ?? message.ai_moderation_flags); + const evidence = parseStringList(message.ai_evidence); + const confidence = message.ai_confidence ?? message.ai_moderation_score ?? null; const [isReanalyzing, setIsReanalyzing] = useState(false); const handleReanalyze = async () => { @@ -62,11 +78,30 @@ export function MessageCard({ message, onReanalyze }: MessageCardProps) {

{displayContent || "(empty message)"}

+
+ {message.ai_severity ? severity: {message.ai_severity} : null} + {message.ai_recommended_action ? action: {message.ai_recommended_action} : null} + {confidence != null ? confidence: {Math.round(confidence * 100)}% : null} + {message.ai_policy_version ? policy: {message.ai_policy_version} : null} + {categories.slice(0, 6).map((category) => ( + {category} + ))} +
{message.ai_analysis ? (
{message.ai_analysis}
) : null} + {evidence.length > 0 ? ( +
+
Evidence
+
    + {evidence.slice(0, 4).map((item, index) => ( +
  • {item}
  • + ))} +
+
+ ) : null} {message.ai_error ? (
AI error: {message.ai_error} diff --git a/frontend/src/components/review/ReviewPanel.tsx b/frontend/src/components/review/ReviewPanel.tsx index 5ecaad3..ce8d2f6 100644 --- a/frontend/src/components/review/ReviewPanel.tsx +++ b/frontend/src/components/review/ReviewPanel.tsx @@ -1,6 +1,12 @@ +import { useMemo, useState } from "react"; import type { MessageRecord } from "../../types/messages"; -import { MessageFeed } from "../messages/MessageFeed"; +import { useReview, type ReviewStatus } from "../../hooks/useReview"; +import { MessageCard } from "../messages/MessageCard"; +import { Badge } from "../ui/badge"; +import { Button } from "../ui/button"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "../ui/card"; +import { Input } from "../ui/input"; +import { Select } from "../ui/select"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs"; export interface ReviewPanelProps { @@ -8,39 +14,180 @@ export interface ReviewPanelProps { onReanalyze: (id: string) => void; } +type ReviewFilter = "all" | "warn" | "flagged" | "error"; + +const statusOptions = [ + { value: "all", label: "All reviewable" }, + { value: "warn", label: "Warn" }, + { value: "flagged", label: "Flagged" }, + { value: "error", label: "Errors" }, +]; + +function parseStringList(value?: string | null): string[] { + if (!value) return []; + try { + const parsed = JSON.parse(value) as unknown; + return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : []; + } catch { + return value + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + } +} + +function ReviewDecisionControls({ + message, + onReanalyze, +}: { + message: MessageRecord; + onReanalyze: (id: string) => void; +}) { + const { createReview, loading, error } = useReview(); + const [notes, setNotes] = useState(""); + const [reviewerId, setReviewerId] = useState("public-eval"); + const [savedStatus, setSavedStatus] = useState(null); + + const submitDecision = async (status: ReviewStatus) => { + const review = await createReview({ + message_id: message.id, + guild_id: message.guild_id, + channel_id: message.channel_id, + reviewer_id: reviewerId.trim() || "public-eval", + status, + notes: notes.trim() || null, + reviewed_at: Date.now(), + }); + setSavedStatus(review.status); + if (status === "rejected") { + onReanalyze(message.id); + } + }; + + return ( +
+
+ AI Eval Decision + {savedStatus ? saved: {savedStatus} : null} +
+
+ setReviewerId(event.target.value)} + placeholder="reviewer label" + /> + setNotes(event.target.value)} + placeholder="reason / evaluation note" + /> +
+
+ + + +
+ {error ?
{error}
: null} +
+ ); +} + export function ReviewPanel({ messages, onReanalyze }: ReviewPanelProps) { - const flaggedItems = messages.filter( + const [statusFilter, setStatusFilter] = useState("all"); + const [severityFilter, setSeverityFilter] = useState(""); + const [categoryFilter, setCategoryFilter] = useState(""); + + const reviewable = useMemo( + () => messages.filter((message) => message.ai_status === "warn" || message.ai_status === "flagged" || message.ai_status === "error"), + [messages], + ); + + const categories = useMemo(() => { + const set = new Set(); + for (const message of reviewable) { + for (const category of parseStringList(message.ai_categories ?? message.ai_moderation_flags)) { + set.add(category); + } + } + return Array.from(set).sort(); + }, [reviewable]); + + const filtered = reviewable.filter((message) => { + if (statusFilter !== "all" && message.ai_status !== statusFilter) return false; + if (severityFilter && message.ai_severity !== severityFilter) return false; + if (categoryFilter) { + const messageCategories = parseStringList(message.ai_categories ?? message.ai_moderation_flags); + if (!messageCategories.includes(categoryFilter)) return false; + } + return true; + }); + + const flaggedItems = filtered.filter( (message) => message.ai_status === "warn" || message.ai_status === "flagged", ); - const errorItems = messages.filter((message) => message.ai_status === "error"); + const errorItems = filtered.filter((message) => message.ai_status === "error"); + + const renderList = (items: MessageRecord[], emptyText: string) => ( +
+ {items.length === 0 ? ( +
+ {emptyText} +
+ ) : ( + items.map((message) => ( +
+ + +
+ )) + )} +
+ ); return ( - Needs Review + Moderation Review & AI Eval - {flaggedItems.length} flagged messages, {errorItems.length} analysis errors. + Public AI evaluation queue: {reviewable.length} reviewable messages, {errorItems.length} analysis errors. +
+ setSeverityFilter(event.target.value)} + placeholder="All severities" + options={["none", "low", "medium", "high", "critical"].map((severity) => ({ value: severity, label: severity }))} + /> +