feat: expand AI moderation with structured analysis, review workflow, and guardrails
- Add structured AI moderation fields (categories, severity, confidence, recommended_action, policy_version, evidence) to messages table - Add moderation_reviews, moderation_actions, and retention_policies tables - Upgrade LLM response parsing to support structured metadata with backwards compatibility for legacy responses - Implement public AI evaluation review UI with decision controls (approve, false positive + reanalyze, escalate) - Add auto-delete guardrails requiring high confidence, severity, and allowed categories; log all attempts to moderation_actions - Add retention manager scaffolding for messages/attachments/voice - Add action executor for moderation actions (mute, warn, kick, ban) - Add review routes: GET/POST/PATCH /api/reviews, GET/POST/PATCH /api/actions - Preserve auth separation: voice/media/recordings gated, review public Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
b938420eb3
commit
c894e5cd75
@@ -1,4 +1,12 @@
|
||||
export type AIStatus = "pending" | "clean" | "warn" | "flagged" | "error";
|
||||
export type AISeverity = "none" | "low" | "medium" | "high" | "critical";
|
||||
export type AIRecommendedAction =
|
||||
| "none"
|
||||
| "monitor"
|
||||
| "warn"
|
||||
| "review"
|
||||
| "delete"
|
||||
| "escalate";
|
||||
|
||||
export interface MessageRecord {
|
||||
id: string;
|
||||
@@ -20,6 +28,12 @@ export interface MessageRecord {
|
||||
ai_moderation_score?: number | null;
|
||||
ai_moderation_raw?: string | null;
|
||||
ai_analysis?: string | null;
|
||||
ai_categories?: string | null;
|
||||
ai_severity?: AISeverity | null;
|
||||
ai_confidence?: number | null;
|
||||
ai_recommended_action?: AIRecommendedAction | null;
|
||||
ai_policy_version?: string | null;
|
||||
ai_evidence?: string | null;
|
||||
ai_analyzed_at?: number | null;
|
||||
ai_error?: string | null;
|
||||
}
|
||||
|
||||
@@ -24,9 +24,25 @@ function getAiIcon(status: string) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseStringList(value?: string | null): string[] {
|
||||
if (!value) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : [];
|
||||
} catch {
|
||||
return value
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
}
|
||||
|
||||
export function MessageCard({ message, onReanalyze }: MessageCardProps) {
|
||||
const displayContent = message.edited_content ?? message.content;
|
||||
const aiStatus = message.ai_status ?? "pending";
|
||||
const categories = parseStringList(message.ai_categories ?? message.ai_moderation_flags);
|
||||
const evidence = parseStringList(message.ai_evidence);
|
||||
const confidence = message.ai_confidence ?? message.ai_moderation_score ?? null;
|
||||
const [isReanalyzing, setIsReanalyzing] = useState(false);
|
||||
|
||||
const handleReanalyze = async () => {
|
||||
@@ -62,11 +78,30 @@ export function MessageCard({ message, onReanalyze }: MessageCardProps) {
|
||||
<p className="whitespace-pre-wrap break-words text-sm leading-6 text-foreground/90">
|
||||
{displayContent || "(empty message)"}
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2 text-xs">
|
||||
{message.ai_severity ? <Badge variant="outline">severity: {message.ai_severity}</Badge> : null}
|
||||
{message.ai_recommended_action ? <Badge variant="outline">action: {message.ai_recommended_action}</Badge> : null}
|
||||
{confidence != null ? <Badge variant="outline">confidence: {Math.round(confidence * 100)}%</Badge> : null}
|
||||
{message.ai_policy_version ? <Badge variant="outline">policy: {message.ai_policy_version}</Badge> : null}
|
||||
{categories.slice(0, 6).map((category) => (
|
||||
<Badge key={category} variant="secondary">{category}</Badge>
|
||||
))}
|
||||
</div>
|
||||
{message.ai_analysis ? (
|
||||
<div className="rounded-xl bg-muted p-3 text-sm text-muted-foreground">
|
||||
{message.ai_analysis}
|
||||
</div>
|
||||
) : null}
|
||||
{evidence.length > 0 ? (
|
||||
<div className="rounded-xl border border-border bg-background/50 p-3 text-xs text-muted-foreground">
|
||||
<div className="mb-1 font-medium text-foreground/80">Evidence</div>
|
||||
<ul className="list-disc space-y-1 pl-4">
|
||||
{evidence.slice(0, 4).map((item, index) => (
|
||||
<li key={`${message.id}-evidence-${index}`}>{item}</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
) : null}
|
||||
{message.ai_error ? (
|
||||
<div className="rounded-xl bg-destructive/10 p-3 text-sm text-destructive">
|
||||
AI error: {message.ai_error}
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import type { MessageRecord } from "../../types/messages";
|
||||
import { MessageFeed } from "../messages/MessageFeed";
|
||||
import { useReview, type ReviewStatus } from "../../hooks/useReview";
|
||||
import { MessageCard } from "../messages/MessageCard";
|
||||
import { Badge } from "../ui/badge";
|
||||
import { Button } from "../ui/button";
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "../ui/card";
|
||||
import { Input } from "../ui/input";
|
||||
import { Select } from "../ui/select";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs";
|
||||
|
||||
export interface ReviewPanelProps {
|
||||
@@ -8,39 +14,180 @@ export interface ReviewPanelProps {
|
||||
onReanalyze: (id: string) => void;
|
||||
}
|
||||
|
||||
type ReviewFilter = "all" | "warn" | "flagged" | "error";
|
||||
|
||||
const statusOptions = [
|
||||
{ value: "all", label: "All reviewable" },
|
||||
{ value: "warn", label: "Warn" },
|
||||
{ value: "flagged", label: "Flagged" },
|
||||
{ value: "error", label: "Errors" },
|
||||
];
|
||||
|
||||
function parseStringList(value?: string | null): string[] {
|
||||
if (!value) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : [];
|
||||
} catch {
|
||||
return value
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
}
|
||||
|
||||
function ReviewDecisionControls({
|
||||
message,
|
||||
onReanalyze,
|
||||
}: {
|
||||
message: MessageRecord;
|
||||
onReanalyze: (id: string) => void;
|
||||
}) {
|
||||
const { createReview, loading, error } = useReview();
|
||||
const [notes, setNotes] = useState("");
|
||||
const [reviewerId, setReviewerId] = useState("public-eval");
|
||||
const [savedStatus, setSavedStatus] = useState<ReviewStatus | null>(null);
|
||||
|
||||
const submitDecision = async (status: ReviewStatus) => {
|
||||
const review = await createReview({
|
||||
message_id: message.id,
|
||||
guild_id: message.guild_id,
|
||||
channel_id: message.channel_id,
|
||||
reviewer_id: reviewerId.trim() || "public-eval",
|
||||
status,
|
||||
notes: notes.trim() || null,
|
||||
reviewed_at: Date.now(),
|
||||
});
|
||||
setSavedStatus(review.status);
|
||||
if (status === "rejected") {
|
||||
onReanalyze(message.id);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="rounded-xl border border-border bg-muted/30 p-3">
|
||||
<div className="mb-2 flex flex-wrap items-center gap-2">
|
||||
<span className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">AI Eval Decision</span>
|
||||
{savedStatus ? <Badge variant="success">saved: {savedStatus}</Badge> : null}
|
||||
</div>
|
||||
<div className="grid gap-2 md:grid-cols-[160px_1fr]">
|
||||
<Input
|
||||
value={reviewerId}
|
||||
onChange={(event) => setReviewerId(event.target.value)}
|
||||
placeholder="reviewer label"
|
||||
/>
|
||||
<Input
|
||||
value={notes}
|
||||
onChange={(event) => setNotes(event.target.value)}
|
||||
placeholder="reason / evaluation note"
|
||||
/>
|
||||
</div>
|
||||
<div className="mt-3 flex flex-wrap gap-2">
|
||||
<Button size="sm" variant="outline" disabled={loading} onClick={() => submitDecision("approved")}>
|
||||
Approve AI
|
||||
</Button>
|
||||
<Button size="sm" variant="outline" disabled={loading} onClick={() => submitDecision("rejected")}>
|
||||
False Positive + Reanalyze
|
||||
</Button>
|
||||
<Button size="sm" variant="outline" disabled={loading} onClick={() => submitDecision("escalated")}>
|
||||
Escalate
|
||||
</Button>
|
||||
</div>
|
||||
{error ? <div className="mt-2 text-xs text-destructive">{error}</div> : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ReviewPanel({ messages, onReanalyze }: ReviewPanelProps) {
|
||||
const flaggedItems = messages.filter(
|
||||
const [statusFilter, setStatusFilter] = useState<ReviewFilter>("all");
|
||||
const [severityFilter, setSeverityFilter] = useState("");
|
||||
const [categoryFilter, setCategoryFilter] = useState("");
|
||||
|
||||
const reviewable = useMemo(
|
||||
() => messages.filter((message) => message.ai_status === "warn" || message.ai_status === "flagged" || message.ai_status === "error"),
|
||||
[messages],
|
||||
);
|
||||
|
||||
const categories = useMemo(() => {
|
||||
const set = new Set<string>();
|
||||
for (const message of reviewable) {
|
||||
for (const category of parseStringList(message.ai_categories ?? message.ai_moderation_flags)) {
|
||||
set.add(category);
|
||||
}
|
||||
}
|
||||
return Array.from(set).sort();
|
||||
}, [reviewable]);
|
||||
|
||||
const filtered = reviewable.filter((message) => {
|
||||
if (statusFilter !== "all" && message.ai_status !== statusFilter) return false;
|
||||
if (severityFilter && message.ai_severity !== severityFilter) return false;
|
||||
if (categoryFilter) {
|
||||
const messageCategories = parseStringList(message.ai_categories ?? message.ai_moderation_flags);
|
||||
if (!messageCategories.includes(categoryFilter)) return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
const flaggedItems = filtered.filter(
|
||||
(message) => message.ai_status === "warn" || message.ai_status === "flagged",
|
||||
);
|
||||
const errorItems = messages.filter((message) => message.ai_status === "error");
|
||||
const errorItems = filtered.filter((message) => message.ai_status === "error");
|
||||
|
||||
const renderList = (items: MessageRecord[], emptyText: string) => (
|
||||
<div className="space-y-3">
|
||||
{items.length === 0 ? (
|
||||
<div className="rounded-xl border border-dashed border-border p-6 text-center text-sm text-muted-foreground">
|
||||
{emptyText}
|
||||
</div>
|
||||
) : (
|
||||
items.map((message) => (
|
||||
<div key={message.id} className="space-y-2">
|
||||
<MessageCard message={message} onReanalyze={onReanalyze} />
|
||||
<ReviewDecisionControls message={message} onReanalyze={onReanalyze} />
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Needs Review</CardTitle>
|
||||
<CardTitle>Moderation Review & AI Eval</CardTitle>
|
||||
<CardDescription>
|
||||
{flaggedItems.length} flagged messages, {errorItems.length} analysis errors.
|
||||
Public AI evaluation queue: {reviewable.length} reviewable messages, {errorItems.length} analysis errors.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="mb-4 grid gap-2 md:grid-cols-3">
|
||||
<Select
|
||||
value={statusFilter}
|
||||
onChange={(event) => setStatusFilter(event.target.value as ReviewFilter)}
|
||||
options={statusOptions}
|
||||
/>
|
||||
<Select
|
||||
value={severityFilter}
|
||||
onChange={(event) => setSeverityFilter(event.target.value)}
|
||||
placeholder="All severities"
|
||||
options={["none", "low", "medium", "high", "critical"].map((severity) => ({ value: severity, label: severity }))}
|
||||
/>
|
||||
<Select
|
||||
value={categoryFilter}
|
||||
onChange={(event) => setCategoryFilter(event.target.value)}
|
||||
placeholder="All categories"
|
||||
options={categories.map((category) => ({ value: category, label: category }))}
|
||||
/>
|
||||
</div>
|
||||
<Tabs defaultValue="flags">
|
||||
<TabsList>
|
||||
<TabsTrigger value="flags">Flags ({flaggedItems.length})</TabsTrigger>
|
||||
<TabsTrigger value="errors">Errors ({errorItems.length})</TabsTrigger>
|
||||
</TabsList>
|
||||
<TabsContent value="flags">
|
||||
<MessageFeed
|
||||
messages={flaggedItems}
|
||||
onReanalyze={onReanalyze}
|
||||
emptyText="No warned or flagged messages."
|
||||
/>
|
||||
{renderList(flaggedItems, "No warned or flagged messages match the filters.")}
|
||||
</TabsContent>
|
||||
<TabsContent value="errors">
|
||||
<MessageFeed
|
||||
messages={errorItems}
|
||||
onReanalyze={onReanalyze}
|
||||
emptyText="No analysis errors."
|
||||
/>
|
||||
{renderList(errorItems, "No analysis errors match the filters.")}
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</CardContent>
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
import { useCallback, useState } from "react";
|
||||
|
||||
export type ReviewStatus = "pending" | "approved" | "rejected" | "escalated";
|
||||
|
||||
export interface MessageReview {
|
||||
id: string;
|
||||
message_id: string;
|
||||
guild_id: string;
|
||||
channel_id: string;
|
||||
reviewer_id: string | null;
|
||||
status: ReviewStatus;
|
||||
notes: string | null;
|
||||
created_at: number;
|
||||
reviewed_at: number | null;
|
||||
}
|
||||
|
||||
export type ModerationActionType =
|
||||
| "delete_message"
|
||||
| "mute_user"
|
||||
| "warn_user"
|
||||
| "kick_user"
|
||||
| "ban_user";
|
||||
|
||||
export interface ModerationAction {
|
||||
id: string;
|
||||
message_id: string | null;
|
||||
user_id: string | null;
|
||||
guild_id: string;
|
||||
action_type: ModerationActionType;
|
||||
reason: string | null;
|
||||
executed_by: string | null;
|
||||
status: "pending" | "executed" | "failed";
|
||||
error: string | null;
|
||||
created_at: number;
|
||||
executed_at: number | null;
|
||||
}
|
||||
|
||||
interface ReviewQuery {
|
||||
guildId?: string;
|
||||
channelId?: string;
|
||||
status?: string[];
|
||||
cursor?: string;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
interface PageResult<T> {
|
||||
data: T[];
|
||||
nextCursor: string | null;
|
||||
}
|
||||
|
||||
export function useReview() {
|
||||
const [reviews, setReviews] = useState<MessageReview[]>([]);
|
||||
const [actions, setActions] = useState<ModerationAction[]>([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [nextCursor, setNextCursor] = useState<string | null>(null);
|
||||
|
||||
const listReviews = useCallback(async (query: ReviewQuery) => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const params = new URLSearchParams();
|
||||
if (query.guildId) params.append("guildId", query.guildId);
|
||||
if (query.channelId) params.append("channelId", query.channelId);
|
||||
if (query.status?.length) params.append("status", query.status.join(","));
|
||||
if (query.cursor) params.append("cursor", query.cursor);
|
||||
params.append("limit", String(query.limit));
|
||||
|
||||
const response = await fetch(`/api/reviews?${params}`);
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const result = (await response.json()) as PageResult<MessageReview>;
|
||||
setReviews(result.data);
|
||||
setNextCursor(result.nextCursor);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Unknown error");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const createReview = useCallback(
|
||||
async (review: Omit<MessageReview, "id" | "created_at">) => {
|
||||
try {
|
||||
const response = await fetch("/api/reviews", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(review),
|
||||
});
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const newReview = (await response.json()) as MessageReview;
|
||||
setReviews((prev) => [newReview, ...prev]);
|
||||
return newReview;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Unknown error";
|
||||
setError(message);
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const updateReview = useCallback(
|
||||
async (
|
||||
id: string,
|
||||
updates: Partial<Omit<MessageReview, "id" | "created_at">>,
|
||||
) => {
|
||||
try {
|
||||
const response = await fetch(`/api/reviews/${id}`, {
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(updates),
|
||||
});
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const updated = (await response.json()) as MessageReview;
|
||||
setReviews((prev) =>
|
||||
prev.map((r) => (r.id === id ? updated : r)),
|
||||
);
|
||||
return updated;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Unknown error";
|
||||
setError(message);
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const listActions = useCallback(
|
||||
async (query: Omit<ReviewQuery, "channelId">) => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const params = new URLSearchParams();
|
||||
if (query.guildId) params.append("guildId", query.guildId);
|
||||
if (query.status?.length) params.append("status", query.status.join(","));
|
||||
if (query.cursor) params.append("cursor", query.cursor);
|
||||
params.append("limit", String(query.limit));
|
||||
|
||||
const response = await fetch(`/api/actions?${params}`);
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const result = (await response.json()) as PageResult<ModerationAction>;
|
||||
setActions(result.data);
|
||||
setNextCursor(result.nextCursor);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Unknown error");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const createAction = useCallback(
|
||||
async (
|
||||
action: Omit<ModerationAction, "id" | "created_at">,
|
||||
) => {
|
||||
try {
|
||||
const response = await fetch("/api/actions", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(action),
|
||||
});
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const newAction = (await response.json()) as ModerationAction;
|
||||
setActions((prev) => [newAction, ...prev]);
|
||||
return newAction;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Unknown error";
|
||||
setError(message);
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const updateAction = useCallback(
|
||||
async (
|
||||
id: string,
|
||||
updates: Partial<Omit<ModerationAction, "id" | "created_at">>,
|
||||
) => {
|
||||
try {
|
||||
const response = await fetch(`/api/actions/${id}`, {
|
||||
method: "PATCH",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(updates),
|
||||
});
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
|
||||
const updated = (await response.json()) as ModerationAction;
|
||||
setActions((prev) =>
|
||||
prev.map((a) => (a.id === id ? updated : a)),
|
||||
);
|
||||
return updated;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Unknown error";
|
||||
setError(message);
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
return {
|
||||
reviews,
|
||||
actions,
|
||||
loading,
|
||||
error,
|
||||
nextCursor,
|
||||
listReviews,
|
||||
createReview,
|
||||
updateReview,
|
||||
listActions,
|
||||
createAction,
|
||||
updateAction,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user