feat: expand AI moderation with structured analysis, review workflow, and guardrails
- Add structured AI moderation fields (categories, severity, confidence, recommended_action, policy_version, evidence) to messages table - Add moderation_reviews, moderation_actions, and retention_policies tables - Upgrade LLM response parsing to support structured metadata with backwards compatibility for legacy responses - Implement public AI evaluation review UI with decision controls (approve, false positive + reanalyze, escalate) - Add auto-delete guardrails requiring high confidence, severity, and allowed categories; log all attempts to moderation_actions - Add retention manager scaffolding for messages/attachments/voice - Add action executor for moderation actions (mute, warn, kick, ban) - Add review routes: GET/POST/PATCH /api/reviews, GET/POST/PATCH /api/actions - Preserve auth separation: voice/media/recordings gated, review public Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
b938420eb3
commit
c894e5cd75
@@ -4,6 +4,14 @@ import type {
|
||||
} from "./broadcaster.js";
|
||||
|
||||
export type AIStatus = "pending" | "clean" | "warn" | "flagged" | "error";
|
||||
export type AISeverity = "none" | "low" | "medium" | "high" | "critical";
|
||||
export type AIRecommendedAction =
|
||||
| "none"
|
||||
| "monitor"
|
||||
| "warn"
|
||||
| "review"
|
||||
| "delete"
|
||||
| "escalate";
|
||||
|
||||
export type { BroadcasterClient, ModerationBroadcaster };
|
||||
|
||||
@@ -27,6 +35,12 @@ export interface MessageRecord {
|
||||
ai_moderation_score?: number | null;
|
||||
ai_moderation_raw?: string | null;
|
||||
ai_analysis?: string | null;
|
||||
ai_categories?: string | null;
|
||||
ai_severity?: AISeverity | null;
|
||||
ai_confidence?: number | null;
|
||||
ai_recommended_action?: AIRecommendedAction | null;
|
||||
ai_policy_version?: string | null;
|
||||
ai_evidence?: string | null;
|
||||
ai_analyzed_at?: number | null;
|
||||
ai_error?: string | null;
|
||||
}
|
||||
@@ -93,6 +107,12 @@ export interface AnalysisResult {
|
||||
flags: string[];
|
||||
score: number;
|
||||
analysis: string;
|
||||
categories?: string[];
|
||||
severity?: AISeverity;
|
||||
confidence?: number;
|
||||
recommendedAction?: AIRecommendedAction;
|
||||
policyVersion?: string;
|
||||
evidence?: string[];
|
||||
}
|
||||
|
||||
export type MediaMode = "music" | "screen";
|
||||
@@ -145,3 +165,50 @@ export interface AnalysisQueueStatus {
|
||||
individualCircuitBreakerActive: boolean;
|
||||
lastError: string | null;
|
||||
}
|
||||
|
||||
export type ReviewStatus = "pending" | "approved" | "rejected" | "escalated";
|
||||
|
||||
export interface MessageReview {
|
||||
id: string;
|
||||
message_id: string;
|
||||
guild_id: string;
|
||||
channel_id: string;
|
||||
reviewer_id: string | null;
|
||||
status: ReviewStatus;
|
||||
notes: string | null;
|
||||
created_at: number;
|
||||
reviewed_at: number | null;
|
||||
}
|
||||
|
||||
export type ModerationActionType =
|
||||
| "delete_message"
|
||||
| "mute_user"
|
||||
| "warn_user"
|
||||
| "kick_user"
|
||||
| "ban_user";
|
||||
|
||||
export interface ModerationAction {
|
||||
id: string;
|
||||
message_id: string | null;
|
||||
user_id: string | null;
|
||||
guild_id: string;
|
||||
action_type: ModerationActionType;
|
||||
reason: string | null;
|
||||
executed_by: string | null;
|
||||
status: "pending" | "executed" | "failed";
|
||||
error: string | null;
|
||||
created_at: number;
|
||||
executed_at: number | null;
|
||||
}
|
||||
|
||||
export interface RetentionPolicy {
|
||||
id: string;
|
||||
guild_id: string;
|
||||
channel_id: string | null;
|
||||
retention_days: number;
|
||||
apply_to_media: boolean;
|
||||
apply_to_voice: boolean;
|
||||
enabled: boolean;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user