feat: add warn category for minor rule violations
- Add "warn" status between "clean" and "flagged" for minor violations - Update AI analyzer system prompt with community rules and warn category - Warn: profanity, OOT, tone issues - requires warning but not deletion - Flagged: NSFW, illegal, hacking, scam, harassment, violence, SARA - requires review/deletion - Update types to support warn status in MessageRecord and AIAnalysisUpdate - Update client UI to show three panels: All Messages, Warned, Flagged - Warned messages show in right-top panel for quick review - Flagged messages show in right-bottom panel for moderation action This resolves: - Need to distinguish between minor and severe violations - Moderators can now warn users before taking action - Better moderation workflow with three-tier system Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
@@ -47,7 +47,7 @@
|
||||
<div class="content-card" style="margin-top:18px"><div class="card-title"><h2>Participants</h2><span class="mini">speaking now</span></div><div id="userList" class="participants"></div></div>
|
||||
</section>
|
||||
|
||||
<section id="text" class="tab-content"><div style="display:grid;grid-template-columns:1fr 1fr;gap:16px;height:100%"><div class="content-card"><div class="card-title"><h2>All Messages</h2><span class="mini">all captures</span></div><div id="textList" class="feed"><div class="empty">Select channel to view text captures</div></div></div><div class="content-card"><div class="card-title"><h2>Flagged</h2><span class="mini">AI flagged only</span></div><div id="flaggedList" class="feed"><div class="empty">No flagged messages</div></div></div></div></section>
|
||||
<section id="text" class="tab-content"><div style="display:grid;grid-template-columns:1fr 1fr;gap:16px;height:100%"><div class="content-card"><div class="card-title"><h2>All Messages</h2><span class="mini">all captures</span></div><div id="textList" class="feed"><div class="empty">Select channel to view text captures</div></div></div><div style="display:grid;grid-template-rows:1fr 1fr;gap:16px"><div class="content-card"><div class="card-title"><h2>Warned</h2><span class="mini">minor violations</span></div><div id="warnedList" class="feed"><div class="empty">No warned messages</div></div></div><div class="content-card"><div class="card-title"><h2>Flagged</h2><span class="mini">requires review</span></div><div id="flaggedList" class="feed"><div class="empty">No flagged messages</div></div></div></div></div></section>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
@@ -71,7 +71,7 @@
|
||||
const SAMPLE_RATE = 24000;
|
||||
const CHANNELS = 1;
|
||||
const el = {
|
||||
wsDot: document.getElementById('wsDot'), wsStatusText: document.getElementById('wsStatusText'), activeTabLabel: document.getElementById('activeTabLabel'), errorBox: document.getElementById('errorBox'), guildSelect: document.getElementById('guildSelect'), channelSelect: document.getElementById('channelSelect'), channelFilter: document.getElementById('channelFilter'), joinVoiceBtn: document.getElementById('joinVoiceBtn'), disconnectVoiceBtn: document.getElementById('disconnectVoiceBtn'), voiceStatusText: document.getElementById('voiceStatusText'), voiceStatusNote: document.getElementById('voiceStatusNote'), toggleBtn: document.getElementById('toggleBtn'), listenBtn: document.getElementById('listenBtn'), listenStatus: document.getElementById('listenStatus'), visualizer: document.getElementById('visualizer'), userList: document.getElementById('userList'), textList: document.getElementById('textList'), flaggedList: document.getElementById('flaggedList')
|
||||
wsDot: document.getElementById('wsDot'), wsStatusText: document.getElementById('wsStatusText'), activeTabLabel: document.getElementById('activeTabLabel'), errorBox: document.getElementById('errorBox'), guildSelect: document.getElementById('guildSelect'), channelSelect: document.getElementById('channelSelect'), channelFilter: document.getElementById('channelFilter'), joinVoiceBtn: document.getElementById('joinVoiceBtn'), disconnectVoiceBtn: document.getElementById('disconnectVoiceBtn'), voiceStatusText: document.getElementById('voiceStatusText'), voiceStatusNote: document.getElementById('voiceStatusNote'), toggleBtn: document.getElementById('toggleBtn'), listenBtn: document.getElementById('listenBtn'), listenStatus: document.getElementById('listenStatus'), visualizer: document.getElementById('visualizer'), userList: document.getElementById('userList'), textList: document.getElementById('textList'), warnedList: document.getElementById('warnedList'), flaggedList: document.getElementById('flaggedList')
|
||||
};
|
||||
for (let i = 0; i < 32; i++) { const bar = document.createElement('div'); bar.className = 'bar'; el.visualizer.appendChild(bar); }
|
||||
const bars = [...document.querySelectorAll('.bar')];
|
||||
@@ -130,7 +130,7 @@
|
||||
|
||||
function renderUsers(users) { el.userList.replaceChildren(); if (users.length === 0) return appendEmpty(el.userList, 'No active speakers'); for (const user of users) { const row = document.createElement('div'); row.className = `user-item${user.speaking ? ' speaking' : ''}`; const img = document.createElement('img'); img.src = user.avatar || ''; img.alt = ''; const name = document.createElement('span'); name.textContent = user.username; row.append(img, name); el.userList.appendChild(row); } }
|
||||
async function fetchText() { if (!state.selectedTextChannel) return renderText(); const result = await apiRequest(`/api/messages?channel=${encodeURIComponent(state.selectedTextChannel)}&type=text&limit=80`); state.text = result.data || []; renderText(); }
|
||||
function renderText() { el.textList.replaceChildren(); el.flaggedList.replaceChildren(); if (!state.selectedTextChannel) { appendEmpty(el.textList, 'Select channel to view text captures'); appendEmpty(el.flaggedList, 'No flagged messages'); return; } if (state.text.length === 0) { appendEmpty(el.textList, 'No text captures yet'); appendEmpty(el.flaggedList, 'No flagged messages'); return; } const flaggedMessages = []; for (const msg of state.text) { const metadata = parseMetadata(msg.metadata); const card = document.createElement('article'); card.className = 'event-card'; const head = document.createElement('div'); head.className = 'event-head'; const author = document.createElement('div'); author.className = 'author'; const avatar = document.createElement('div'); avatar.className = 'avatar'; if (msg.avatar_url) { const img = document.createElement('img'); img.src = msg.avatar_url; img.alt = ''; avatar.appendChild(img); } const name = document.createElement('div'); name.className = 'name'; name.textContent = msg.username || msg.user_id; author.append(avatar, name); const time = document.createElement('div'); time.className = 'time'; time.textContent = new Date(msg.created_at).toLocaleString(); head.append(author, time); const text = document.createElement('div'); text.className = 'message-text'; text.textContent = msg.edited_content || msg.content || '(empty message)'; card.append(head, text); appendAIAnalysis(card, msg); appendMedia(card, metadata); const badges = document.createElement('div'); badges.className = 'badges'; if (metadata.reference?.messageId) appendBadge(badges, 'reply', ''); if (msg.thread_id) appendBadge(badges, metadata.channel?.threadName ? `thread: ${metadata.channel.threadName}` : 'thread', ''); if (msg.edited_at) appendBadge(badges, 'edited', 'edit'); if (msg.deleted_at) appendBadge(badges, 'deleted', 'delete'); card.appendChild(badges); el.textList.appendChild(card); if (msg.ai_status === 'flagged') flaggedMessages.push(card.cloneNode(true)); } if (flaggedMessages.length === 0) { appendEmpty(el.flaggedList, 'No flagged messages'); } else { flaggedMessages.forEach((card) => el.flaggedList.appendChild(card)); } }
|
||||
function renderText() { el.textList.replaceChildren(); el.warnedList.replaceChildren(); el.flaggedList.replaceChildren(); if (!state.selectedTextChannel) { appendEmpty(el.textList, 'Select channel to view text captures'); appendEmpty(el.warnedList, 'No warned messages'); appendEmpty(el.flaggedList, 'No flagged messages'); return; } if (state.text.length === 0) { appendEmpty(el.textList, 'No text captures yet'); appendEmpty(el.warnedList, 'No warned messages'); appendEmpty(el.flaggedList, 'No flagged messages'); return; } const warnedMessages = []; const flaggedMessages = []; for (const msg of state.text) { const metadata = parseMetadata(msg.metadata); const card = document.createElement('article'); card.className = 'event-card'; const head = document.createElement('div'); head.className = 'event-head'; const author = document.createElement('div'); author.className = 'author'; const avatar = document.createElement('div'); avatar.className = 'avatar'; if (msg.avatar_url) { const img = document.createElement('img'); img.src = msg.avatar_url; img.alt = ''; avatar.appendChild(img); } const name = document.createElement('div'); name.className = 'name'; name.textContent = msg.username || msg.user_id; author.append(avatar, name); const time = document.createElement('div'); time.className = 'time'; time.textContent = new Date(msg.created_at).toLocaleString(); head.append(author, time); const text = document.createElement('div'); text.className = 'message-text'; text.textContent = msg.edited_content || msg.content || '(empty message)'; card.append(head, text); appendAIAnalysis(card, msg); appendMedia(card, metadata); const badges = document.createElement('div'); badges.className = 'badges'; if (metadata.reference?.messageId) appendBadge(badges, 'reply', ''); if (msg.thread_id) appendBadge(badges, metadata.channel?.threadName ? `thread: ${metadata.channel.threadName}` : 'thread', ''); if (msg.edited_at) appendBadge(badges, 'edited', 'edit'); if (msg.deleted_at) appendBadge(badges, 'deleted', 'delete'); card.appendChild(badges); el.textList.appendChild(card); if (msg.ai_status === 'warn') warnedMessages.push(card.cloneNode(true)); else if (msg.ai_status === 'flagged') flaggedMessages.push(card.cloneNode(true)); } if (warnedMessages.length === 0) { appendEmpty(el.warnedList, 'No warned messages'); } else { warnedMessages.forEach((card) => el.warnedList.appendChild(card)); } if (flaggedMessages.length === 0) { appendEmpty(el.flaggedList, 'No flagged messages'); } else { flaggedMessages.forEach((card) => el.flaggedList.appendChild(card)); } }
|
||||
function appendAIAnalysis(card, msg) { const status = msg.ai_status || 'pending'; const wrap = document.createElement('div'); wrap.className = 'badges'; const badge = document.createElement('span'); badge.className = `badge ${status === 'flagged' ? 'delete' : status === 'clean' ? 'edit' : ''}`; badge.textContent = `AI: ${status}`; wrap.appendChild(badge); if (msg.ai_moderation_flags) { const flags = document.createElement('span'); flags.className = 'badge delete'; try { flags.textContent = JSON.parse(msg.ai_moderation_flags).join(', '); } catch { flags.textContent = msg.ai_moderation_flags; } wrap.appendChild(flags); } card.appendChild(wrap); if (msg.ai_analysis) { const analysis = document.createElement('div'); analysis.className = 'embed-description'; analysis.textContent = msg.ai_analysis; card.appendChild(analysis); } if (msg.ai_error) { const error = document.createElement('div'); error.className = 'embed-description'; error.textContent = `AI error: ${msg.ai_error}`; card.appendChild(error); } }
|
||||
function appendMedia(card, metadata) { const stickers = document.createElement('div'); stickers.className = 'sticker-strip'; for (const sticker of metadata.stickers || []) { const img = document.createElement('img'); img.className = 'sticker-img'; img.src = sticker.url; img.alt = sticker.name; stickers.appendChild(img); } if (stickers.childElementCount) card.appendChild(stickers); const embeds = document.createElement('div'); embeds.className = 'feed'; for (const embed of metadata.embeds || []) { const item = document.createElement('div'); item.className = 'embed-card'; if (embed.title) { const title = document.createElement(embed.url ? 'a' : 'div'); title.className = 'embed-title'; title.textContent = embed.title; if (embed.url) { title.href = embed.url; title.target = '_blank'; title.rel = 'noreferrer'; } item.appendChild(title); } if (embed.description) { const desc = document.createElement('div'); desc.className = 'embed-description'; desc.textContent = embed.description; item.appendChild(desc); } if (embed.image || embed.thumbnail) { const img = document.createElement('img'); img.className = 'embed-image'; img.src = embed.image || embed.thumbnail; img.alt = embed.title || 'embed image'; item.appendChild(img); } embeds.appendChild(item); } if (embeds.childElementCount) card.appendChild(embeds); const attachments = document.createElement('div'); attachments.className = 'attachment-strip'; for (const attachment of metadata.attachments || []) { const link = document.createElement('a'); link.className = 'attachment-chip'; link.href = attachment.url; link.target = '_blank'; link.rel = 'noreferrer'; link.textContent = `${attachment.name} (${(attachment.size / 1024).toFixed(1)}KB)`; attachments.appendChild(link); } if (attachments.childElementCount) card.appendChild(attachments); }
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ interface ChatCompletionResponse {
|
||||
}
|
||||
|
||||
interface LLMAnalysis {
|
||||
status: "clean" | "flagged";
|
||||
status: "clean" | "warn" | "flagged";
|
||||
flags: string[];
|
||||
score: number;
|
||||
analysis: string;
|
||||
@@ -68,7 +68,7 @@ function parseLLMAnalysis(content: string): LLMAnalysis {
|
||||
if (jsonStart >= 0 && jsonEnd > jsonStart) {
|
||||
try {
|
||||
const parsed = JSON.parse(content.slice(jsonStart, jsonEnd + 1));
|
||||
const status = parsed.status === "flagged" ? "flagged" : "clean";
|
||||
const status = parsed.status === "flagged" ? "flagged" : parsed.status === "warn" ? "warn" : "clean";
|
||||
const flags = Array.isArray(parsed.flags) ? parsed.flags.map(String) : [];
|
||||
const score = Math.max(0, Math.min(1, Number(parsed.score) || 0));
|
||||
const analysis = typeof parsed.analysis === "string" ? parsed.analysis : content;
|
||||
@@ -79,7 +79,7 @@ function parseLLMAnalysis(content: string): LLMAnalysis {
|
||||
}
|
||||
|
||||
return {
|
||||
status: /flagged|bahaya|berisiko|toxic|hate|harassment|violence|sexual|self-harm/i.test(content) ? "flagged" : "clean",
|
||||
status: /flagged|bahaya|berisiko|toxic|hate|harassment|violence|sexual|self-harm|illegal|scam|hacking/i.test(content) ? "flagged" : /warn|profanity|oot|tone|sopan/i.test(content) ? "warn" : "clean",
|
||||
flags: [],
|
||||
score: 0,
|
||||
analysis: content.trim() || "Tidak ada analisis dari LLM.",
|
||||
@@ -99,7 +99,25 @@ async function runLLMAnalysis(texts: string[]): Promise<{ results: LLMAnalysis[]
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "Kamu analis moderation Discord. Nilai setiap pesan untuk toxic, harassment, hate, violence, sexual, self-harm, spam, scam, atau unsafe content. Balas JSON array dengan schema: [{\"status\":\"clean|flagged\",\"flags\":[\"...\"],\"score\":0..1,\"analysis\":\"ringkasan singkat Bahasa Indonesia + alasan + aksi disarankan\"}]. Satu JSON object per pesan dalam array.",
|
||||
content: `Kamu moderator Discord komunitas. Analisis setiap pesan dengan 3 kategori:
|
||||
- CLEAN: Pesan normal, tidak melanggar aturan
|
||||
- WARN: Melanggar aturan minor (profanity ringan, OOT, tone kurang sopan) - butuh peringatan tapi tidak dihapus
|
||||
- FLAGGED: Melanggar aturan berat (NSFW, ilegal, hacking, scam, harassment, violence, SARA, gore, spam) - butuh review moderator untuk penghapusan
|
||||
|
||||
Aturan komunitas:
|
||||
1. Jaga Sikap: Bahasa sopan, hormati semua tanpa diskriminasi
|
||||
2. Hindari Konflik: Jangan pancing keributan, selesaikan masalah pribadi
|
||||
3. Sesuai Channel: Jangan OOT (Out of Topic)
|
||||
4. Konten Eksplisit Dilarang: NSFW, ilegal, pornografi, kekerasan, SARA
|
||||
5. Tidak Ada Ruang LGBT: Komunitas tidak toleran terhadap LGBT
|
||||
6. Jaga Privasi: Jangan sebarkan info pribadi
|
||||
7. Profil Sopan: Username, foto, tag harus pantas
|
||||
8. Jangan Spam/Scam: Hoaks, phishing, spam, promosi, judi, referral dilarang
|
||||
9. Pertanyaan Jelas: Langsung ke inti, jangan "Boleh nanya?"
|
||||
10. Diskusi Berkualitas: Jawaban relevan, akurat, tidak menyesatkan
|
||||
|
||||
Balas JSON array dengan schema: [{"status":"clean|warn|flagged","flags":["..."],"score":0..1,"analysis":"ringkasan Bahasa Indonesia + alasan + aksi disarankan"}]
|
||||
Satu JSON object per pesan dalam array.`,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
@@ -124,12 +142,15 @@ async function runLLMAnalysis(texts: string[]): Promise<{ results: LLMAnalysis[]
|
||||
try {
|
||||
const parsed = JSON.parse(content.substring(jsonStart, jsonEnd + 1));
|
||||
if (Array.isArray(parsed)) {
|
||||
results = parsed.map((item: any) => ({
|
||||
status: item.status === "flagged" ? "flagged" : "clean",
|
||||
results = parsed.map((item: any) => {
|
||||
const status = item.status === "flagged" ? "flagged" : item.status === "warn" ? "warn" : "clean";
|
||||
return {
|
||||
status,
|
||||
flags: Array.isArray(item.flags) ? item.flags.map(String) : [],
|
||||
score: Math.max(0, Math.min(1, Number(item.score) || 0)),
|
||||
analysis: typeof item.analysis === "string" ? item.analysis : content,
|
||||
}));
|
||||
};
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Fall through to individual parsing
|
||||
@@ -159,7 +180,7 @@ async function analyzeAndStoreBatch(db: SqliteDatabase, messages: MessageRecord[
|
||||
const result = results[i] || parseLLMAnalysis("");
|
||||
|
||||
const row = updateMessageAIAnalysis(db, message.id, {
|
||||
status: result.status,
|
||||
status: result.status as "pending" | "clean" | "warn" | "flagged" | "error",
|
||||
flags: JSON.stringify(result.flags),
|
||||
score: result.score,
|
||||
raw: JSON.stringify(raw),
|
||||
|
||||
@@ -222,7 +222,7 @@ export function updateAttachmentAsFailedUpload(
|
||||
}
|
||||
|
||||
interface AIAnalysisUpdate {
|
||||
status: "pending" | "clean" | "flagged" | "error";
|
||||
status: "pending" | "clean" | "warn" | "flagged" | "error";
|
||||
flags?: string | null;
|
||||
score?: number | null;
|
||||
raw?: string | null;
|
||||
|
||||
@@ -13,7 +13,7 @@ export interface MessageRecord {
|
||||
deleted_at: number | null;
|
||||
type: "text" | "edited" | "deleted";
|
||||
metadata: string | null;
|
||||
ai_status?: "pending" | "clean" | "flagged" | "error" | null;
|
||||
ai_status?: "pending" | "clean" | "warn" | "flagged" | "error" | null;
|
||||
ai_moderation_flags?: string | null;
|
||||
ai_moderation_score?: number | null;
|
||||
ai_moderation_raw?: string | null;
|
||||
|
||||
Reference in New Issue
Block a user