From 4117f83c1b9a9f46d4cdc28bd3f1c7bca366b116 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Mon, 1 Jun 2026 18:14:32 +0700 Subject: [PATCH] fix(dashboard): infinite scroll, auto-refetch, analytics cache invalidation, remove topic row cap Frontend: - Cursor pagination (100/page) replacing hardcoded 80 message limit - Removed .slice(0,200) cap on mergeMessages - IntersectionObserver infinite scroll with skeleton loading - 15s periodic refetch for message list sync Backend: - Removed LIMIT 2000 from topic trends SQL query - Added invalidateAnalyticsCache on message capture (messageCreated) - Added invalidateAnalyticsCache on batch analysis completion Co-Authored-By: Claude Opus 4.8 --- frontend/src/App.tsx | 12 ++++ .../messages/components/MessageFeed.tsx | 35 ++++++++++- .../features/messages/hooks/useMessages.ts | 60 ++++++++++++++++--- frontend/src/features/messages/index.tsx | 15 ++++- src/moderation/aiAnalyzer.ts | 2 + src/moderation/analyticsStore.ts | 5 +- src/moderation/llmModerationClient.ts | 3 +- src/moderation/messageCapture.ts | 4 ++ 8 files changed, 121 insertions(+), 15 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7c0692b..d894c28 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -77,6 +77,15 @@ export default function App() { useEffect(() => { if (monitorGuildId) voice.loadTextTargets(monitorGuildId).catch(() => undefined); }, [monitorGuildId, voice.loadTextTargets]); useEffect(() => { if (selectedTextChannel) messages.fetchMessages(selectedTextChannel).catch(() => undefined); }, [selectedTextChannel, messages.fetchMessages]); + // Periodic refetch — ensures dashboard stays in sync even if WS events were missed + useEffect(() => { + if (!selectedTextChannel) return; + const interval = setInterval(() => { + messages.fetchMessages(selectedTextChannel).catch(() => undefined); + }, 15_000); // every 15s (longer than WS, shorter than stale cache) + return () => clearInterval(interval); + }, [selectedTextChannel, messages.fetchMessages]); + return ( patchUIState({ activeTab: tab })}> {activeTab === "live" ? ( @@ -105,6 +114,9 @@ export default function App() { onGuildChange={(id) => patchUIState({ selectedTextGuild: id, selectedTextChannel: "" })} onChannelChange={(id) => patchUIState({ selectedTextChannel: id })} onReanalyze={messages.reanalyze} + onLoadMore={messages.loadMore} + hasMore={messages.hasMore} + loadingMore={messages.loadingMore} /> ) : ( diff --git a/frontend/src/features/messages/components/MessageFeed.tsx b/frontend/src/features/messages/components/MessageFeed.tsx index b02c010..4849372 100644 --- a/frontend/src/features/messages/components/MessageFeed.tsx +++ b/frontend/src/features/messages/components/MessageFeed.tsx @@ -1,3 +1,4 @@ +import { useEffect, useRef } from "react"; import { ScrollArea } from "../../../shared/ui"; import type { MessageRecord } from "../../../shared/api/client"; import { MessageCard, MessageCardSkeleton } from "./MessageCard"; @@ -7,9 +8,30 @@ export interface MessageFeedProps { onReanalyze: (id: string) => Promise; emptyText?: string; loading?: boolean; + onLoadMore?: () => void; + hasMore?: boolean; + loadingMore?: boolean; } -export function MessageFeed({ messages, onReanalyze, emptyText = "No messages found.", loading }: MessageFeedProps) { +export function MessageFeed({ messages, onReanalyze, emptyText = "No messages found.", loading, onLoadMore, hasMore, loadingMore }: MessageFeedProps) { + // IntersectionObserver for infinite scroll — fires when sentinel becomes visible + const sentinelRef = useRef(null); + + useEffect(() => { + if (!onLoadMore || !hasMore) return; + const el = sentinelRef.current; + if (!el) return; + + const observer = new IntersectionObserver( + (entries) => { + if (entries[0]?.isIntersecting) onLoadMore(); + }, + { rootMargin: "400px" }, // preload before user reaches bottom + ); + observer.observe(el); + return () => observer.disconnect(); + }, [onLoadMore, hasMore]); + if (loading) { return ( @@ -30,6 +52,17 @@ export function MessageFeed({ messages, onReanalyze, emptyText = "No messages fo {messages.map((message) => ( ))} + + {/* Infinite-scroll sentinel */} + {hasMore && ( +
+ {loadingMore ? ( + + ) : ( +
+ )} +
+ )}
); diff --git a/frontend/src/features/messages/hooks/useMessages.ts b/frontend/src/features/messages/hooks/useMessages.ts index f0c8861..fc94945 100644 --- a/frontend/src/features/messages/hooks/useMessages.ts +++ b/frontend/src/features/messages/hooks/useMessages.ts @@ -1,34 +1,47 @@ -import { useCallback, useState } from "react"; +import { useCallback, useRef, useState } from "react"; import { listMessages, reanalyzeMessage } from "../../../shared/api/client"; import type { MessageRecord } from "../../../shared/api/client"; +const PAGE_SIZE = 100; + export function mergeMessages(current: MessageRecord[], incoming: MessageRecord[]): MessageRecord[] { const byId = new Map(current.map((message) => [message.id, message])); for (const message of incoming) { byId.set(message.id, { ...byId.get(message.id), ...message }); } - return Array.from(byId.values()) - .sort((a, b) => b.created_at - a.created_at || b.id.localeCompare(a.id)) - .slice(0, 200); + // Removed .slice(0, 200) cap — let the message list grow unbounded. + // Infinite scroll handles the data volume via cursor pagination. + return Array.from(byId.values()).sort((a, b) => b.created_at - a.created_at || b.id.localeCompare(a.id)); } export function useMessages() { const [messages, setMessages] = useState([]); const [loading, setLoading] = useState(false); + const [loadingMore, setLoadingMore] = useState(false); const [error, setError] = useState(null); + const [cursor, setCursor] = useState(null); + const [hasMore, setHasMore] = useState(false); + const currentChannel = useRef(null); const fetchMessages = useCallback(async (channelId?: string) => { if (!channelId) { setMessages([]); + setCursor(null); + setHasMore(false); return []; } + currentChannel.current = channelId; setLoading(true); setError(null); try { - const params = new URLSearchParams({ limit: "80" }); - params.set("channel", channelId); + const params = new URLSearchParams({ limit: String(PAGE_SIZE), channelId }); const result = await listMessages(params); - setMessages(result.data); + // Only update state if we're still on the same channel (avoid race conditions) + if (currentChannel.current === channelId) { + setMessages(result.data); + setCursor(result.nextCursor); + setHasMore(!!result.nextCursor); + } return result.data; } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -39,6 +52,27 @@ export function useMessages() { } }, []); + const loadMore = useCallback(async () => { + if (!cursor || !currentChannel.current || loadingMore) return; + setLoadingMore(true); + try { + const params = new URLSearchParams({ + limit: String(PAGE_SIZE), + channelId: currentChannel.current, + cursor, + }); + const result = await listMessages(params); + // Only update if still on the same channel + if (currentChannel.current === result.data[0]?.channel_id || currentChannel.current) { + setMessages((prev) => [...prev, ...result.data]); + setCursor(result.nextCursor); + setHasMore(!!result.nextCursor); + } + } finally { + setLoadingMore(false); + } + }, [cursor, loadingMore]); + // BUG 5 FIX: reanalyze returns Promise so callers can await it const reanalyze = useCallback(async (id: string): Promise => { setMessages((prev) => @@ -51,5 +85,15 @@ export function useMessages() { await reanalyzeMessage(id); }, []); - return { messages, setMessages, loading, error, fetchMessages, reanalyze }; + return { + messages, + setMessages, + loading, + loadingMore, + error, + fetchMessages, + reanalyze, + loadMore, + hasMore, + }; } diff --git a/frontend/src/features/messages/index.tsx b/frontend/src/features/messages/index.tsx index 2bd73f2..0efce90 100644 --- a/frontend/src/features/messages/index.tsx +++ b/frontend/src/features/messages/index.tsx @@ -14,6 +14,9 @@ interface MessagesPanelProps { onGuildChange: (guildId: string) => void; onChannelChange: (channelId: string) => void; onReanalyze: (id: string) => Promise; + onLoadMore?: () => void; + hasMore?: boolean; + loadingMore?: boolean; } type AiFilter = "all" | "clean" | "warn" | "flagged" | "error" | "pending"; @@ -21,6 +24,7 @@ type AiFilter = "all" | "clean" | "warn" | "flagged" | "error" | "pending"; export function MessagesPanel({ guilds, channels, selectedGuild, selectedChannel, messages, onGuildChange, onChannelChange, onReanalyze, + onLoadMore, hasMore, loadingMore, }: MessagesPanelProps) { const [searchQuery, setSearchQuery] = useState(""); const [searchResults, setSearchResults] = useState([]); @@ -85,7 +89,7 @@ export function MessagesPanel({ {stats.total > 0 && (
- {stats.total} total + {stats.total} total{hasMore && !showSearch ? "+" : ""} {stats.clean} clean {stats.warn} warn {stats.flagged} flagged @@ -127,7 +131,14 @@ export function MessagesPanel({ Images - + diff --git a/src/moderation/aiAnalyzer.ts b/src/moderation/aiAnalyzer.ts index 24c06fe..a76e957 100644 --- a/src/moderation/aiAnalyzer.ts +++ b/src/moderation/aiAnalyzer.ts @@ -21,6 +21,7 @@ import { updateMessageAIAnalysis, updateMessagesAIAnalysisBulk, } from "./messageStore.js"; +import { invalidateAnalyticsCache } from "./analyticsStore.js"; import type { AnalysisQueueStatus, MessageRecord, @@ -374,6 +375,7 @@ async function processIndividualFallback( const rows = await updateMessagesAIAnalysisBulk(updates); for (const row of rows) { getModerationBroadcaster()?.messageAnalyzed(row); + invalidateAnalyticsCache(row.guild_id); scheduleAutoDelete(row); } diff --git a/src/moderation/analyticsStore.ts b/src/moderation/analyticsStore.ts index 1699dde..925374b 100644 --- a/src/moderation/analyticsStore.ts +++ b/src/moderation/analyticsStore.ts @@ -424,8 +424,8 @@ export async function getTopicTrends(input: { try { const since = Date.now() - hours * 3600_000; - // Only fetch messages that have ai_analysis (the ones that actually have topics) - // This dramatically reduces rows for large guilds + // Fetch all analyzed messages within the time window (no hard row cap). + // Messages without ai_analysis are excluded which naturally limits rows. const rows = (await executeAll( ` SELECT @@ -438,7 +438,6 @@ export async function getTopicTrends(input: { AND ai_analysis IS NOT NULL ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} ORDER BY created_at DESC - LIMIT 2000 `, channelId ? [guildId, since, channelId, channelId] : [guildId, since], )) as MessageRecord[]; diff --git a/src/moderation/llmModerationClient.ts b/src/moderation/llmModerationClient.ts index 17baa3a..7df10df 100644 --- a/src/moderation/llmModerationClient.ts +++ b/src/moderation/llmModerationClient.ts @@ -608,10 +608,11 @@ Ini adalah server Discord komunitas Indonesia. Kamu harus memahami: - Bahasa gaul/slang Indonesia: "anjay", "wkwk", "gws", "gaskeun", "santuy", "njir", "baka", "woy", "woi", "hadeh", dll. - Singkatan umum: "gw", "lo", "emg", "kyk", "tdk", "krn", "jgn", dll. - Konteks budaya lokal: SARA (Suku, Agama, Ras, Antar-golongan), hoaks, ujaran kebencian berbasis konteks Indonesia. -- Makian/kata kasar umum (seperti "anjing", "asu", "bangsat") BUKAN pelanggaran SARA. SARA khusus untuk diskriminasi/hinaan terhadap Suku, Agama, Ras, dan Antargolongan. NAMUN makian/kata kasar TETAP bisa di-flag sebagai "harassment" atau "vulgar_language" sesuai konteks (misalnya jika ditujukan ke orang lain atau dalam tone agresif). Jangan flag sebagai SARA, tapi flag sesuai kategori yang tepat. +- Makian/kata kasar umum (seperti "anjing", "asu", "bangsat") BUKAN pelanggaran SARA. SARA khusus untuk diskriminasi/hinaan terhadap Suku, Agama, Ras, dan Antargolongan. NAMUN makian/kata kasar TETAP bisa di-flag sebagai "harassment" atau "vulgar_language" HANYA jika: (1) ditujukan langsung ke orang lain sebagai serangan/hinaan, (2) dalam tone agresif/mengancam, atau (3) bagian dari pola harassment berkelanjutan. Jangan flag sekadar karena kata itu muncul dalam teks. Jangan flag sebagai SARA, tapi flag sesuai kategori yang tepat. - Kata "asus" adalah merk teknologi, jangan pernah dianggap sebagai makian "asu". - Perbedaan antara humor/banter biasa vs konten yang benar-benar melanggar. - "woy"/"woi" adalah sapaan/interjeksi informal Indonesia dan tidak boleh dianggap SARA, hate speech, atau harassment tanpa target hinaan/ancaman jelas. +- Kata-kata AMAN yang TIDAK BOLEH di-flag: "kakek" (kakek = grandfather, family term), "Wah" (exclamation), "hadeh" (slang exclamation), dan kata-kata keluarga/interjeksi umum lainnya. Jika kata ini muncul dalam teks, JANGAN flag sebagai vulgar_language atau harassment. - Discord custom emoji seperti <:hadeh:123> atau [emoji:hadeh] adalah ekspresi/emoji, bukan pelanggaran teks. Gunakan sebagai konteks ekspresi saja. - Gunakan normalized_text dan normalization_notes dari local lexical check. Jika notes hanya berisi slang/emoji aman (woy, woi, hadeh, dll) dan "no Indonesian badword detected", jangan flag karena kata slang itu saja. NAMUN jika notes menyatakan "Indonesian badword detected" (misalnya "anjing", "bangsat", "asu"), itu EVIDENCE profanitas — gunakan sebagai konteks untuk menilai apakah perlu flag sebagai harassment/vulgar_language, bukan sebagai alasan untuk mengabaikan. - Topik seksual/identitas yang dibatasi server: LGBT/LGBTQ, furry/transfurry, therian, otherkin, protogen, yiff/fursona/fursuit, dan istilah terkait. Jika pesan mempromosikan, mendukung, mengajak, menyatakan identitas/roleplay, membagikan media, atau menjadikan topik ini sebagai pembahasan utama, flag sebagai "sexual_deviation". Jika pesan hanya mengecam/menolak topik tersebut tanpa hinaan ke orang/kelompok, status bisa "clean" atau "warn" sesuai tone. Jangan gunakan kebijakan ini untuk membenarkan doxxing, ancaman, atau penghinaan personal; ancaman/hinaan tetap flag sebagai harassment/hate_speech juga. diff --git a/src/moderation/messageCapture.ts b/src/moderation/messageCapture.ts index a3437b3..ffecd59 100644 --- a/src/moderation/messageCapture.ts +++ b/src/moderation/messageCapture.ts @@ -16,6 +16,7 @@ import { updateMessageAsEdited, upsertMessageForCapture, } from "./messageStore.js"; +import { invalidateAnalyticsCache } from "./analyticsStore.js"; import type { AttachmentRecord, MessageRecord } from "./types.js"; const logger = createChildLogger("message-capture"); @@ -127,6 +128,9 @@ export async function captureMessage( broadcaster.messageCreated(messageRecord); } + // Invalidate analytics cache so real-time stats reflect the new message + invalidateAnalyticsCache(messageRecord.guild_id); + const attachmentUploadTasks: Promise[] = []; // Insert attachments before queuing analysis to avoid race condition