feat: enhance message capture and processing with backlog support

POSTGRESQL_SETUP_COMPLETE.md

@@ -1,132 +0,0 @@
-# PostgreSQL Setup - Complete ✅
-
-**Date:** 2026-05-14  
-**Status:** ✅ Production Ready with Neon PostgreSQL
-
-## Summary
-
-Bot Discord moderation telah berhasil dikonfigurasi untuk menggunakan **PostgreSQL** (Neon) sebagai database utama dengan Drizzle ORM.
-
-## What Was Done
-
-### 1. Database Connection Fixed
-- ✅ Identified database name: `neondb` (bukan `dcbot`)
-- ✅ Updated `.env` dengan DATABASE_URL yang benar
-- ✅ Tested koneksi ke Neon PostgreSQL - berhasil
-
-### 2. Drizzle ORM Updated
-- ✅ Updated `src/database/drizzle.ts` untuk support DATABASE_URL
-- ✅ Regenerated migrations untuk PostgreSQL syntax
-- ✅ Ran migrations successfully: `pnpm run db:migrate:programmatic`
-
-### 3. Bot Tested
-- ✅ Bot startup dengan PostgreSQL - berhasil
-- ✅ Database initialized dengan type: postgres
-- ✅ Message capture working
-- ✅ AI analysis worker started
-- ✅ WebSocket server listening
-
-## Current Configuration
-
-```env
-DATABASE_TYPE=postgres
-DATABASE_URL=postgresql://neondb_owner:npg_2ziHMPwZCet9@ep-long-glitter-ao3sjoyu-pooler.c-2.ap-southeast-1.aws.neon.tech/neondb?sslmode=verify-full&channel_binding=require&connect_timeout=10
-```
-
-## Database Schema Created
-
-✅ **Tables created in PostgreSQL:**
-- `muxer_jobs` - Job queue untuk audio processing
-- `messages` - Text messages dengan AI analysis
-- `attachments` - File metadata dengan foreign key
-- `ui_state` - Persistent UI state
-- `__drizzle_migrations` - Migration tracking
-
-## Commands Available
-
-```bash
-# Start bot dengan PostgreSQL
-pnpm run dev
-
-# Generate migrations setelah schema changes
-pnpm run db:generate
-
-# Run migrations (programmatic - recommended)
-pnpm run db:migrate:programmatic
-
-# Run migrations (Drizzle Kit CLI)
-pnpm run db:migrate
-
-# Open Drizzle Studio untuk visual data management
-pnpm run db:studio
-```
-
-## Verification
-
-### Bot Startup Log
-```
-✅ PostgreSQL database initialized
-✅ Database initialized (type: postgres)
-✅ Bot logged in
-✅ Message capture handlers registered
-✅ AI analysis worker started
-✅ WebSocket server listening on port 3000
-✅ Web interface listening
-✅ Message inserted (from Discord)
-```
-
-### Database Tables
-```sql
-SELECT table_name FROM information_schema.tables 
-WHERE table_schema = 'public';
-
--- Results:
--- muxer_jobs
--- messages
--- attachments
--- ui_state
--- __drizzle_migrations
-```
-
-## Commits
-
-```
-47ae7f8 chore: remove temporary test files
-35269b5 feat: configure postgresql as primary database with neon connection
-c63a614 docs: add comprehensive drizzle orm migration final summary
-9889d20 feat: add programmatic migration runner for better PostgreSQL support
-b580430 docs: add drizzle orm migration completion summary
-b9d0a06 fix: update drizzle config to read env vars directly for CLI compatibility
-b600dad fix: correct import ordering and update tests for drizzle-orm migration
-50d4517 refactor: remove old database adapter files
-9ff0f0b feat: update application initialization for drizzle
-1c4b0af refactor: migrate messageStore to drizzle-orm
-dfe3444 refactor: migrate muxer-queue to drizzle-orm
-7e528a4 feat: create drizzle database client
-4e28cf9 feat: add drizzle configuration and initial migrations
-52b36c9 feat: create drizzle schema definitions
-b833b6d feat: add drizzle-orm and drizzle-kit dependencies
-```
-
-## Key Features
-
-✅ **Type-Safe Queries** - Full TypeScript support dengan Drizzle ORM  
-✅ **PostgreSQL Support** - Neon cloud database integration  
-✅ **Automatic Migrations** - Drizzle Kit generates migrations  
-✅ **Connection Pooling** - Configurable pool size  
-✅ **Production Ready** - All tests passing, zero errors  
-
-## Next Steps
-
-1. **Monitor bot performance** dengan PostgreSQL
-2. **Use Drizzle Studio** untuk visual data management: `pnpm run db:studio`
-3. **For schema changes**: Update `src/database/schema.ts` → `pnpm run db:generate` → `pnpm run db:migrate:programmatic`
-4. **Backup strategy** - Setup regular backups di Neon dashboard
-
-## Status
-
-🎉 **PostgreSQL migration complete and verified!**
-
-Bot Discord moderation sekarang menggunakan PostgreSQL (Neon) sebagai database utama dengan Drizzle ORM untuk type-safe operations.
-
-**Ready for production deployment!** ✅

biome.json

@@ -14,9 +14,6 @@
       "style": {
         "noNonNullAssertion": "warn",
         "useNodejsImportProtocol": "warn"
-      },
-      "suspicious": {
-        "noExplicitAny": "warn"
       }
     }
   }

frontend/src/App.tsx

@@ -6,6 +6,19 @@ import type { DashboardEvent } from "./ws/client";
 import { MessageFeed } from "./components/messages/MessageFeed";
 import { ReviewPanel } from "./components/review/ReviewPanel";
 
+function mergeMessages(
+  current: MessageRecord[],
+  incoming: MessageRecord[],
+): MessageRecord[] {
+  const byId = new Map(current.map((message) => [message.id, message]));
+  for (const message of incoming) {
+    byId.set(message.id, { ...byId.get(message.id), ...message });
+  }
+  return Array.from(byId.values())
+    .sort((a, b) => b.created_at - a.created_at || b.id.localeCompare(a.id))
+    .slice(0, 200);
+}
+
 export default function App() {
   const [messages, setMessages] = useState<MessageRecord[]>([]);
   const [wsStatus, setWsStatus] = useState<string>("connecting");
@@ -17,7 +30,7 @@ export default function App() {
     listMessages(new URLSearchParams({ limit: "30" }))
       .then((result) => {
         if (!cancelled) {
-          setMessages(result.data);
+          setMessages(mergeMessages([], result.data));
         }
       })
       .catch((err) => {
@@ -29,20 +42,10 @@ export default function App() {
     const ws = connectDashboardSocket((event: DashboardEvent) => {
       switch (event.type) {
         case "message_created":
-          setMessages((prev) => {
+          setMessages((prev) => mergeMessages(prev, [event.data]));
-            const existing = prev.some((message) => message.id === event.data.id);
-            if (existing) {
-              return prev.map((message) =>
-                message.id === event.data.id ? event.data : message,
-              );
-            }
-            return [event.data, ...prev].slice(0, 200);
-          });
           break;
         case "message_analyzed":
-          setMessages((prev) =>
+          setMessages((prev) => mergeMessages(prev, [event.data]));
-            prev.map((m) => (m.id === event.data.id ? event.data : m)),
-          );
           break;
         case "message_updated":
           setMessages((prev) =>

src/moderation/backlogSync.ts

@@ -29,7 +29,7 @@ async function syncChannelMessages(
         continue;
       }
 
-      await captureMessage(message, "text");
+      await captureMessage(message, "text", { source: "backlog" });
       synced++;
     }
 

src/moderation/messageCapture.ts

@@ -22,9 +22,18 @@ import type {
 
 const logger = createChildLogger("message-capture");
 
+type ModerationGlobal = typeof globalThis & {
+  moderationBroadcaster?: ModerationBroadcaster;
+};
+
+function getModerationBroadcaster(): ModerationBroadcaster | undefined {
+  return (globalThis as ModerationGlobal).moderationBroadcaster;
+}
+
 export async function captureMessage(
   message: Message,
   type: "text" | "edited" | "deleted",
+  options: { source?: "live" | "backlog" } = {},
 ): Promise<void> {
   const location = getMessageLocation(message);
   const metadata = getMessageMetadata(message);
@@ -46,17 +55,19 @@ export async function captureMessage(
     metadata: JSON.stringify(metadata),
   };
 
-  await upsertMessageForCapture(messageRecord);
+  const inserted = await upsertMessageForCapture(messageRecord);
-  queueMessageAnalysis(message.id);
+  if (!inserted) {
+    return;
+  }
 
-  const broadcaster = (globalThis as any).moderationBroadcaster as
+  const isBacklog = options.source === "backlog";
-    | ModerationBroadcaster
+  if (!isBacklog) {
-    | undefined;
+    queueMessageAnalysis(message.id);
-  if (broadcaster) {
+  }
-    broadcaster.messageCreated({
+
-      ...messageRecord,
+  const broadcaster = getModerationBroadcaster();
-      type: "text",
+  if (broadcaster && !isBacklog) {
-    });
+    broadcaster.messageCreated(messageRecord);
   }
 
   if (message.attachments.size > 0) {
@@ -132,9 +143,7 @@ export function registerMessageCapture(client: Client): void {
         );
         queueMessageAnalysis(newMessage.id);
 
-        const broadcaster = (globalThis as any).moderationBroadcaster as
+        const broadcaster = getModerationBroadcaster();
-          | ModerationBroadcaster
-          | undefined;
         if (broadcaster) {
           broadcaster.messageUpdated({
             id: newMessage.id,
@@ -164,9 +173,7 @@ export function registerMessageCapture(client: Client): void {
       const deletedAt = Date.now();
       await updateMessageAsDeleted(message.id, deletedAt);
 
-      const broadcaster = (globalThis as any).moderationBroadcaster as
+      const broadcaster = getModerationBroadcaster();
-        | ModerationBroadcaster
-        | undefined;
       if (broadcaster) {
         broadcaster.messageDeleted({
           id: message.id,

src/moderation/messageStore.ts

@@ -57,26 +57,26 @@ export async function insertMessage(message: MessageRecord): Promise<void> {
 
 export async function upsertMessageForCapture(
   message: MessageRecord,
-): Promise<void> {
+): Promise<boolean> {
   try {
     const db = getDatabase() as any;
-
-    // Set ai_status to pending for new or recaptured/edited text
     const messageWithAIStatus = {
       ...message,
       ai_status: "pending" as const,
     };
 
-    // Try insert first (fast path for new messages)
+    const rows = await db
-    await db
       .insert(messagesTable)
       .values(messageWithAIStatus)
-      .onConflictDoNothing();
+      .onConflictDoNothing()
+      .returning({ id: messagesTable.id });
 
+    const inserted = rows.length > 0;
     logger.debug(
-      { messageId: message.id, channelId: message.channel_id },
+      { messageId: message.id, channelId: message.channel_id, inserted },
-      "Message upserted for capture",
+      inserted ? "Message inserted for capture" : "Message already captured",
     );
+    return inserted;
   } catch (error) {
     logger.error(
       {

tests/moderation/messageCapture.test.ts

@@ -0,0 +1,156 @@
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  vi,
+} from "vitest";
+import {
+  closeDatabase,
+  getDatabase,
+  initializeDatabase,
+} from "../../src/database/drizzle";
+import { captureMessage } from "../../src/moderation/messageCapture";
+import type { ModerationBroadcaster } from "../../src/moderation/types";
+
+const queueMessageAnalysis = vi.fn();
+
+type TestMessage = Parameters<typeof captureMessage>[0];
+type ModerationTestGlobal = typeof globalThis & {
+  moderationBroadcaster?: Partial<ModerationBroadcaster>;
+};
+
+interface TestDatabase {
+  run(sql: string): void;
+}
+
+function getTestDatabase(): TestDatabase {
+  return getDatabase() as unknown as TestDatabase;
+}
+
+vi.mock("../../src/moderation/aiAnalyzer", () => ({
+  queueMessageAnalysis: (id: string) => queueMessageAnalysis(id),
+}));
+
+function createMessage(id = "message-1"): TestMessage {
+  return {
+    id,
+    guildId: "guild-1",
+    channelId: "channel-1",
+    author: {
+      id: "user-1",
+      username: "alice",
+      bot: false,
+      avatarURL: () => null,
+    },
+    content: "hello",
+    cleanContent: "hello",
+    createdTimestamp: 1_700_000_000_000,
+    attachments: new Map(),
+    stickers: new Map(),
+    embeds: [],
+    member: null,
+    reference: null,
+    channel: {
+      id: "channel-1",
+      name: "general",
+      isThread: () => false,
+    },
+  } as unknown as TestMessage;
+}
+
+async function createTables() {
+  const db = getTestDatabase();
+  db.run(`
+    CREATE TABLE IF NOT EXISTS "messages" (
+      "id" text PRIMARY KEY NOT NULL,
+      "guild_id" text NOT NULL,
+      "channel_id" text NOT NULL,
+      "thread_id" text,
+      "user_id" text NOT NULL,
+      "username" text NOT NULL,
+      "avatar_url" text,
+      "content" text NOT NULL,
+      "edited_content" text,
+      "created_at" integer NOT NULL,
+      "edited_at" integer,
+      "deleted_at" integer,
+      "type" text DEFAULT 'text' NOT NULL,
+      "metadata" text,
+      "ai_status" text DEFAULT 'pending' NOT NULL,
+      "ai_moderation_flags" text,
+      "ai_moderation_score" real,
+      "ai_moderation_raw" text,
+      "ai_analysis" text,
+      "ai_analyzed_at" integer,
+      "ai_error" text
+    )
+  `);
+  db.run(`
+    CREATE TABLE IF NOT EXISTS "attachments" (
+      "id" text PRIMARY KEY NOT NULL,
+      "message_id" text NOT NULL,
+      "guild_id" text NOT NULL,
+      "channel_id" text NOT NULL,
+      "thread_id" text,
+      "user_id" text NOT NULL,
+      "filename" text NOT NULL,
+      "size" integer NOT NULL,
+      "type" text NOT NULL,
+      "discord_url" text NOT NULL,
+      "uploaded_url" text,
+      "upload_status" text DEFAULT 'pending' NOT NULL,
+      "upload_error" text,
+      "created_at" integer NOT NULL,
+      "uploaded_at" integer
+    )
+  `);
+}
+
+describe("captureMessage", () => {
+  beforeAll(async () => {
+    await initializeDatabase();
+    await createTables();
+  });
+
+  beforeEach(async () => {
+    queueMessageAnalysis.mockClear();
+    const db = getTestDatabase();
+    db.run(`DELETE FROM "attachments"`);
+    db.run(`DELETE FROM "messages"`);
+    delete (globalThis as ModerationTestGlobal).moderationBroadcaster;
+  });
+
+  afterAll(async () => {
+    await closeDatabase();
+  });
+
+  it("does not requeue or rebroadcast a duplicate captured message", async () => {
+    const message = createMessage();
+    const messageCreated = vi.fn();
+    (globalThis as ModerationTestGlobal).moderationBroadcaster = {
+      messageCreated,
+    };
+
+    await captureMessage(message, "text");
+    await captureMessage(message, "text");
+
+    expect(queueMessageAnalysis).toHaveBeenCalledTimes(1);
+    expect(messageCreated).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not queue or broadcast backlog captures one message at a time", async () => {
+    const message = createMessage("backlog-message-1");
+    const messageCreated = vi.fn();
+    (globalThis as ModerationTestGlobal).moderationBroadcaster = {
+      messageCreated,
+    };
+
+    await captureMessage(message, "text", { source: "backlog" });
+
+    expect(queueMessageAnalysis).not.toHaveBeenCalled();
+    expect(messageCreated).not.toHaveBeenCalled();
+  });
+});