feat(moderation): enhance message capture and storage with thread support
- Added functions to retrieve message location, sticker metadata, and display content in messageCapture.ts. - Updated captureMessage function to store thread information and sticker metadata in the database. - Modified messageStore.ts to support querying messages and attachments by thread ID. - Updated types.ts to include thread_id in AttachmentRecord. - Altered database schema in muxer-queue.ts to add thread_id column to attachments. - Introduced ChannelSummary interface and listWatchableChannels method in voiceController.ts to fetch watchable channels. - Added API endpoint in webserver.ts to retrieve channels for a given guild.
This commit is contained in:
+1
-1
@@ -2,4 +2,4 @@ node_modules
|
||||
recordings
|
||||
.env
|
||||
dist/
|
||||
.muxer-queue.db
|
||||
.muxer-queue.**
|
||||
@@ -0,0 +1,64 @@
|
||||
# Backlog Sync Rich Metadata Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Fetch prior Discord messages up to 24 hours on startup, persist rich Discord-client-like metadata, and render rich message content in homepage tabs.
|
||||
|
||||
**Architecture:** Add `messageMetadata.ts` for reusable extraction, `backlogSync.ts` for bounded startup history fetch, reuse existing store/uploader. UI reads metadata JSON and renders stickers, embeds, attachments/replies/thread badges.
|
||||
|
||||
**Tech Stack:** Bun, TypeScript, discord.js-selfbot-v13, bun:sqlite, Express/WebSocket, vanilla HTML/CSS/JS.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Extract rich message metadata
|
||||
|
||||
**Files:**
|
||||
- Create: `src/moderation/messageMetadata.ts`
|
||||
- Modify: `src/moderation/messageCapture.ts`
|
||||
|
||||
- [ ] Create helper functions: `getMessageLocation`, `getStickerMetadata`, `getEmbedMetadata`, `getAttachmentMetadata`, `getMessageMetadata`, `getDisplayContent`.
|
||||
- [ ] Replace duplicate capture helper logic with imports from `messageMetadata.ts`.
|
||||
- [ ] Verify: `bun run typecheck`.
|
||||
|
||||
### Task 2: Make message inserts idempotent
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/moderation/messageStore.ts`
|
||||
|
||||
- [ ] Change message insert to `INSERT OR IGNORE` so backlog sync and live events do not conflict.
|
||||
- [ ] Change attachment insert to `INSERT OR IGNORE`.
|
||||
- [ ] Verify: `bun run typecheck && bun run test`.
|
||||
|
||||
### Task 3: Add backlog sync
|
||||
|
||||
**Files:**
|
||||
- Create: `src/moderation/backlogSync.ts`
|
||||
- Modify: `src/index.ts`
|
||||
- Modify: `src/config.ts`
|
||||
- Modify: `.env.example`
|
||||
|
||||
- [ ] Add config: `BACKLOG_SYNC_HOURS=24`, `BACKLOG_SYNC_BATCH_SIZE=100`.
|
||||
- [ ] Fetch text/thread channels from monitored guild on ready.
|
||||
- [ ] For each channel/thread, page `channel.messages.fetch({ limit, before })` until older than cutoff.
|
||||
- [ ] Store messages with rich metadata and attachments.
|
||||
- [ ] Start sync after registering live capture; run async and log progress.
|
||||
- [ ] Verify: `bun run typecheck && bun run test`.
|
||||
|
||||
### Task 4: Render richer UI
|
||||
|
||||
**Files:**
|
||||
- Modify: `public/index.html`
|
||||
|
||||
- [ ] Render metadata embeds as embed cards.
|
||||
- [ ] Render attachments as inline previews/links in Text tab.
|
||||
- [ ] Render reply and thread badges.
|
||||
- [ ] Keep sticker rendering.
|
||||
- [ ] Verify static JS syntax by typecheck/tests where applicable.
|
||||
|
||||
### Task 5: Final verification
|
||||
|
||||
**Files:** all touched files
|
||||
|
||||
- [ ] Run `bun run typecheck`.
|
||||
- [ ] Run `bun run test`.
|
||||
- [ ] Verify short DB init with `bun -e 'import("./src/muxer-queue.ts").then((m)=>{const db=m.getDatabase(); db.close(); console.log("sqlite ok")})'`.
|
||||
+900
-453
File diff suppressed because it is too large
Load Diff
@@ -8,29 +8,73 @@ import type { MessageRecord, AttachmentRecord } from "./types";
|
||||
|
||||
const logger = createChildLogger("message-capture");
|
||||
|
||||
function getMessageLocation(message: Message): {
|
||||
channelId: string;
|
||||
threadId: string | null;
|
||||
threadName: string | null;
|
||||
} {
|
||||
const channel = message.channel as TextChannel | ThreadChannel;
|
||||
if (!channel.isThread?.()) {
|
||||
return { channelId: message.channelId, threadId: null, threadName: null };
|
||||
}
|
||||
|
||||
return {
|
||||
channelId: channel.parentId ?? message.channelId,
|
||||
threadId: channel.id,
|
||||
threadName: channel.name,
|
||||
};
|
||||
}
|
||||
|
||||
function getStickerMetadata(message: Message): Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
url: string;
|
||||
}> {
|
||||
return Array.from(message.stickers.values()).map((sticker) => ({
|
||||
id: sticker.id,
|
||||
name: sticker.name,
|
||||
url: sticker.url,
|
||||
}));
|
||||
}
|
||||
|
||||
function getDisplayContent(message: Message): string {
|
||||
if (message.content.trim().length > 0) return message.content;
|
||||
|
||||
const stickers = getStickerMetadata(message);
|
||||
if (stickers.length > 0) {
|
||||
return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" ");
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
async function captureMessage(
|
||||
db: SqliteDatabase,
|
||||
message: Message,
|
||||
type: "text" | "edited" | "deleted",
|
||||
): Promise<void> {
|
||||
const channel = message.channel as TextChannel | ThreadChannel;
|
||||
const threadId = channel.isThread?.() ? channel.id : null;
|
||||
const location = getMessageLocation(message);
|
||||
const stickers = getStickerMetadata(message);
|
||||
const metadata = {
|
||||
stickers,
|
||||
threadName: location.threadName,
|
||||
};
|
||||
|
||||
const messageRecord: MessageRecord = {
|
||||
id: message.id,
|
||||
guild_id: message.guildId!,
|
||||
channel_id: message.channelId,
|
||||
thread_id: threadId,
|
||||
channel_id: location.channelId,
|
||||
thread_id: location.threadId,
|
||||
user_id: message.author!.id,
|
||||
username: message.author!.username,
|
||||
avatar_url: message.author!.avatarURL() || null,
|
||||
content: message.content,
|
||||
content: getDisplayContent(message),
|
||||
edited_content: null,
|
||||
created_at: message.createdTimestamp,
|
||||
edited_at: null,
|
||||
deleted_at: null,
|
||||
type,
|
||||
metadata: null,
|
||||
metadata: JSON.stringify(metadata),
|
||||
};
|
||||
|
||||
insertMessage(db, messageRecord);
|
||||
@@ -38,13 +82,7 @@ async function captureMessage(
|
||||
const broadcaster = globalThis as any;
|
||||
if (broadcaster.broadcastMessageCreated) {
|
||||
broadcaster.broadcastMessageCreated({
|
||||
id: message.id,
|
||||
channel_id: message.channelId,
|
||||
user_id: message.author!.id,
|
||||
username: message.author!.username,
|
||||
avatar_url: message.author!.avatarURL() || null,
|
||||
content: message.content,
|
||||
created_at: message.createdTimestamp,
|
||||
...messageRecord,
|
||||
type: "text",
|
||||
});
|
||||
}
|
||||
@@ -55,7 +93,8 @@ async function captureMessage(
|
||||
id: attachment.id,
|
||||
message_id: message.id,
|
||||
guild_id: message.guildId!,
|
||||
channel_id: message.channelId,
|
||||
channel_id: location.channelId,
|
||||
thread_id: location.threadId,
|
||||
user_id: message.author!.id,
|
||||
filename: attachment.name || "unknown",
|
||||
size: attachment.size,
|
||||
@@ -77,7 +116,7 @@ async function captureMessage(
|
||||
id: attachment.id,
|
||||
message_id: message.id,
|
||||
filename: attachment.name || "unknown",
|
||||
channel_id: message.channelId,
|
||||
channel_id: location.channelId,
|
||||
created_at: Date.now(),
|
||||
});
|
||||
}
|
||||
@@ -129,13 +168,13 @@ export function registerMessageCapture(client: Client, db: SqliteDatabase): void
|
||||
|
||||
if (existing) {
|
||||
const editedAt = Date.now();
|
||||
updateMessageAsEdited(db, newMessage.id, newMessage.content || "", editedAt);
|
||||
updateMessageAsEdited(db, newMessage.id, getDisplayContent(newMessage as Message), editedAt);
|
||||
|
||||
const broadcaster = globalThis as any;
|
||||
if (broadcaster.broadcastMessageUpdated) {
|
||||
broadcaster.broadcastMessageUpdated({
|
||||
id: newMessage.id,
|
||||
edited_content: newMessage.content || "",
|
||||
edited_content: getDisplayContent(newMessage as Message),
|
||||
edited_at: editedAt,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -96,12 +96,12 @@ export function getMessagesByChannel(
|
||||
try {
|
||||
const stmt = db.prepare(`
|
||||
SELECT * FROM messages
|
||||
WHERE channel_id = ?
|
||||
WHERE channel_id = ? OR thread_id = ?
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
`);
|
||||
|
||||
const rows = stmt.all(channelId, limit, offset) as MessageRecord[];
|
||||
const rows = stmt.all(channelId, channelId, limit, offset) as MessageRecord[];
|
||||
return rows;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
@@ -116,9 +116,9 @@ export function insertAttachment(db: SqliteDatabase, attachment: AttachmentRecor
|
||||
try {
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO attachments (
|
||||
id, message_id, guild_id, channel_id, user_id, filename, size, type,
|
||||
id, message_id, guild_id, channel_id, thread_id, user_id, filename, size, type,
|
||||
discord_url, uploaded_url, upload_status, upload_error, created_at, uploaded_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
stmt.run(
|
||||
@@ -126,6 +126,7 @@ export function insertAttachment(db: SqliteDatabase, attachment: AttachmentRecor
|
||||
attachment.message_id,
|
||||
attachment.guild_id,
|
||||
attachment.channel_id,
|
||||
attachment.thread_id,
|
||||
attachment.user_id,
|
||||
attachment.filename,
|
||||
attachment.size,
|
||||
@@ -157,12 +158,12 @@ export function getAttachmentsByChannel(
|
||||
try {
|
||||
const stmt = db.prepare(`
|
||||
SELECT * FROM attachments
|
||||
WHERE channel_id = ?
|
||||
WHERE channel_id = ? OR thread_id = ?
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
`);
|
||||
|
||||
const rows = stmt.all(channelId, limit, offset) as AttachmentRecord[];
|
||||
const rows = stmt.all(channelId, channelId, limit, offset) as AttachmentRecord[];
|
||||
return rows;
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
|
||||
@@ -20,6 +20,7 @@ export interface AttachmentRecord {
|
||||
message_id: string;
|
||||
guild_id: string;
|
||||
channel_id: string;
|
||||
thread_id: string | null;
|
||||
user_id: string;
|
||||
filename: string;
|
||||
size: number;
|
||||
|
||||
@@ -84,6 +84,7 @@ function initializeDatabase(): SqliteDatabase {
|
||||
message_id TEXT NOT NULL,
|
||||
guild_id TEXT NOT NULL,
|
||||
channel_id TEXT NOT NULL,
|
||||
thread_id TEXT,
|
||||
user_id TEXT NOT NULL,
|
||||
filename TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
@@ -102,6 +103,12 @@ function initializeDatabase(): SqliteDatabase {
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(upload_status);
|
||||
`);
|
||||
|
||||
try {
|
||||
database.exec("ALTER TABLE attachments ADD COLUMN thread_id TEXT");
|
||||
} catch {
|
||||
// Column already exists on databases initialized after the moderation schema was added.
|
||||
}
|
||||
|
||||
return database;
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,12 @@ export interface VoiceChannelSummary {
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface ChannelSummary {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export class VoiceController {
|
||||
private activeGuildId: string | null = null;
|
||||
private activeChannelId: string | null = null;
|
||||
@@ -63,6 +69,27 @@ export class VoiceController {
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
async listWatchableChannels(guildId: string): Promise<ChannelSummary[]> {
|
||||
const guild = this.getGuild(guildId);
|
||||
await guild.channels.fetch().catch(() => null);
|
||||
|
||||
return guild.channels.cache
|
||||
.filter((channel) =>
|
||||
["GUILD_TEXT", "GUILD_PUBLIC_THREAD", "GUILD_PRIVATE_THREAD"].includes(
|
||||
channel.type,
|
||||
),
|
||||
)
|
||||
.map((channel) => {
|
||||
const parentName = channel.isThread?.() ? channel.parent?.name : null;
|
||||
return {
|
||||
id: channel.id,
|
||||
name: parentName ? `${parentName} / ${channel.name}` : channel.name,
|
||||
type: channel.type,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
async connect(guildId: string, channelId: string): Promise<VoiceStatus> {
|
||||
if (!this.client.isReady()) {
|
||||
throw new AppError(
|
||||
|
||||
@@ -106,6 +106,14 @@ export function startWebserver(
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/api/guilds/:guildId/channels", async (req, res, next) => {
|
||||
try {
|
||||
res.json(await voiceController.listWatchableChannels(req.params.guildId));
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/api/connect", async (req, res, next) => {
|
||||
try {
|
||||
const { guildId, channelId } = req.body as {
|
||||
|
||||
Reference in New Issue
Block a user