From 0b8111de8116703c30a9ea8cc6ace06ba3e0b80a Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Wed, 13 May 2026 21:04:45 +0700 Subject: [PATCH] feat(moderation): implement backlog message synchronization and enhance message metadata handling --- .env.example | 2 + public/index.html | 93 +++++++++++++++-- src/config.ts | 2 + src/index.ts | 4 + src/moderation/backlogSync.ts | 120 +++++++++++++++++++++ src/moderation/messageCapture.ts | 51 +-------- src/moderation/messageMetadata.ts | 167 ++++++++++++++++++++++++++++++ src/moderation/messageStore.ts | 4 +- src/voiceController.ts | 25 +++-- 9 files changed, 402 insertions(+), 66 deletions(-) create mode 100644 src/moderation/backlogSync.ts create mode 100644 src/moderation/messageMetadata.ts diff --git a/.env.example b/.env.example index 8d0269c..26e859e 100644 --- a/.env.example +++ b/.env.example @@ -35,4 +35,6 @@ PICSER_UPLOAD_URL=https://picser.asepharyana.tech/api/upload ATTACHMENT_UPLOAD_TIMEOUT_MS=30000 ATTACHMENT_MAX_SIZE_MB=100 ATTACHMENT_RETRY_ATTEMPTS=3 +BACKLOG_SYNC_HOURS=24 +BACKLOG_SYNC_BATCH_SIZE=100 diff --git a/public/index.html b/public/index.html index 4605089..1e73613 100644 --- a/public/index.html +++ b/public/index.html @@ -377,8 +377,13 @@ .time { color: var(--faint); font: 600 11px/1 "JetBrains Mono", monospace; white-space: nowrap; } .message-text { color: #dbe6f7; line-height: 1.6; white-space: pre-wrap; word-break: break-word; } - .sticker-strip { display: flex; gap: 10px; flex-wrap: wrap; } + .sticker-strip, .attachment-strip { display: flex; gap: 10px; flex-wrap: wrap; } .sticker-img { width: 96px; height: 96px; object-fit: contain; border-radius: 16px; background: rgba(0,0,0,0.22); border: 1px solid var(--line); padding: 8px; } + .attachment-chip { color: var(--cyan); text-decoration: none; border: 1px solid var(--line); border-radius: 14px; padding: 8px 10px; font: 700 12px/1 "JetBrains Mono", monospace; background: rgba(0,229,255,0.06); } + .embed-card { border-left: 4px solid var(--blue); border-radius: 16px; padding: 12px; background: rgba(98,117,255,0.08); display: grid; gap: 8px; } + .embed-title { font-weight: 900; color: var(--text); } + .embed-description { color: var(--muted); line-height: 1.5; white-space: pre-wrap; } + .embed-image { max-width: 360px; width: 100%; border-radius: 14px; border: 1px solid var(--line); } .badges { display: flex; gap: 8px; flex-wrap: wrap; } .badge { @@ -770,27 +775,93 @@ const text = document.createElement('div'); text.className = 'message-text'; text.textContent = msg.edited_content || msg.content || '(empty message)'; - const stickers = document.createElement('div'); - stickers.className = 'sticker-strip'; - for (const sticker of metadata.stickers || []) { - const img = document.createElement('img'); - img.className = 'sticker-img'; - img.src = sticker.url; - img.alt = sticker.name; - stickers.appendChild(img); - } + const stickers = renderStickers(metadata.stickers || []); + const embeds = renderEmbeds(metadata.embeds || []); + const attachments = renderAttachments(metadata.attachments || []); const badges = document.createElement('div'); badges.className = 'badges'; - if (msg.thread_id) appendBadge(badges, metadata.threadName ? `thread: ${metadata.threadName}` : 'thread', ''); + if (metadata.reference?.messageId) appendBadge(badges, 'reply', ''); + if (msg.thread_id) appendBadge(badges, metadata.channel?.threadName ? `thread: ${metadata.channel.threadName}` : 'thread', ''); if (msg.edited_at) appendBadge(badges, 'edited', 'edit'); if (msg.deleted_at) appendBadge(badges, 'deleted', 'delete'); card.append(head, text); if (stickers.childElementCount > 0) card.appendChild(stickers); + if (embeds.childElementCount > 0) card.appendChild(embeds); + if (attachments.childElementCount > 0) card.appendChild(attachments); card.appendChild(badges); el.textList.appendChild(card); } } + function renderStickers(stickers) { + const wrap = document.createElement('div'); + wrap.className = 'sticker-strip'; + for (const sticker of stickers) { + const img = document.createElement('img'); + img.className = 'sticker-img'; + img.src = sticker.url; + img.alt = sticker.name; + wrap.appendChild(img); + } + return wrap; + } + + function renderEmbeds(embeds) { + const wrap = document.createElement('div'); + wrap.className = 'feed'; + for (const embed of embeds) { + const card = document.createElement('div'); + card.className = 'embed-card'; + if (embed.title) { + const title = document.createElement(embed.url ? 'a' : 'div'); + title.className = 'embed-title'; + title.textContent = embed.title; + if (embed.url) { + title.href = embed.url; + title.target = '_blank'; + title.rel = 'noreferrer'; + } + card.appendChild(title); + } + if (embed.description) { + const desc = document.createElement('div'); + desc.className = 'embed-description'; + desc.textContent = embed.description; + card.appendChild(desc); + } + for (const field of embed.fields || []) { + const fieldNode = document.createElement('div'); + fieldNode.className = 'embed-description'; + fieldNode.textContent = `${field.name}: ${field.value}`; + card.appendChild(fieldNode); + } + if (embed.image || embed.thumbnail) { + const img = document.createElement('img'); + img.className = 'embed-image'; + img.src = embed.image || embed.thumbnail; + img.alt = embed.title || 'embed image'; + card.appendChild(img); + } + wrap.appendChild(card); + } + return wrap; + } + + function renderAttachments(attachments) { + const wrap = document.createElement('div'); + wrap.className = 'attachment-strip'; + for (const attachment of attachments) { + const link = document.createElement('a'); + link.className = 'attachment-chip'; + link.href = attachment.url; + link.target = '_blank'; + link.rel = 'noreferrer'; + link.textContent = `${attachment.name} (${(attachment.size / 1024).toFixed(1)}KB)`; + wrap.appendChild(link); + } + return wrap; + } + function renderImages() { el.imageGrid.replaceChildren(); if (!state.selectedChannel) return appendEmpty(el.imageGrid, 'Select channel to view image captures'); diff --git a/src/config.ts b/src/config.ts index f214f6c..ca63759 100644 --- a/src/config.ts +++ b/src/config.ts @@ -32,6 +32,8 @@ const configSchema = z.object({ ATTACHMENT_UPLOAD_TIMEOUT_MS: z.coerce.number().positive().default(30000), ATTACHMENT_MAX_SIZE_MB: z.coerce.number().positive().default(100), ATTACHMENT_RETRY_ATTEMPTS: z.coerce.number().positive().default(3), + BACKLOG_SYNC_HOURS: z.coerce.number().positive().default(24), + BACKLOG_SYNC_BATCH_SIZE: z.coerce.number().int().positive().max(100).default(100), }); export type AppConfig = z.infer; diff --git a/src/index.ts b/src/index.ts index 5b900bf..020d3c6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ import { discordPlayer } from "./player"; import { VoiceController } from "./voiceController"; import { startWebserver } from "./webserver"; import { registerMessageCapture } from "./moderation/messageCapture"; +import { syncBacklogMessages } from "./moderation/backlogSync"; import { getDatabase } from "./muxer-queue"; const logger = createChildLogger("bot"); @@ -55,6 +56,9 @@ async function gracefulShutdown(signal: string) { client.on("ready", async () => { logger.info({ user: client.user?.tag }, "Bot logged in"); registerMessageCapture(client, db); + syncBacklogMessages(client, db).catch((error) => { + logger.warn({ error }, "Backlog sync failed"); + }); startWebserver(config.WEBSERVER_PORT, client, voiceController); }); diff --git a/src/moderation/backlogSync.ts b/src/moderation/backlogSync.ts new file mode 100644 index 0000000..15d4ad0 --- /dev/null +++ b/src/moderation/backlogSync.ts @@ -0,0 +1,120 @@ +import type { Client, Message } from "discord.js-selfbot-v13"; +import { config } from "../config"; +import { createChildLogger } from "../logger"; +import type { SqliteDatabase } from "../muxer-queue"; +import { captureMessage } from "./messageCapture"; + +const logger = createChildLogger("backlog-sync"); + +function isWatchableChannel(channel: { type?: string; messages?: unknown }): boolean { + return Boolean( + channel.messages && + ["GUILD_TEXT", "GUILD_PUBLIC_THREAD", "GUILD_PRIVATE_THREAD"].includes( + channel.type ?? "", + ), + ); +} + +async function collectWatchableChannels(guild: any): Promise { + const channels: any[] = []; + for (const channel of guild.channels.cache.values()) { + if (isWatchableChannel(channel)) { + channels.push(channel); + } + + if (channel.threads?.fetch) { + for (const archived of [false, true]) { + const fetched = await channel.threads + .fetch({ archived, limit: 100 }) + .catch(() => null); + if (!fetched?.threads) continue; + for (const thread of fetched.threads.values()) { + if (isWatchableChannel(thread)) channels.push(thread); + } + } + } + } + + return Array.from(new Map(channels.map((channel) => [channel.id, channel])).values()); +} + +async function syncChannelMessages( + db: SqliteDatabase, + channel: any, + cutoffTime: number, +): Promise { + let before: string | undefined; + let synced = 0; + let shouldContinue = true; + + while (shouldContinue) { + const batch = await channel.messages.fetch({ + limit: config.BACKLOG_SYNC_BATCH_SIZE, + ...(before ? { before } : {}), + }); + + if (batch.size === 0) break; + + const messages = Array.from(batch.values()) as Message[]; + for (const message of messages) { + if (message.author?.bot) continue; + if (message.createdTimestamp < cutoffTime) { + shouldContinue = false; + continue; + } + + await captureMessage(db, message, "text"); + synced++; + } + + before = messages[messages.length - 1]?.id; + if (!before || batch.size < config.BACKLOG_SYNC_BATCH_SIZE) break; + } + + return synced; +} + +export async function syncBacklogMessages( + client: Client, + db: SqliteDatabase, +): Promise { + if (!config.MONITOR_GUILD_ID) { + logger.warn("MONITOR_GUILD_ID not configured, skipping backlog sync"); + return; + } + + const guild = client.guilds.cache.get(config.MONITOR_GUILD_ID); + if (!guild) { + logger.warn({ guildId: config.MONITOR_GUILD_ID }, "Monitor guild not found, skipping backlog sync"); + return; + } + + const cutoffTime = Date.now() - config.BACKLOG_SYNC_HOURS * 60 * 60 * 1000; + await guild.channels.fetch().catch(() => null); + + const channels = await collectWatchableChannels(guild); + + let total = 0; + logger.info( + { guildId: guild.id, channels: channels.length, hours: config.BACKLOG_SYNC_HOURS }, + "Starting message backlog sync", + ); + + for (const channel of channels) { + try { + const count = await syncChannelMessages(db, channel as any, cutoffTime); + total += count; + logger.info({ channelId: channel.id, count }, "Backlog channel sync completed"); + } catch (error) { + logger.warn( + { + channelId: channel.id, + error: error instanceof Error ? error.message : String(error), + }, + "Backlog channel sync failed", + ); + } + } + + logger.info({ total }, "Message backlog sync completed"); +} diff --git a/src/moderation/messageCapture.ts b/src/moderation/messageCapture.ts index 4e8ad43..8a963a3 100644 --- a/src/moderation/messageCapture.ts +++ b/src/moderation/messageCapture.ts @@ -1,64 +1,21 @@ -import type { Client, Message, TextChannel, ThreadChannel } from "discord.js-selfbot-v13"; +import type { Client, Message } from "discord.js-selfbot-v13"; import { createChildLogger } from "../logger"; import type { SqliteDatabase } from "../muxer-queue"; import { config } from "../config"; import { insertMessage, insertAttachment } from "./messageStore"; import { processAttachmentUpload } from "./attachmentUploader"; +import { getDisplayContent, getMessageLocation, getMessageMetadata } from "./messageMetadata"; import type { MessageRecord, AttachmentRecord } from "./types"; const logger = createChildLogger("message-capture"); -function getMessageLocation(message: Message): { - channelId: string; - threadId: string | null; - threadName: string | null; -} { - const channel = message.channel as TextChannel | ThreadChannel; - if (!channel.isThread?.()) { - return { channelId: message.channelId, threadId: null, threadName: null }; - } - - return { - channelId: channel.parentId ?? message.channelId, - threadId: channel.id, - threadName: channel.name, - }; -} - -function getStickerMetadata(message: Message): Array<{ - id: string; - name: string; - url: string; -}> { - return Array.from(message.stickers.values()).map((sticker) => ({ - id: sticker.id, - name: sticker.name, - url: sticker.url, - })); -} - -function getDisplayContent(message: Message): string { - if (message.content.trim().length > 0) return message.content; - - const stickers = getStickerMetadata(message); - if (stickers.length > 0) { - return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" "); - } - - return ""; -} - -async function captureMessage( +export async function captureMessage( db: SqliteDatabase, message: Message, type: "text" | "edited" | "deleted", ): Promise { const location = getMessageLocation(message); - const stickers = getStickerMetadata(message); - const metadata = { - stickers, - threadName: location.threadName, - }; + const metadata = getMessageMetadata(message); const messageRecord: MessageRecord = { id: message.id, diff --git a/src/moderation/messageMetadata.ts b/src/moderation/messageMetadata.ts new file mode 100644 index 0000000..17114b7 --- /dev/null +++ b/src/moderation/messageMetadata.ts @@ -0,0 +1,167 @@ +import type { Message, TextChannel, ThreadChannel } from "discord.js-selfbot-v13"; + +export interface MessageLocation { + channelId: string; + threadId: string | null; + threadName: string | null; + channelName: string | null; +} + +export interface RichMessageMetadata { + stickers: Array<{ id: string; name: string; url: string; format: string | null }>; + embeds: Array<{ + title: string | null; + description: string | null; + url: string | null; + color: number | null; + image: string | null; + thumbnail: string | null; + author: { name: string | null; url: string | null; iconURL: string | null } | null; + footer: { text: string | null; iconURL: string | null } | null; + fields: Array<{ name: string; value: string; inline: boolean }>; + }>; + attachments: Array<{ + id: string; + name: string; + url: string; + contentType: string | null; + size: number; + }>; + author: { + id: string; + username: string; + tag: string | null; + avatarURL: string | null; + bot: boolean; + }; + member: { + displayName: string | null; + roles: Array<{ id: string; name: string }>; + joinedTimestamp: number | null; + } | null; + channel: MessageLocation; + reference: { + messageId: string | null; + channelId: string | null; + guildId: string | null; + } | null; +} + +export function getMessageLocation(message: Message): MessageLocation { + const channel = message.channel as TextChannel | ThreadChannel; + if (!channel.isThread?.()) { + return { + channelId: message.channelId, + threadId: null, + threadName: null, + channelName: "name" in channel ? channel.name : null, + }; + } + + return { + channelId: channel.parentId ?? message.channelId, + threadId: channel.id, + threadName: channel.name, + channelName: channel.parent?.name ?? null, + }; +} + +export function getStickerMetadata(message: Message): RichMessageMetadata["stickers"] { + return Array.from(message.stickers.values()).map((sticker) => ({ + id: sticker.id, + name: sticker.name, + url: sticker.url, + format: sticker.format ?? null, + })); +} + +export function getAttachmentMetadata(message: Message): RichMessageMetadata["attachments"] { + return Array.from(message.attachments.values()).map((attachment) => ({ + id: attachment.id, + name: attachment.name || "unknown", + url: attachment.url, + contentType: attachment.contentType ?? null, + size: attachment.size, + })); +} + +export function getEmbedMetadata(message: Message): RichMessageMetadata["embeds"] { + return message.embeds.map((embed) => ({ + title: embed.title ?? null, + description: embed.description ?? null, + url: embed.url ?? null, + color: embed.color ?? null, + image: embed.image?.url ?? null, + thumbnail: embed.thumbnail?.url ?? null, + author: embed.author + ? { + name: embed.author.name ?? null, + url: embed.author.url ?? null, + iconURL: embed.author.iconURL ?? null, + } + : null, + footer: embed.footer + ? { + text: embed.footer.text ?? null, + iconURL: embed.footer.iconURL ?? null, + } + : null, + fields: embed.fields.map((field) => ({ + name: field.name, + value: field.value, + inline: Boolean(field.inline), + })), + })); +} + +export function getMessageMetadata(message: Message): RichMessageMetadata { + const member = message.member; + return { + stickers: getStickerMetadata(message), + embeds: getEmbedMetadata(message), + attachments: getAttachmentMetadata(message), + author: { + id: message.author.id, + username: message.author.username, + tag: "tag" in message.author ? message.author.tag : null, + avatarURL: message.author.avatarURL() ?? null, + bot: Boolean(message.author.bot), + }, + member: member + ? { + displayName: member.displayName ?? null, + roles: member.roles.cache.map((role) => ({ id: role.id, name: role.name })), + joinedTimestamp: member.joinedTimestamp ?? null, + } + : null, + channel: getMessageLocation(message), + reference: message.reference + ? { + messageId: message.reference.messageId ?? null, + channelId: message.reference.channelId ?? null, + guildId: message.reference.guildId ?? null, + } + : null, + }; +} + +export function getDisplayContent(message: Message): string { + if (message.content.trim().length > 0) return message.content; + + const stickers = getStickerMetadata(message); + if (stickers.length > 0) { + return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" "); + } + + const attachments = getAttachmentMetadata(message); + if (attachments.length > 0) { + return attachments.map((attachment) => `[Attachment: ${attachment.name}]`).join(" "); + } + + const embeds = getEmbedMetadata(message); + if (embeds.length > 0) { + return embeds.map((embed) => embed.title || embed.description || "[Embed]").join(" "); + } + + return ""; +} diff --git a/src/moderation/messageStore.ts b/src/moderation/messageStore.ts index 87adfc5..1dcde9e 100644 --- a/src/moderation/messageStore.ts +++ b/src/moderation/messageStore.ts @@ -7,7 +7,7 @@ const logger = createChildLogger("message-store"); export function insertMessage(db: SqliteDatabase, message: MessageRecord): void { try { const stmt = db.prepare(` - INSERT INTO messages ( + INSERT OR IGNORE INTO messages ( id, guild_id, channel_id, thread_id, user_id, username, avatar_url, content, edited_content, created_at, edited_at, deleted_at, type, metadata ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) @@ -115,7 +115,7 @@ export function getMessagesByChannel( export function insertAttachment(db: SqliteDatabase, attachment: AttachmentRecord): void { try { const stmt = db.prepare(` - INSERT INTO attachments ( + INSERT OR IGNORE INTO attachments ( id, message_id, guild_id, channel_id, thread_id, user_id, filename, size, type, discord_url, uploaded_url, upload_status, upload_error, created_at, uploaded_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) diff --git a/src/voiceController.ts b/src/voiceController.ts index 8ff5b50..878c8a2 100644 --- a/src/voiceController.ts +++ b/src/voiceController.ts @@ -73,12 +73,25 @@ export class VoiceController { const guild = this.getGuild(guildId); await guild.channels.fetch().catch(() => null); - return guild.channels.cache - .filter((channel) => - ["GUILD_TEXT", "GUILD_PUBLIC_THREAD", "GUILD_PRIVATE_THREAD"].includes( - channel.type, - ), - ) + const channels = [...guild.channels.cache.values()].filter((channel) => + ["GUILD_TEXT", "GUILD_PUBLIC_THREAD", "GUILD_PRIVATE_THREAD"].includes( + channel.type, + ), + ); + + for (const channel of guild.channels.cache.values()) { + const threadParent = channel as typeof channel & { + threads?: { fetch: (options: { archived: boolean; limit: number }) => Promise }; + }; + if (!threadParent.threads?.fetch) continue; + for (const archived of [false, true]) { + const fetched = await threadParent.threads.fetch({ archived, limit: 100 }).catch(() => null); + if (!fetched?.threads) continue; + channels.push(...fetched.threads.values()); + } + } + + return Array.from(new Map(channels.map((channel) => [channel.id, channel])).values()) .map((channel) => { const parentName = channel.isThread?.() ? channel.parent?.name : null; return {