feat(moderation): implement backlog message synchronization and enhance message metadata handling

2026-05-13 21:04:45 +07:00
parent d55b56c897
commit 0b8111de81
9 changed files with 402 additions and 66 deletions
@@ -0,0 +1,120 @@
+import type { Client, Message } from "discord.js-selfbot-v13";
+import { config } from "../config";
+import { createChildLogger } from "../logger";
+import type { SqliteDatabase } from "../muxer-queue";
+import { captureMessage } from "./messageCapture";
+
+const logger = createChildLogger("backlog-sync");
+
+function isWatchableChannel(channel: { type?: string; messages?: unknown }): boolean {
+  return Boolean(
+    channel.messages &&
+      ["GUILD_TEXT", "GUILD_PUBLIC_THREAD", "GUILD_PRIVATE_THREAD"].includes(
+        channel.type ?? "",
+      ),
+  );
+}
+
+async function collectWatchableChannels(guild: any): Promise<any[]> {
+  const channels: any[] = [];
+  for (const channel of guild.channels.cache.values()) {
+    if (isWatchableChannel(channel)) {
+      channels.push(channel);
+    }
+
+    if (channel.threads?.fetch) {
+      for (const archived of [false, true]) {
+        const fetched = await channel.threads
+          .fetch({ archived, limit: 100 })
+          .catch(() => null);
+        if (!fetched?.threads) continue;
+        for (const thread of fetched.threads.values()) {
+          if (isWatchableChannel(thread)) channels.push(thread);
+        }
+      }
+    }
+  }
+
+  return Array.from(new Map(channels.map((channel) => [channel.id, channel])).values());
+}
+
+async function syncChannelMessages(
+  db: SqliteDatabase,
+  channel: any,
+  cutoffTime: number,
+): Promise<number> {
+  let before: string | undefined;
+  let synced = 0;
+  let shouldContinue = true;
+
+  while (shouldContinue) {
+    const batch = await channel.messages.fetch({
+      limit: config.BACKLOG_SYNC_BATCH_SIZE,
+      ...(before ? { before } : {}),
+    });
+
+    if (batch.size === 0) break;
+
+    const messages = Array.from(batch.values()) as Message[];
+    for (const message of messages) {
+      if (message.author?.bot) continue;
+      if (message.createdTimestamp < cutoffTime) {
+        shouldContinue = false;
+        continue;
+      }
+
+      await captureMessage(db, message, "text");
+      synced++;
+    }
+
+    before = messages[messages.length - 1]?.id;
+    if (!before || batch.size < config.BACKLOG_SYNC_BATCH_SIZE) break;
+  }
+
+  return synced;
+}
+
+export async function syncBacklogMessages(
+  client: Client,
+  db: SqliteDatabase,
+): Promise<void> {
+  if (!config.MONITOR_GUILD_ID) {
+    logger.warn("MONITOR_GUILD_ID not configured, skipping backlog sync");
+    return;
+  }
+
+  const guild = client.guilds.cache.get(config.MONITOR_GUILD_ID);
+  if (!guild) {
+    logger.warn({ guildId: config.MONITOR_GUILD_ID }, "Monitor guild not found, skipping backlog sync");
+    return;
+  }
+
+  const cutoffTime = Date.now() - config.BACKLOG_SYNC_HOURS * 60 * 60 * 1000;
+  await guild.channels.fetch().catch(() => null);
+
+  const channels = await collectWatchableChannels(guild);
+
+  let total = 0;
+  logger.info(
+    { guildId: guild.id, channels: channels.length, hours: config.BACKLOG_SYNC_HOURS },
+    "Starting message backlog sync",
+  );
+
+  for (const channel of channels) {
+    try {
+      const count = await syncChannelMessages(db, channel as any, cutoffTime);
+      total += count;
+      logger.info({ channelId: channel.id, count }, "Backlog channel sync completed");
+    } catch (error) {
+      logger.warn(
+        {
+          channelId: channel.id,
+          error: error instanceof Error ? error.message : String(error),
+        },
+        "Backlog channel sync failed",
+      );
+    }
+  }
+
+  logger.info({ total }, "Message backlog sync completed");
+}
@@ -1,64 +1,21 @@
-import type { Client, Message, TextChannel, ThreadChannel } from "discord.js-selfbot-v13";
+import type { Client, Message } from "discord.js-selfbot-v13";
 import { createChildLogger } from "../logger";
 import type { SqliteDatabase } from "../muxer-queue";
 import { config } from "../config";
 import { insertMessage, insertAttachment } from "./messageStore";
 import { processAttachmentUpload } from "./attachmentUploader";
+import { getDisplayContent, getMessageLocation, getMessageMetadata } from "./messageMetadata";
 import type { MessageRecord, AttachmentRecord } from "./types";

 const logger = createChildLogger("message-capture");

-function getMessageLocation(message: Message): {
-  channelId: string;
-  threadId: string | null;
-  threadName: string | null;
-} {
-  const channel = message.channel as TextChannel | ThreadChannel;
-  if (!channel.isThread?.()) {
-    return { channelId: message.channelId, threadId: null, threadName: null };
-  }
-
-  return {
-    channelId: channel.parentId ?? message.channelId,
-    threadId: channel.id,
-    threadName: channel.name,
-  };
-}
-
-function getStickerMetadata(message: Message): Array<{
-  id: string;
-  name: string;
-  url: string;
-}> {
-  return Array.from(message.stickers.values()).map((sticker) => ({
-    id: sticker.id,
-    name: sticker.name,
-    url: sticker.url,
-  }));
-}
-
-function getDisplayContent(message: Message): string {
-  if (message.content.trim().length > 0) return message.content;
-
-  const stickers = getStickerMetadata(message);
-  if (stickers.length > 0) {
-    return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" ");
-  }
-
-  return "";
-}
-
-async function captureMessage(
+export async function captureMessage(
  db: SqliteDatabase,
  message: Message,
  type: "text" | "edited" | "deleted",
 ): Promise<void> {
  const location = getMessageLocation(message);
-  const stickers = getStickerMetadata(message);
-  const metadata = {
-    stickers,
-    threadName: location.threadName,
-  };
+  const metadata = getMessageMetadata(message);

  const messageRecord: MessageRecord = {
    id: message.id,
@@ -0,0 +1,167 @@
+import type { Message, TextChannel, ThreadChannel } from "discord.js-selfbot-v13";
+
+export interface MessageLocation {
+  channelId: string;
+  threadId: string | null;
+  threadName: string | null;
+  channelName: string | null;
+}
+
+export interface RichMessageMetadata {
+  stickers: Array<{ id: string; name: string; url: string; format: string | null }>;
+  embeds: Array<{
+    title: string | null;
+    description: string | null;
+    url: string | null;
+    color: number | null;
+    image: string | null;
+    thumbnail: string | null;
+    author: { name: string | null; url: string | null; iconURL: string | null } | null;
+    footer: { text: string | null; iconURL: string | null } | null;
+    fields: Array<{ name: string; value: string; inline: boolean }>;
+  }>;
+  attachments: Array<{
+    id: string;
+    name: string;
+    url: string;
+    contentType: string | null;
+    size: number;
+  }>;
+  author: {
+    id: string;
+    username: string;
+    tag: string | null;
+    avatarURL: string | null;
+    bot: boolean;
+  };
+  member: {
+    displayName: string | null;
+    roles: Array<{ id: string; name: string }>;
+    joinedTimestamp: number | null;
+  } | null;
+  channel: MessageLocation;
+  reference: {
+    messageId: string | null;
+    channelId: string | null;
+    guildId: string | null;
+  } | null;
+}
+
+export function getMessageLocation(message: Message): MessageLocation {
+  const channel = message.channel as TextChannel | ThreadChannel;
+  if (!channel.isThread?.()) {
+    return {
+      channelId: message.channelId,
+      threadId: null,
+      threadName: null,
+      channelName: "name" in channel ? channel.name : null,
+    };
+  }
+
+  return {
+    channelId: channel.parentId ?? message.channelId,
+    threadId: channel.id,
+    threadName: channel.name,
+    channelName: channel.parent?.name ?? null,
+  };
+}
+
+export function getStickerMetadata(message: Message): RichMessageMetadata["stickers"] {
+  return Array.from(message.stickers.values()).map((sticker) => ({
+    id: sticker.id,
+    name: sticker.name,
+    url: sticker.url,
+    format: sticker.format ?? null,
+  }));
+}
+
+export function getAttachmentMetadata(message: Message): RichMessageMetadata["attachments"] {
+  return Array.from(message.attachments.values()).map((attachment) => ({
+    id: attachment.id,
+    name: attachment.name || "unknown",
+    url: attachment.url,
+    contentType: attachment.contentType ?? null,
+    size: attachment.size,
+  }));
+}
+
+export function getEmbedMetadata(message: Message): RichMessageMetadata["embeds"] {
+  return message.embeds.map((embed) => ({
+    title: embed.title ?? null,
+    description: embed.description ?? null,
+    url: embed.url ?? null,
+    color: embed.color ?? null,
+    image: embed.image?.url ?? null,
+    thumbnail: embed.thumbnail?.url ?? null,
+    author: embed.author
+      ? {
+          name: embed.author.name ?? null,
+          url: embed.author.url ?? null,
+          iconURL: embed.author.iconURL ?? null,
+        }
+      : null,
+    footer: embed.footer
+      ? {
+          text: embed.footer.text ?? null,
+          iconURL: embed.footer.iconURL ?? null,
+        }
+      : null,
+    fields: embed.fields.map((field) => ({
+      name: field.name,
+      value: field.value,
+      inline: Boolean(field.inline),
+    })),
+  }));
+}
+
+export function getMessageMetadata(message: Message): RichMessageMetadata {
+  const member = message.member;
+  return {
+    stickers: getStickerMetadata(message),
+    embeds: getEmbedMetadata(message),
+    attachments: getAttachmentMetadata(message),
+    author: {
+      id: message.author.id,
+      username: message.author.username,
+      tag: "tag" in message.author ? message.author.tag : null,
+      avatarURL: message.author.avatarURL() ?? null,
+      bot: Boolean(message.author.bot),
+    },
+    member: member
+      ? {
+          displayName: member.displayName ?? null,
+          roles: member.roles.cache.map((role) => ({ id: role.id, name: role.name })),
+          joinedTimestamp: member.joinedTimestamp ?? null,
+        }
+      : null,
+    channel: getMessageLocation(message),
+    reference: message.reference
+      ? {
+          messageId: message.reference.messageId ?? null,
+          channelId: message.reference.channelId ?? null,
+          guildId: message.reference.guildId ?? null,
+        }
+      : null,
+  };
+}
+
+export function getDisplayContent(message: Message): string {
+  if (message.content.trim().length > 0) return message.content;
+
+  const stickers = getStickerMetadata(message);
+  if (stickers.length > 0) {
+    return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" ");
+  }
+
+  const attachments = getAttachmentMetadata(message);
+  if (attachments.length > 0) {
+    return attachments.map((attachment) => `[Attachment: ${attachment.name}]`).join(" ");
+  }
+
+  const embeds = getEmbedMetadata(message);
+  if (embeds.length > 0) {
+    return embeds.map((embed) => embed.title || embed.description || "[Embed]").join(" ");
+  }
+
+  return "";
+}
@@ -7,7 +7,7 @@ const logger = createChildLogger("message-store");
 export function insertMessage(db: SqliteDatabase, message: MessageRecord): void {
  try {
    const stmt = db.prepare(`
-      INSERT INTO messages (
+      INSERT OR IGNORE INTO messages (
        id, guild_id, channel_id, thread_id, user_id, username, avatar_url,
        content, edited_content, created_at, edited_at, deleted_at, type, metadata
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
@@ -115,7 +115,7 @@ export function getMessagesByChannel(
 export function insertAttachment(db: SqliteDatabase, attachment: AttachmentRecord): void {
  try {
    const stmt = db.prepare(`
-      INSERT INTO attachments (
+      INSERT OR IGNORE INTO attachments (
        id, message_id, guild_id, channel_id, thread_id, user_id, filename, size, type,
        discord_url, uploaded_url, upload_status, upload_error, created_at, uploaded_at
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)