feat: add multimodal analysis support to LLM moderation client by processing image attachments

src/moderation/aiAnalysisWorker.ts

@@ -3,6 +3,7 @@ import { initializeDatabase } from "../database/drizzle.ts";
 import { buildConversationPromptMessages } from "./conversationContext.ts";
 import { runModerationAnalysis } from "./llmModerationClient.ts";
 import {
+  getAttachmentsForMessages,
   getConversationContextBefore,
   updateMessageAIAnalysis,
 } from "./messageStore.ts";
@@ -66,9 +67,15 @@ async function processAnalysisRequest({
       maxTokens: MAX_CONTEXT_TOKENS,
     });
 
+    const targetIds = messages.map((m) => m.id);
+    const contextIds = contextBefore.map((m) => m.id);
+    const allMessageIds = [...targetIds, ...contextIds];
+    const attachments = await getAttachmentsForMessages(allMessageIds);
+
     const result = await runModerationAnalysis({
       targets: messages,
       contextText: promptMessages.join("\n"),
+      attachments,
     });
 
     const rows: MessageRecord[] = [];

src/moderation/aiAnalyzer.ts

@@ -146,7 +146,7 @@ async function runAnalysisInWorker(
   return new Promise((resolve, reject) => {
     const worker = new Worker(
       new URL("./aiAnalysisWorker.ts", import.meta.url),
-      { execArgv: process.execArgv }
+      { execArgv: process.execArgv },
     );
 
     worker.once("message", (response: AnalysisWorkerResponse) => {

src/moderation/llmModerationClient.ts

@@ -1,7 +1,7 @@
 import { config } from "../config.ts";
 import { createChildLogger } from "../logger.ts";
 import { retryWithBackoff } from "../retry.ts";
-import type { AnalysisResult, MessageRecord } from "./types";
+import type { AnalysisResult, AttachmentRecord, MessageRecord } from "./types";
 
 const log = createChildLogger("llmModerationClient");
 
@@ -174,6 +174,7 @@ export function parseModerationResponse(
 interface ModerationInput {
   targets: MessageRecord[];
   contextText: string;
+  attachments?: AttachmentRecord[];
 }
 
 interface ModerationOutput {
@@ -188,7 +189,7 @@ interface ModerationOutput {
 export async function runModerationAnalysis(
   input: ModerationInput,
 ): Promise<ModerationOutput> {
-  const { targets, contextText } = input;
+  const { targets, contextText, attachments } = input;
 
   if (!targets.length) {
     throw new Error("No targets provided for analysis");
@@ -220,6 +221,88 @@ Each result must have:
 
 Return ONLY valid JSON, no other text.`;
 
+  // Check for image attachments to support multimodal analysis
+  const imageAttachments = (attachments || []).filter(
+    (att) =>
+      (att.uploaded_url || att.discord_url) && att.type.startsWith("image/"),
+  );
+
+  let messageContent:
+    | string
+    | Array<{ type: string; text?: string; image_url?: { url: string } }>;
+  if (imageAttachments.length > 0) {
+    const contentParts: Array<{
+      type: string;
+      text?: string;
+      image_url?: { url: string };
+    }> = [];
+
+    // Download and convert all images to base64 data URLs
+    for (const att of imageAttachments) {
+      try {
+        const urlToUse = att.uploaded_url || att.discord_url;
+        log.info(
+          { attachmentId: att.id, url: urlToUse },
+          "Downloading attachment for base64 encoding",
+        );
+        const res = await fetch(urlToUse);
+        if (res.ok) {
+          const buffer = await res.arrayBuffer();
+          const base64Str = Buffer.from(buffer).toString("base64");
+          const dataUrl = `data:${att.type};base64,${base64Str}`;
+
+          contentParts.push({
+            type: "image_url",
+            image_url: {
+              url: dataUrl,
+            },
+          });
+
+          contentParts.push({
+            type: "text",
+            text: `\n[Image Attachment for Message ID: ${att.message_id}, Filename: ${att.filename}]`,
+          });
+        } else {
+          log.warn(
+            { attachmentId: att.id, status: res.status },
+            "Failed to fetch attachment image",
+          );
+        }
+      } catch (err) {
+        log.warn(
+          {
+            attachmentId: att.id,
+            error: err instanceof Error ? err.message : String(err),
+          },
+          "Error base64 encoding attachment",
+        );
+      }
+    }
+
+    contentParts.push({
+      type: "text",
+      text: prompt,
+    });
+
+    messageContent = contentParts;
+  } else {
+    // If no image is present, send a transparent 1x1 dummy PNG to satisfy multimodal omni requirements
+    const dummyPng =
+      "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
+    messageContent = [
+      {
+        type: "image_url",
+        image_url: {
+          url: dummyPng,
+        },
+      },
+      {
+        type: "text",
+        text: prompt,
+      },
+    ];
+  }
+
   const result = await retryWithBackoff(
     async () => {
       const controller = new AbortController();
@@ -243,10 +326,14 @@ Return ONLY valid JSON, no other text.`;
               messages: [
                 {
                   role: "user",
-                  content: prompt,
+                  content: messageContent,
                 },
               ],
-              temperature: 0.3,
+              temperature: 0.6,
+              top_p: 0.95,
+              max_tokens: 65536,
+              reasoning_budget: 16384,
+              chat_template_kwargs: { enable_thinking: true },
             }),
           },
         );

src/moderation/messageStore.ts

@@ -1,4 +1,14 @@
-import { and, asc, desc, eq, isNull, or, type SQL, sql } from "drizzle-orm";
+import {
+  and,
+  asc,
+  desc,
+  eq,
+  inArray,
+  isNull,
+  or,
+  type SQL,
+  sql,
+} from "drizzle-orm";
 import { getDatabase } from "../database/drizzle.ts";
 import { attachmentsTable, messagesTable } from "../database/schema.ts";
 import { createChildLogger } from "../logger.ts";
@@ -605,3 +615,27 @@ export async function getPendingConversationKeys(
     throw error;
   }
 }
+
+export async function getAttachmentsForMessages(
+  messageIds: string[],
+): Promise<AttachmentRecord[]> {
+  try {
+    if (messageIds.length === 0) return [];
+    const database = db();
+    const rows = await database
+      .select()
+      .from(attachmentsTable)
+      .where(inArray(attachmentsTable.message_id, messageIds));
+
+    return rows as AttachmentRecord[];
+  } catch (error) {
+    logger.error(
+      {
+        messageIds,
+        error: error instanceof Error ? error.message : String(error),
+      },
+      "Failed to get attachments for messages",
+    );
+    throw error;
+  }
+}