feat: implement robust JSON parsing and multimodal image capping

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/moderation/llmModerationClient.ts

@@ -17,6 +17,55 @@ interface RawModerationResponse {
   results: RawModerationResult[];
 }
 
+/**
+ * Helper to extract a JSON object from a potentially conversational or markdown-wrapped string.
+ * It first scans for markdown json code blocks, then falls back to trying all start/end brace pairs from largest to smallest.
+ */
+export function extractJson(content: string): any {
+  // 1. Try to find markdown json code blocks: ```json ... ``` or ``` ... ```
+  const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g;
+  const matches = content.matchAll(codeBlockRegex);
+  for (const match of matches) {
+    const codeContent = match[1].trim();
+    try {
+      const parsed = JSON.parse(codeContent);
+      if (parsed && typeof parsed === "object") {
+        return parsed;
+      }
+    } catch (e) {
+      // Continue to next code block
+    }
+  }
+
+  // 2. If no code blocks parse successfully, try scanning for {...} pairs
+  const openBraces: number[] = [];
+  const closeBraces: number[] = [];
+  for (let i = 0; i < content.length; i++) {
+    if (content[i] === "{") openBraces.push(i);
+    if (content[i] === "}") closeBraces.push(i);
+  }
+
+  // Try pairs from largest span to smallest
+  for (const start of openBraces) {
+    for (let j = closeBraces.length - 1; j >= 0; j--) {
+      const end = closeBraces[j];
+      if (end > start) {
+        const candidate = content.substring(start, end + 1);
+        try {
+          const parsed = JSON.parse(candidate);
+          if (parsed && typeof parsed === "object") {
+            return parsed;
+          }
+        } catch (e) {
+          // ignore and try next
+        }
+      }
+    }
+  }
+
+  throw new Error("No JSON object found in response");
+}
+
 /**
  * Parses LLM moderation response and validates against target IDs.
  * Extracts JSON from surrounding text, validates structure, and transforms to AnalysisResult[].
@@ -26,44 +75,8 @@ export function parseModerationResponse(
   content: string,
   targetIds: string[],
 ): AnalysisResult[] {
-  // Find first opening brace and last closing brace
+  // Extract and parse JSON object
-  const startIdx = content.indexOf("{");
+  const parsed = extractJson(content);
-  const endIdx = content.lastIndexOf("}");
-
-  if (startIdx === -1 || endIdx === -1 || endIdx < startIdx) {
-    throw new Error("No JSON object found in response");
-  }
-
-  // Attempt to parse the largest possible JSON object
-  let parsed: unknown;
-  const candidate = content.substring(startIdx, endIdx + 1);
-
-  try {
-    parsed = JSON.parse(candidate);
-  } catch (error) {
-    // If full substring fails, try scanning backwards from the last }
-    let lastError: Error =
-      error instanceof Error ? error : new Error(String(error));
-
-    for (let i = endIdx - 1; i > startIdx; i--) {
-      if (content[i] === "}") {
-        try {
-          parsed = JSON.parse(content.substring(startIdx, i + 1));
-          break;
-        } catch (innerError) {
-          lastError =
-            innerError instanceof Error
-              ? innerError
-              : new Error(String(innerError));
-          continue;
-        }
-      }
-    }
-
-    if (!parsed) {
-      throw new Error(`Failed to parse JSON: ${lastError.message}`);
-    }
-  }
 
   // Validate structure
   if (!parsed || typeof parsed !== "object" || !("results" in parsed)) {
@@ -222,10 +235,21 @@ Each result must have:
 Return ONLY valid JSON, no other text.`;
 
   // Check for image attachments to support multimodal analysis
-  const imageAttachments = (attachments || []).filter(
+  const targetIdSet = new Set(targets.map((t) => t.id));
-    (att) =>
+  const imageAttachments = (attachments || [])
-      (att.uploaded_url || att.discord_url) && att.type.startsWith("image/"),
+    .filter(
-  );
+      (att) =>
+        (att.uploaded_url || att.discord_url) && att.type.startsWith("image/"),
+    )
+    .sort((a, b) => {
+      const aIsTarget = targetIdSet.has(a.message_id) ? 1 : 0;
+      const bIsTarget = targetIdSet.has(b.message_id) ? 1 : 0;
+      if (aIsTarget !== bIsTarget) {
+        return bIsTarget - aIsTarget; // Target messages first
+      }
+      return b.created_at - a.created_at; // Most recent first
+    })
+    .slice(0, 8); // Cap at 8 to prevent LLM API limits (e.g. Nemotron/Omni models 8-image limit)
 
   let messageContent:
     | string

tests/moderation/llmModerationClient.test.ts

@@ -253,6 +253,51 @@ describe("parseModerationResponse", () => {
 
     expect(result[0].score).toBe(0);
   });
+
+  it("extracts JSON correctly from complex conversational output with thinking blocks containing braces", () => {
+    const content = `Based on the messages, I will analyze them.
+    <thinking>
+      The JSON structure should be:
+      {
+        "results": [ ... ]
+      }
+    </thinking>
+    Here is the results array:
+    {
+      "results": [
+        {
+          "message_id": "m1",
+          "status": "clean",
+          "flags": [],
+          "score": 0.2,
+          "analysis": "Benign"
+        }
+      ]
+    }`;
+    const result = parseModerationResponse(content, ["m1"]);
+    expect(result).toHaveLength(1);
+    expect(result[0].messageId).toBe("m1");
+  });
+
+  it("extracts JSON from markdown code block wrapping", () => {
+    const content = `Sure! Here is the JSON structure:
+\`\`\`json
+{
+  "results": [
+    {
+      "message_id": "m1",
+      "status": "clean",
+      "flags": [],
+      "score": 0.2,
+      "analysis": "Benign"
+    }
+  ]
+}
+\`\`\``;
+    const result = parseModerationResponse(content, ["m1"]);
+    expect(result).toHaveLength(1);
+    expect(result[0].messageId).toBe("m1");
+  });
 });
 
 describe("runModerationAnalysis", () => {
@@ -432,4 +477,101 @@ describe("runModerationAnalysis", () => {
       "You are a content moderation assistant.",
     );
   });
+
+  it("caps image attachments to 8 and prioritizes targets over context", async () => {
+    const mockResponse = {
+      choices: [
+        {
+          message: {
+            content: JSON.stringify({
+              results: [
+                {
+                  message_id: "m1",
+                  status: "clean",
+                  flags: [],
+                  score: 0.1,
+                  analysis: "OK",
+                },
+              ],
+            }),
+          },
+        },
+      ],
+    };
+
+    global.fetch = vi.fn().mockImplementation((url: string) => {
+      if (url.includes("picser.tech") || url.includes("discord.com")) {
+        return Promise.resolve({
+          ok: true,
+          arrayBuffer: async () => {
+            const buffer = Buffer.from("fake-bytes");
+            return buffer.buffer.slice(
+              buffer.byteOffset,
+              buffer.byteOffset + buffer.byteLength,
+            );
+          },
+        });
+      }
+      return Promise.resolve({
+        ok: true,
+        text: async () => JSON.stringify(mockResponse),
+        json: async () => mockResponse,
+      });
+    });
+
+    const createAttachment = (id: string, msgId: string, createdAt: number) => ({
+      id,
+      message_id: msgId,
+      guild_id: "guild123",
+      channel_id: "channel123",
+      thread_id: null,
+      user_id: "user123",
+      filename: `${id}.png`,
+      size: 500,
+      type: "image/png",
+      discord_url: `https://discord.com/${id}.png`,
+      uploaded_url: `https://picser.tech/${id}.png`,
+      upload_status: "uploaded" as const,
+      upload_error: null,
+      created_at: createdAt,
+      uploaded_at: createdAt,
+    });
+
+    // 10 attachments total (3 targets, 7 context)
+    const attachments = [
+      createAttachment("c1", "context1", 100),
+      createAttachment("c2", "context2", 200),
+      createAttachment("t1", "m1", 300), // Target 1
+      createAttachment("c3", "context3", 400),
+      createAttachment("t2", "m1", 500), // Target 2
+      createAttachment("c4", "context4", 600),
+      createAttachment("c5", "context5", 700),
+      createAttachment("t3", "m1", 800), // Target 3
+      createAttachment("c6", "context6", 900),
+      createAttachment("c7", "context7", 1000),
+    ];
+
+    await runModerationAnalysis({
+      targets: [createMessageRecord({ id: "m1" })],
+      contextText: "test context",
+      attachments,
+    });
+
+    const fetchCalls = (global.fetch as any).mock.calls;
+    // Should download exactly 8 images (since it's capped at 8) plus 1 call for completion API = 9 calls total.
+    expect(fetchCalls.length).toBe(9);
+
+    // Target attachments (t3, t2, t1) must be fetched, then context in descending order of created_at:
+    // Sorted order: t3 (800), t2 (500), t1 (300), c7 (1000), c6 (900), c5 (700), c4 (600), c3 (400)
+    // Excluded: c2 (200), c1 (100)
+    const downloadedUrls = fetchCalls
+      .slice(0, 8)
+      .map((call: any) => call[0]);
+
+    expect(downloadedUrls).toContain("https://picser.tech/t3.png");
+    expect(downloadedUrls).toContain("https://picser.tech/t2.png");
+    expect(downloadedUrls).toContain("https://picser.tech/t1.png");
+    expect(downloadedUrls).toContain("https://picser.tech/c7.png");
+    expect(downloadedUrls).not.toContain("https://picser.tech/c1.png");
+  });
 });