From 5fc0e8b3cb6b170633368be17824cc5ded63ca5a Mon Sep 17 00:00:00 2001
From: asepharyana <superaseph@gmail.com>
Date: Thu, 24 Sep 2026 18:03:41 +0700
Subject: [PATCH] fix(gateway): surface failed URL fetches to moderation LLM
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Links that fail to fetch (Facebook share 400, login walls, anti-bot)
previously vanished from the prompt entirely — the LLM only saw the raw
URL in the message body, so its analysis degraded to a template like
"Pengguna membagikan tautan Facebook". Now the text batch tracks
failed-fetch URLs and injects <web_content fetch_error="true"> telling
the LLM the page content is unverifiable and to NOT invent it, plus a
prompt rule: describe what the user actually expressed from the text +
conversation context, or admit the subject is unclear — never make the
link itself the subject.
---
 .../modules/ai-moderation/prompts/output.ts   |  2 ++
 .../ai-moderation/textBatchProcessor.ts       | 28 +++++++++++++++----
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/services/discord-gateway/src/modules/ai-moderation/prompts/output.ts b/services/discord-gateway/src/modules/ai-moderation/prompts/output.ts
index 591614ea..43ac9136 100644
--- a/services/discord-gateway/src/modules/ai-moderation/prompts/output.ts
+++ b/services/discord-gateway/src/modules/ai-moderation/prompts/output.ts
@@ -56,6 +56,8 @@ Wajib sebutkan ISI/KONTEN spesifik apa yang dibicarakan pengirim — bukan templ
 
 **CRITICAL — dilarang menulis analysis generik:** JANGAN PERNAH menulis "Pesan hanya berisi...", "Tidak ada indikasi pelanggaran", atau template seperti "Pengirim mengirimkan sebuah file GIF tanpa pelanggaran." Selalu sebutkan ISI/KONTEN spesifik, apa yang dibicarakan, apa yang terlihat.
 
+- **LINK yang gagal diambil (fetch_error=true):** JANGAN jadikan keberadaan link sebagai subjek analysis. <web_content url=... fetch_error="true"> = konten halaman tidak bisa diverifikasi (login wall / anti-bot / HTTP error). Analysis harus: (1) jelaskan apa yang pengirim lakukan/ekspresikan dari TEKS pesan + <conversation_context> (mis. "membagikan video lalu berekspresi 'Auuuu' — nada santai mengekspresikan kekaguman/kesenangan"), (2) jika teks juga minim dan konteks tidak jelas, AKUI dengan jujur: "Isi link tidak dapat diverifikasi otomatis; dari teks pesan dan konteks sekitar, percakapan tampak <santai/sehari-hari/tidak jelas subjeknya>." JANGAN mengarang isi halaman.
+
 - **BALASAN (reply):** jelaskan konteks balasannya (apa dibicarakan, siapa dibalas tanpa nama, bagaimana tanggapan pengirim).
 - Gunakan Media analysis untuk mendeskripsikan gambar. Analisis harus MEMBERI KONTEKS, bukan hanya status.`;
 
diff --git a/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts b/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts
index 93552126..3d514258 100644
--- a/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts
+++ b/services/discord-gateway/src/modules/ai-moderation/textBatchProcessor.ts
@@ -50,6 +50,8 @@ interface UrlFetchResult {
   text: Map<string, string>;
   image: Map<string, { data: Buffer; mimeType: string }>;
   title: Map<string, string>;
+  /** URL → reason the fetch failed (HTTP status / throw / unsupported). */
+  error: Map<string, string>;
 }
 
 /** Provider-reported token usage from a raw LLM payload (may be absent). */
@@ -169,6 +171,7 @@ export async function runTextOnlyBatch(
         text: new Map(),
         image: new Map(),
         title: new Map(),
+        error: new Map(),
       } satisfies UrlFetchResult;
     }
     const results = await Promise.allSettled(
@@ -177,9 +180,13 @@ export async function runTextOnlyBatch(
     const textMap = new Map<string, string>();
     const imageMap = new Map<string, { data: Buffer; mimeType: string }>();
     const titleMap = new Map<string, string>();
+    const errorMap = new Map<string, string>();
     for (let i = 0; i < urlArr.length; i++) {
       const r = results[i];
-      if (r.status !== "fulfilled") continue;
+      if (r.status !== "fulfilled") {
+        errorMap.set(urlArr[i], "fetch threw");
+        continue;
+      }
       const v = r.value;
       if (v.type === "text" && v.textContent) {
         textMap.set(urlArr[i], v.textContent);
@@ -188,9 +195,11 @@ export async function runTextOnlyBatch(
         // Direct image link (or og:image followed from an HTML page) —
         // kept for vision analysis below.
         imageMap.set(urlArr[i], { data: v.data, mimeType: v.mimeType });
+      } else {
+        errorMap.set(urlArr[i], v.error ?? "unsupported content");
       }
     }
-    return { text: textMap, image: imageMap, title: titleMap };
+    return { text: textMap, image: imageMap, title: titleMap, error: errorMap };
   })();
 
   const webSearchPromise = (async () => {
@@ -224,6 +233,7 @@ export async function runTextOnlyBatch(
     glossaryPromise,
   ]);
   const urlFetchMap = urlFetchMaps.text;
+  const urlFetchErrors = urlFetchMaps.error;
 
   // Deduplicate identical short messages
   const shortContentGroups = new Map<string, MessageRecord[]>();
@@ -368,10 +378,16 @@ export async function runTextOnlyBatch(
             const urlContexts = msgUrls
               .map((url) => {
                 const ft = urlFetchMap.get(url);
-                if (!ft) return null;
-                const title = urlTitles.get(url);
-                const titleAttr = title ? ` title="${escapeXml(title)}"` : "";
-                return `<web_content url="${escapeXml(url)}"${titleAttr}>${escapeXml(ft)}</web_content>`;
+                if (ft) {
+                  const title = urlTitles.get(url);
+                  const titleAttr = title ? ` title="${escapeXml(title)}"` : "";
+                  return `<web_content url="${escapeXml(url)}"${titleAttr}>${escapeXml(ft)}</web_content>`;
+                }
+                const fetchError = urlFetchErrors.get(url);
+                if (fetchError) {
+                  return `<web_content url="${escapeXml(url)}" fetch_error="true">Konten tidak dapat diambil otomatis (${escapeXml(fetchError)}). Analisis hanya dari teks pesan; JANGAN mengarang isi halaman.</web_content>`;
+                }
+                return null;
               })
               .filter(Boolean)
               .join("\n");