feat: search ALL messages via SearXNG + Redis cache dedup

- Remove shouldSearchContent() trigger gate — search runs on all messages
- extractSearchQueries() now extracts from ANY message, not just trigger-matched
- Redis cache (24h TTL) prevents redundant searches for same query
- initSearxngCache() lazy-connects via config.REDIS_URL
- Cache miss→API, hit→skip — fire-and-forget writes
- Both text batch + media path simplified

services/discord-gateway/src/modules/ai-moderation/llmModerationClient.ts

@@ -22,9 +22,9 @@ import { buildSystemPrompt as buildSystemPromptModular, sanitizeAiContent } from
 import { logModerationAnalysis, logModerationError } from "./responseLogger.js";
 import {
   searchSearxng,
-  shouldSearchContent,
   extractSearchQueries,
   formatSearchResults,
+  initSearxngCache,
 } from "./searxngSearch.js";
 import {
   getStickerFromCache,
@@ -746,17 +746,14 @@ async function runTextOnlyBatch(
   // Uses SearXNG to look up references mentioned in messages (e.g. anime
   // titles, drug names). Results are injected as <web_search> XML tags
   // so the LLM can make informed decisions instead of guessing.
+  // All messages are searched — Redis cache prevents redundant lookups.
   const searxngResults = new Map<string, string>(); // query → formatted XML
   {
     const queries = new Set<string>();
     for (const msg of targets) {
       const content = msg.edited_content ?? msg.content;
-      if (shouldSearchContent(content)) {
-        // Extract specific search terms from triggers (e.g. "boku no pico")
-        // instead of sending the entire message as a query
-        for (const q of extractSearchQueries(content)) {
-          queries.add(q);
-        }
+      for (const q of extractSearchQueries(content)) {
+        queries.add(q);
       }
     }
     if (queries.size > 0) {
@@ -1114,7 +1111,7 @@ async function prepareMediaMessage(
 
   // ── 5. SearXNG search for suspicious content ──
   let searxngXml = "";
-  if (shouldSearchContent(content)) {
+  {
     const queries = extractSearchQueries(content);
     if (queries.length > 0) {
       const results = await Promise.allSettled(
@@ -1318,6 +1315,9 @@ export async function runModerationAnalysis(
 ): Promise<ModerationOutput> {
   const { targets, contextText, attachments } = input;
 
+  // Lazy init SearXNG Redis cache (once per process)
+  initSearxngCache(config.REDIS_URL);
+
   if (!targets.length) {
     throw new Error("No targets provided for analysis");
   }

services/discord-gateway/src/modules/ai-moderation/searxngSearch.ts

@@ -1,3 +1,4 @@
+import Redis from "ioredis";
 import { createChildLogger } from "@bete/shared/logger";
 
 const log = createChildLogger("searxng-search");
@@ -5,6 +6,35 @@ const log = createChildLogger("searxng-search");
 const SEARXNG_BASE_URL = "https://searxng.imrnes.team";
 const MAX_RESULTS = 3;
 const TIMEOUT_MS = 8000;
+const CACHE_TTL = 86400; // 24 hours
+const CACHE_PREFIX = "searxng:";
+
+let redis: Redis | null = null;
+
+/**
+ * Initialize Redis connection for SearXNG cache.
+ * Safe to call multiple times — only creates one connection.
+ */
+export function initSearxngCache(redisUrl: string): void {
+  if (redis) return;
+  redis = new Redis(redisUrl, {
+    maxRetriesPerRequest: 3,
+    retryStrategy(times) {
+      const delay = Math.min(times * 200, 2000);
+      return delay;
+    },
+    lazyConnect: true,
+    enableReadyCheck: false,
+  });
+  redis.on("error", (err) => {
+    log.warn({ err: err.message }, "SearXNG Redis cache error");
+  });
+  redis.connect().catch(() => {
+    log.warn("SearXNG Redis cache unavailable — falling back to no-cache");
+    redis = null;
+  });
+  log.info("SearXNG Redis cache initialized");
+}
 
 export interface SearxngResult {
   title: string;
@@ -14,12 +44,28 @@ export interface SearxngResult {
 
 /**
  * Search SearXNG for a query and return structured results.
- * Safe-guarded: only text results, no images fetched.
+ * Uses Redis cache when available — same query within 24h returns cached results.
  */
 export async function searchSearxng(
   query: string,
   category: "general" | "news" | "science" = "general",
 ): Promise<SearxngResult[]> {
+  const cacheKey = `${CACHE_PREFIX}${category}:${query.toLowerCase().trim()}`;
+
+  // Try cache first
+  if (redis) {
+    try {
+      const cached = await redis.get(cacheKey);
+      if (cached) {
+        log.debug({ query, category }, "SearXNG cache HIT");
+        return JSON.parse(cached) as SearxngResult[];
+      }
+    } catch {
+      // Cache read failed, continue to API
+    }
+  }
+
+  // Cache miss — hit SearXNG API
   try {
     const url = `${SEARXNG_BASE_URL}/search?q=${encodeURIComponent(query)}&format=json&language=id&categories=${category}`;
     const controller = new AbortController();
@@ -44,12 +90,21 @@ export async function searchSearxng(
       results?: Array<{ title?: string; url?: string; content?: string }>;
     };
     const results = data.results ?? [];
-
-    return results.slice(0, MAX_RESULTS).map((r) => ({
+    const mapped = results.slice(0, MAX_RESULTS).map((r) => ({
       title: r.title ?? "",
       url: r.url ?? "",
       snippet: (r.content ?? "").slice(0, 500),
     }));
+
+    // Store in cache (fire and forget — don't block on write)
+    if (redis) {
+      redis.setex(cacheKey, CACHE_TTL, JSON.stringify(mapped)).catch(() => {
+        // Cache write failed silently
+      });
+    }
+
+    log.debug({ query, category, resultCount: mapped.length }, "SearXNG search OK");
+    return mapped;
   } catch (err) {
     log.warn(
       { error: err instanceof Error ? err.message : String(err), query },
@@ -60,68 +115,67 @@ export async function searchSearxng(
 }
 
 /**
- * Trigger rules — each entry is a regex pattern.
- * The matched text is extracted as the search query.
- * Use capturing groups to isolate the specific term to search.
- */
-const SEARXNG_TRIGGERS: RegExp[] = [
-  // Anime rujukan/konten mencurigakan — cari judul yang disebut
-  /\b(nonton\s+\w+(?:\s+\w+){0,4}\s+(anime|kartun|film))\b/i,
-  /\b(tonton\s+\w+(?:\s+\w+){0,4}\s+(anime|kartun|film))\b/i,
-  /\b(rekomendasi\s+(anime|kartun|film)\s+\w+)\b/i,
-  // Istilah konten dewasa/seksual dalam konteks anime/media
-  /\b(anime\s*(18\+|dewasa|bokep|hentai))\b/i,
-  /\b(kartun\s*(18\+|dewasa|bokep))\b/i,
-  /\b(l[o0]l[i1]|sh[o0]t[o0]|lolicon|shotacon)\b/i,
-  /\bhentai\b/i,
-  // Kata "nonton" + sesuatu yang mungkin judul konten
-  /\bnonton\s+(bokep|porno|dewasa|18)\b/i,
-  // Narkoba
-  /\b(jenis?\s+?narkoba|jenis?\s+?narkotika|ngefly|fly\s*high|research\s*chemical|rc\s+drugs)\b/i,
-  // Scam/phishing
-  /\b(phishing|penipuan|scam|skimming)\b/i,
-  // Judi online
-  /\b(situs\s+judi|jud\s*online|slot\s+gacor|deposit\s+jud|bandar\s+(togel|slot))\b/i,
-  // SARA/penistaan — istilah agama yang mungkin diparodikan
-  /\b(kitab\s+(suc|palsu)|nabi\s+palsu|agama\s+palsu|membuat\s+agama)\b/i,
-];
-
-/**
- * Determine if content should trigger a SearXNG lookup.
- * Only search for suspicious/ambiguous content to avoid unnecessary cost.
- */
-export function shouldSearchContent(content: string): boolean {
-  return SEARXNG_TRIGGERS.some((re) => re.test(content));
-}
-
-/**
- * Extract specific search terms from content based on trigger matches.
- * Returns up to 3 clean queries (e.g. ["boku no pico", "sexual_deviation"])
- * instead of the entire message text.
+ * Extract meaningful search queries from message content.
+ * Uses multiple strategies to find terms worth searching.
+ * Returns up to 3 clean queries.
  */
 export function extractSearchQueries(content: string): string[] {
   const queries = new Set<string>();
 
-  // Also check for quoted phrases (they're explicit intent)
+  // 1. Quoted phrases (explicit user intent)
   const quotedPhrases = content.match(/"([^"]+)"|'([^']+)'/g);
   if (quotedPhrases) {
     for (const phrase of quotedPhrases) {
-      const clean = phrase.replace(/["']/g, "").trim().toLowerCase();
+      const clean = phrase.replace(/["']/g, "").trim();
       if (clean.length >= 3) queries.add(clean);
     }
   }
 
-  // Extract matched groups from trigger patterns
-  for (const re of SEARXNG_TRIGGERS) {
-    const match = content.match(re);
-    if (match) {
-      // Use the first capture group (the specific term) if available
-      const term = match[1] ?? match[2] ?? match[0];
-      const clean = term.replace(/\s+/g, " ").trim().toLowerCase();
-      if (clean.length >= 3) queries.add(clean);
+  // 2. "nonton X" pattern — extract the title
+  const nontonMatch = content.match(
+    /\b(nonton|tonton|rekomen|cari|search|google)\s+(.+?)(?:\s+(?:anime|kartun|film|movie|series|serial))?\s*[!?.]*$/i,
+  );
+  if (nontonMatch) {
+    const title = nontonMatch[2].trim();
+    if (title.length >= 2 && title.length <= 80) {
+      queries.add(title);
     }
   }
 
+  // 3. "X anime/film" pattern — title before category
+  const titleBeforeCategory = content.match(
+    /\b(\w[\w\s]{2,40})\s+(?:anime|kartun|film|movie|series|serial)\b/i,
+  );
+  if (titleBeforeCategory) {
+    const title = titleBeforeCategory[1].trim();
+    if (title.length >= 3 && !/^(yang|yang|sama|dari|untuk|ini|itu|ada)$/i.test(title)) {
+      queries.add(title);
+    }
+  }
+
+  // 4. Standalone proper nouns (2+ words, capitalized) that look like titles
+  const properNouns = content.match(
+    /\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,4})\b/g,
+  );
+  if (properNouns) {
+    for (const noun of properNouns) {
+      // Skip common non-title proper nouns
+      const skip = /^(Discord|YouTube|Google|Facebook|Instagram|Twitter|Github|ChatGPT|OpenAI|Claude|Telegram|WhatsApp|TikTok|Netflix|Spotify|Steam|Instagram)$/i;
+      if (!skip.test(noun) && noun.length >= 5) {
+        queries.add(noun);
+      }
+    }
+  }
+
+  // 5. Terms that suggest research intent
+  const researchTerms = content.match(
+    /\b(apa\s+(?:itu|sih)|what\s+is|siapa\s+itu|who\s+is|arti|meaning|definisi|definition)\s+(.{3,60})/i,
+  );
+  if (researchTerms) {
+    const term = researchTerms[2].trim().replace(/[?!.]+$/, "");
+    if (term.length >= 3) queries.add(term);
+  }
+
   return Array.from(queries).slice(0, 3);
 }