GMW/services/discord-gateway/src/modules/ai-moderation/urlFetcher.ts

import { resolve } from "node:dns/promises";
import { isIP } from "node:net";
import { LRUCache } from "lru-cache";
import { createChildLogger } from "@/shared/logger/index";
import { createAbortControllerWithTimeout } from "@/shared/utils/index";

const _log = createChildLogger("urlFetcher");

export interface FetchedUrlContext {
  url: string;
  type: "image" | "text" | "error";
  data?: Buffer;
  mimeType?: string;
  textContent?: string;
  /** Page title from og:title / <title> — strong signal for the LLM. */
  title?: string;
  error?: string;
}

const MAX_FETCH_SIZE = 5 * 1024 * 1024; // 5 MB
const FETCH_TIMEOUT_MS = 8000;
const URL_REGEX = /https?:\/\/[^\s<]+[^<.,:;"')\]\s]/gi;

/**
 * Basic SSRF protection.
 * Note: A sophisticated attacker could still use DNS rebinding.
 */
async function isSafeUrl(urlStr: string): Promise<boolean> {
  try {
    const parsed = new URL(urlStr);
    const host = parsed.hostname;

    // Block obvious local IPs/hostnames
    if (
      host === "localhost" ||
      host === "127.0.0.1" ||
      host === "::1" ||
      host.startsWith("192.168.") ||
      host.startsWith("10.") ||
      /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(host)
    ) {
      return false;
    }

    // Try resolving to check if it resolves to a local IP
    if (!isIP(host)) {
      try {
        const addresses = await resolve(host);
        for (const ip of addresses) {
          if (
            ip === "127.0.0.1" ||
            ip.startsWith("192.168.") ||
            ip.startsWith("10.") ||
            /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(ip)
          ) {
            return false;
          }
        }
      } catch (_err) {
        // If DNS fails, we can't fetch it anyway
        return false;
      }
    }

    return true;
  } catch (_err) {
    return false;
  }
}

function extractOgImage(html: string): string | null {
  // Look for <meta ... property="og:image" ... content="..."> or <meta ... name="twitter:image" ... content="...">
  const ogRegex =
    /<meta[^>]*(?:property|name)=["'](?:og:image|twitter:image)["'][^>]*content=["']([^"']+)["']/i;
  const match = html.match(ogRegex);
  if (match?.[1]) {
    // Unescape basic HTML entities
    return match[1].replace(/&amp;/g, "&").replace(/&quot;/g, '"');
  }

  // Try reversed attribute order: <meta ... content="..." ... property="og:image">
  const ogRegexRev =
    /<meta[^>]*content=["']([^"']+)["'][^>]*(?:property|name)=["'](?:og:image|twitter:image)["']/i;
  const matchRev = html.match(ogRegexRev);
  if (matchRev?.[1]) {
    return matchRev[1].replace(/&amp;/g, "&").replace(/&quot;/g, '"');
  }

  return null;
}

export interface OgMeta {
  title: string | null;
  description: string | null;
  siteName: string | null;
}

/**
 * Extracts OpenGraph / twitter meta + <title> from raw HTML. Both attribute
 * orders are accepted (<meta property=... content=...> and reversed).
 */
export function extractOgMeta(html: string): OgMeta {
  const metaValue = (name: string): string | null => {
    const re = new RegExp(
      `<meta[^>]*(?:property|name)=["']${name}["'][^>]*content=["']([^"']+)["']`,
      "i",
    );
    const m = html.match(re);
    if (m?.[1]) return m[1].replace(/&amp;/g, "&").replace(/&quot;/g, '"');
    const reRev = new RegExp(
      `<meta[^>]*content=["']([^"']+)["'][^>]*(?:property|name)=["']${name}["']`,
      "i",
    );
    const mRev = html.match(reRev);
    return mRev?.[1]
      ? mRev[1].replace(/&amp;/g, "&").replace(/&quot;/g, '"')
      : null;
  };

  const title =
    metaValue("og:title") ||
    metaValue("twitter:title") ||
    html.match(/<title[^>]*>([^<]+)<\/title>/i)?.[1]?.trim() ||
    null;
  const description =
    metaValue("og:description") ||
    metaValue("twitter:description") ||
    metaValue("description") ||
    null;
  const siteName =
    metaValue("og:site_name") || metaValue("application-name") || null;

  return { title, description, siteName };
}

function truncateAndCleanHtml(html: string, maxLen = 1000): string {
  // Strip <script> and <style> entirely
  let text = html.replace(
    /<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,
    " ",
  );
  text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, " ");
  // Strip all other HTML tags
  text = text.replace(/<[^>]+>/g, " ");
  // Replace multiple spaces/newlines
  text = text.replace(/\s+/g, " ").trim();

  return text.substring(0, maxLen);
}

export async function fetchUrlSafely(
  url: string,
  depth = 0,
): Promise<FetchedUrlContext> {
  // Text results are memoized (in-process, short TTL): the same link recurs
  // across batches and re-downloading + re-parsing the page each time was
  // pure latency. Images are NEVER cached here — they are vision evidence
  // and multi-MB buffers don't belong in an LRU. Errors are not cached so a
  // transient network blip retries on the next batch.
  if (depth === 0) {
    const memo = textFetchMemo.get(url);
    if (memo) return memo;
    // In-flight dedupe: concurrent callers share one live request.
    const existing = textInFlight.get(url);
    if (existing) return existing;
    const promise = fetchUrlSafelyUncached(url, depth)
      .then((fetched) => {
        if (fetched.type === "text") textFetchMemo.set(url, fetched);
        return fetched;
      })
      .finally(() => {
        textInFlight.delete(url);
      });
    textInFlight.set(url, promise);
    return promise;
  }
  return fetchUrlSafelyUncached(url, depth);
}

/** In-process memo of successful TEXT fetches (30 min TTL, bounded size). */
const textFetchMemo = new LRUCache<string, FetchedUrlContext>({
  max: 500,
  ttl: 30 * 60 * 1000,
});

/** Concurrent same-URL text fetches collapse into one live request. */
const textInFlight = new LRUCache<string, Promise<FetchedUrlContext>>({
  max: 100,
  ttl: 60_000,
});

async function fetchUrlSafelyUncached(
  url: string,
  depth = 0,
): Promise<FetchedUrlContext> {
  if (depth > 1) {
    return { url, type: "error", error: "Max redirect/meta depth reached" };
  }

  if (!(await isSafeUrl(url))) {
    return { url, type: "error", error: "Unsafe URL blocked" };
  }

  const { controller, clear } =
    createAbortControllerWithTimeout(FETCH_TIMEOUT_MS);

  try {
    const response = await fetch(url, {
      signal: controller.signal,
      headers: {
        "User-Agent":
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 DiscordBot/2.0",
        Accept: "image/webp,image/apng,image/*,*/*;q=0.8",
      },
      // Do not follow more than a few redirects natively, fetch handles up to 20 by default
    });

    if (!response.ok) {
      return { url, type: "error", error: `HTTP ${response.status}` };
    }

    const contentType = response.headers.get("content-type") || "";
    const contentLength = parseInt(
      response.headers.get("content-length") || "0",
      10,
    );

    if (contentLength > MAX_FETCH_SIZE) {
      return { url, type: "error", error: "Content too large" };
    }

    const buffer = await response.arrayBuffer();
    if (buffer.byteLength > MAX_FETCH_SIZE) {
      return { url, type: "error", error: "Downloaded content too large" };
    }

    if (contentType.startsWith("image/")) {
      return {
        url,
        type: "image",
        data: Buffer.from(buffer),
        mimeType: contentType,
      };
    }

    if (
      contentType.startsWith("text/html") ||
      contentType.startsWith("text/plain")
    ) {
      const text = Buffer.from(buffer).toString("utf-8");

      // If it's HTML, try to find an og:image first (for Tenor/Giphy etc)
      if (contentType.startsWith("text/html")) {
        const ogImage = extractOgImage(text);
        if (ogImage?.startsWith("http")) {
          // Fetch the og:image instead
          return fetchUrlSafely(ogImage, depth + 1);
        }
      }

      // Fallback to text content
      const cleaned = truncateAndCleanHtml(text, 1000);
      return {
        url,
        type: "text",
        textContent: cleaned,
        title: extractOgMeta(text).title ?? undefined,
      };
    }

    return {
      url,
      type: "error",
      error: `Unsupported content type: ${contentType}`,
    };
  } catch (err) {
    return {
      url,
      type: "error",
      error: err instanceof Error ? err.message : String(err),
    };
  } finally {
    clear();
  }
}

export function extractUrlsFromText(text: string): string[] {
  const matches = text.match(URL_REGEX);
  if (!matches) return [];
  // Deduplicate and filter out things that obviously aren't valid
  return Array.from(new Set(matches)).filter((url) => {
    try {
      new URL(url);
      return true;
    } catch {
      return false;
    }
  });
}