feat(core): implement data retention, metrics, and enhanced media handling

This commit introduces several significant improvements across the backend and gateway services:

- **Data Retention**: Added an automated cleanup scheduler in `discord-gateway` to prune expired messages, attachments, and voice recordings based on configurable retention policies.
- **Observability**: Integrated `prom-client` in the `backend` service to expose Prometheus metrics via `/api/metrics` and added default Node.js runtime metrics.
- **Media Handling**: Enhanced `MediaHandler` in `discord-gateway` to support media URL resolution and improved playback status tracking.
- **API & Config**: Expanded the configuration endpoint to expose more system settings and reorganized `.env.example` for better readability.
- **Refactoring & Cleanup**:
    - Removed unused `better-sqlite3` dependency.
    - Refactored voice channel routing.
    - Improved error handling and testing coverage with comprehensive unit tests for shared utilities and error classes.
- **Documentation**: Added `MEMORY.md` for project context.

services/backend/src/modules/config/config.routes.ts

@@ -9,6 +9,18 @@ export function createConfigRouter(): Router {
   router.get("/config", (_req, res) => {
     res.json({
       monitorGuildId: config.MONITOR_GUILD_ID || null,
+      webserverPort: config.WEBSERVER_PORT,
+      nodeEnv: config.NODE_ENV,
+      backlogSyncHours: config.BACKLOG_SYNC_HOURS,
+      backlogSyncBatchSize: config.BACKLOG_SYNC_BATCH_SIZE,
+      retentionMessagesDays: config.RETENTION_MESSAGES_DAYS,
+      retentionAttachmentsDays: config.RETENTION_ATTACHMENTS_DAYS,
+      retentionVoiceDays: config.RETENTION_VOICE_DAYS,
+      autoDeleteFlaggedEnabled: config.AUTO_DELETE_FLAGGED_ENABLED,
+      aiAnalysisEnabled: config.AI_ANALYSIS_ENABLED,
+      voiceGuildId: config.VOICE_GUILD_ID || null,
+      voiceChannelId: config.VOICE_CHANNEL_ID || null,
+      logLevel: config.LOG_LEVEL,
     });
   });
 

services/backend/src/modules/health/health.controller.ts

@@ -1,4 +1,5 @@
 import type { Request, Response } from "express";
+import { register } from "prom-client";
 import { asyncHandler } from "../../shared/middlewares/index.js";
 import { healthService } from "./health.service.js";
 
@@ -10,3 +11,10 @@ export const handleHealthCheck = asyncHandler(
     res.status(status).json(result);
   },
 );
+
+export const handleMetrics = asyncHandler(
+  async (_req: Request, res: Response) => {
+    res.set("Content-Type", register.contentType);
+    res.end(await register.metrics());
+  },
+);

services/backend/src/modules/health/health.routes.ts

@@ -1,6 +1,11 @@
+import { collectDefaultMetrics, register } from "prom-client";
 import type { Router } from "express";
 import express from "express";
-import { handleHealthCheck } from "./health.controller.js";
+import { handleHealthCheck, handleMetrics } from "./health.controller.js";
+
+// Initialize default Node.js runtime metrics (event loop lag, memory, GC, etc.)
+// Called once at module load, not per-request.
+collectDefaultMetrics();
 
 export function createHealthRouter(): Router {
   const router = express.Router();
@@ -8,5 +13,8 @@ export function createHealthRouter(): Router {
   // GET /api/health
   router.get("/health", handleHealthCheck);
 
+  // GET /api/metrics — Prometheus scrape endpoint
+  router.get("/metrics", handleMetrics);
+
   return router;
 }

services/backend/src/modules/voice/guilds.routes.ts

@@ -2,7 +2,11 @@ import { createChildLogger } from "@bete/shared/logger";
 import type { Request, Response, Router } from "express";
 import express from "express";
 import { asyncHandler } from "../../shared/middlewares/index.js";
-import { getGuilds, getTextChannels } from "./voice.service.js";
+import {
+  getGuilds,
+  getTextChannels,
+  getVoiceChannels,
+} from "./voice.service.js";
 
 const logger = createChildLogger("guilds.routes");
 
@@ -32,5 +36,18 @@ export function createGuildsRouter(): Router {
     }),
   );
 
+  // GET /api/guilds/:guildId/voice-channels
+  router.get(
+    "/:guildId/voice-channels",
+    asyncHandler(async (req: Request, res: Response) => {
+      const guildId = Array.isArray(req.params.guildId)
+        ? req.params.guildId[0]
+        : req.params.guildId;
+      logger.debug({ guildId }, "Fetching voice channels");
+      const channels = await getVoiceChannels(guildId);
+      res.json(channels);
+    }),
+  );
+
   return router;
 }

services/backend/src/modules/voice/voice.routes.ts

@@ -3,7 +3,6 @@ import express from "express";
 import {
   handleConnectVoice,
   handleDisconnectVoice,
-  handleGetVoiceChannels,
   handleGetVoiceStatus,
   handleVoiceCommand,
 } from "./voice.controller.js";
@@ -20,9 +19,6 @@ export function createVoiceRouter(): Router {
   // POST /api/disconnect
   router.post("/disconnect", handleDisconnectVoice);
 
-  // GET /api/guilds/:guildId/voice-channels
-  router.get("/guilds/:guildId/voice-channels", handleGetVoiceChannels);
-
   // POST /api/voice/command — send arbitrary voice command (transmit start/stop)
   router.post("/voice/command", handleVoiceCommand);
 

services/backend/tests/placeholder.test.ts

@@ -1,7 +1,268 @@
-import { describe, expect, it } from "vitest";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
 
-describe("backend", () => {
-  it("should load without errors", () => {
-    expect(true).toBe(true);
+// ─── Shared Error Classes ────────────────────────────────────────────────────
+import {
+  AppError,
+  NotFoundError,
+  ValidationError,
+  UnauthorizedError,
+  DatabaseError,
+  ConfigError,
+} from "@bete/shared/errors";
+
+// ─── Shared utilities ─────────────────────────────────────────────────────────
+import { delay, retryWithBackoff, encodeCursor, decodeCursor, pageResult } from "@bete/shared/utils";
+
+// ─── Backend middleware ──────────────────────────────────────────────────────
+import { asyncHandler, requireParam } from "../src/shared/middlewares/index.js";
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// 1. AppError / Error Hierarchy Tests
+// ═══════════════════════════════════════════════════════════════════════════════
+describe("AppError subclasses", () => {
+  it("AppError stores message, code, statusCode, and details", () => {
+    const err = new AppError("custom", "CUSTOM", 418, { reason: "teapot" });
+    expect(err).toBeInstanceOf(Error);
+    expect(err.message).toBe("custom");
+    expect(err.code).toBe("CUSTOM");
+    expect(err.statusCode).toBe(418);
+    expect(err.details).toEqual({ reason: "teapot" });
+    expect(err.name).toBe("AppError");
+  });
+
+  it("AppError defaults statusCode to 500", () => {
+    const err = new AppError("msg", "X");
+    expect(err.statusCode).toBe(500);
+  });
+
+  it("NotFoundError has 404 status and formatted message", () => {
+    const err = new NotFoundError("User");
+    expect(err).toBeInstanceOf(AppError);
+    expect(err.statusCode).toBe(404);
+    expect(err.code).toBe("NOT_FOUND");
+    expect(err.message).toBe("User not found");
+    expect(err.name).toBe("NotFoundError");
+  });
+
+  it("NotFoundError appends id when provided", () => {
+    const err = new NotFoundError("Message", "abc-123");
+    expect(err.message).toBe("Message not found: abc-123");
+  });
+
+  it("ValidationError has 400 status and forwards details", () => {
+    const details = { field: "email" };
+    const err = new ValidationError("Invalid input", details);
+    expect(err).toBeInstanceOf(AppError);
+    expect(err.statusCode).toBe(400);
+    expect(err.code).toBe("VALIDATION_ERROR");
+    expect(err.details).toBe(details);
+    expect(err.name).toBe("ValidationError");
+  });
+
+  it("UnauthorizedError has 401 status and default message", () => {
+    const err = new UnauthorizedError();
+    expect(err.statusCode).toBe(401);
+    expect(err.code).toBe("UNAUTHORIZED");
+    expect(err.message).toBe("Unauthorized");
+  });
+
+  it("UnauthorizedError accepts custom message", () => {
+    const err = new UnauthorizedError("Access denied");
+    expect(err.message).toBe("Access denied");
+  });
+
+  it("DatabaseError has 500 status and forwards details", () => {
+    const err = new DatabaseError("DB down", { cause: "timeout" });
+    expect(err.statusCode).toBe(500);
+    expect(err.code).toBe("DATABASE_ERROR");
+    expect(err.details).toEqual({ cause: "timeout" });
+  });
+
+  it("ConfigError has 500 status", () => {
+    const err = new ConfigError("Bad config");
+    expect(err.statusCode).toBe(500);
+    expect(err.code).toBe("CONFIG_ERROR");
+  });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// 2. Utility Function Tests
+// ═══════════════════════════════════════════════════════════════════════════════
+describe("delay", () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("resolves after the given time", async () => {
+    vi.useFakeTimers();
+    const promise = delay(500);
+    vi.advanceTimersByTime(500);
+    await expect(promise).resolves.toBeUndefined();
+  });
+
+  it("rejects are not triggered on non-matching timer", async () => {
+    vi.useFakeTimers();
+    const promise = delay(1000);
+    // Advance only part way — the timer should NOT fire yet
+    vi.advanceTimersByTime(500);
+    // The timer is still pending; the promise has not resolved yet
+    // We advance the rest
+    vi.advanceTimersByTime(500);
+    await expect(promise).resolves.toBeUndefined();
+  });
+});
+
+describe("retryWithBackoff", () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("returns the result on first success without retrying", async () => {
+    const fn = vi.fn().mockResolvedValue("ok");
+    await expect(retryWithBackoff(fn)).resolves.toBe("ok");
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it("re-throws after exhausting all retries", async () => {
+    const fn = vi.fn().mockRejectedValue(new Error("persistent"));
+    await expect(
+      retryWithBackoff(fn, { retries: 1, minTimeout: 1, maxTimeout: 5 }),
+    ).rejects.toThrow("persistent");
+    // initial call + 1 retry
+    expect(fn.mock.calls.length).toBeGreaterThanOrEqual(2);
+  });
+
+  it("throws AbortError immediately when signal is already aborted", async () => {
+    const ac = new AbortController();
+    ac.abort();
+    const fn = vi.fn().mockResolvedValue("ok");
+    await expect(
+      retryWithBackoff(fn, { retries: 3, signal: ac.signal }),
+    ).rejects.toThrow("Aborted");
+    expect(fn).not.toHaveBeenCalled();
+  });
+
+  it("respects abort signal during retry", async () => {
+    vi.useFakeTimers();
+    const ac = new AbortController();
+    const fn = vi.fn().mockRejectedValue(new Error("fail"));
+
+    const promise = retryWithBackoff(fn, {
+      retries: 5,
+      minTimeout: 100,
+      signal: ac.signal,
+    });
+
+    // Schedule abort after first failure + backoff starts
+    setTimeout(() => ac.abort(), 150);
+    vi.advanceTimersByTime(200);
+    await vi.waitFor(async () => {
+      await expect(promise).rejects.toThrow("Aborted");
+    });
+  });
+});
+
+describe("pagination utilities", () => {
+  it("encodeCursor produces a base64 string", () => {
+    const result = encodeCursor({ created_at: 1000, id: "msg-1" });
+    expect(typeof result).toBe("string");
+    expect(result.length).toBeGreaterThan(0);
+  });
+
+  it("encodeCursor round-trips through decodeCursor", () => {
+    const data = { created_at: 1234567890, id: "abc-def" };
+    const cursor = encodeCursor(data);
+    expect(decodeCursor(cursor)).toEqual(data);
+  });
+
+  it("decodeCursor returns null for undefined / empty", () => {
+    expect(decodeCursor()).toBeNull();
+    expect(decodeCursor("")).toBeNull();
+  });
+
+  it("decodeCursor returns null for malformed input", () => {
+    // Completely invalid base64
+    expect(decodeCursor("!!!not-valid!!!")).toBeNull();
+    // Valid base64 but not JSON
+    const notJson = Buffer.from("not-json").toString("base64");
+    expect(decodeCursor(notJson)).toBeNull();
+    // Valid JSON but wrong shape (missing created_at / id)
+    const wrongShape = Buffer.from(JSON.stringify({ foo: "bar" })).toString("base64");
+    expect(decodeCursor(wrongShape)).toBeNull();
+  });
+
+  it("pageResult truncates and sets nextCursor when rows exceed limit", () => {
+    const rows = [
+      { id: "a", created_at: 100 },
+      { id: "b", created_at: 200 },
+      { id: "c", created_at: 300 },
+    ];
+    const { data, nextCursor } = pageResult(rows, 2);
+    expect(data).toHaveLength(2);
+    expect(data[0].id).toBe("a");
+    expect(nextCursor).toBeTruthy();
+  });
+
+  it("pageResult returns null nextCursor when fewer rows than limit", () => {
+    const rows = [{ id: "a", created_at: 100 }];
+    const { data, nextCursor } = pageResult(rows, 2);
+    expect(data).toHaveLength(1);
+    expect(nextCursor).toBeNull();
+  });
+
+  it("pageResult returns empty data for empty input", () => {
+    const { data, nextCursor } = pageResult([], 10);
+    expect(data).toEqual([]);
+    expect(nextCursor).toBeNull();
+  });
+});
+
+// ═══════════════════════════════════════════════════════════════════════════════
+// 3. Middleware Tests
+// ═══════════════════════════════════════════════════════════════════════════════
+describe("asyncHandler", () => {
+  it("passes thrown errors to next()", async () => {
+    const error = new Error("handler-error");
+    const wrapped = asyncHandler(async () => {
+      throw error;
+    });
+    const next = vi.fn();
+
+    wrapped({} as any, {} as any, next);
+
+    // .catch(next) is a microtask — flush the queue
+    await Promise.resolve();
+    await Promise.resolve();
+    expect(next).toHaveBeenCalledWith(error);
+  });
+
+  it("does not call next when handler resolves successfully", async () => {
+    const wrapped = asyncHandler(async (_req: any, _res: any, _next: any) => {
+      // no-op
+    });
+    const next = vi.fn();
+
+    wrapped({} as any, {} as any, next);
+    await Promise.resolve();
+    await Promise.resolve();
+    expect(next).not.toHaveBeenCalled();
+  });
+});
+
+describe("requireParam", () => {
+  it("returns the value for a non-empty string", () => {
+    expect(requireParam("hello", "param", "name")).toBe("hello");
+  });
+
+  it("throws ValidationError for undefined", () => {
+    expect(() => requireParam(undefined, "query", "q")).toThrow(ValidationError);
+  });
+
+  it("throws ValidationError for empty string", () => {
+    expect(() => requireParam("", "param", "id")).toThrow(ValidationError);
+  });
+
+  it("throws with a descriptive message", () => {
+    expect(() => requireParam(null, "header", "X-Token")).toThrow("Missing header: X-Token");
   });
 });