refactor: large codebase cleanup - consolidate schemas, migrate to Drizzle ORM, extract frontend components, modernize Docker builds

- Consolidate all DB schema definitions into packages/shared as single source of truth
- Migrate backend from raw SQL to Drizzle ORM across all modules
- Extract frontend inline UI into separate component files
- Refactor discord-gateway circuitBreaker into conversationState + moderationState
- Convert messageStore to Proxy singleton pattern
- Add validateBody/validateQuery middleware + Zod schemas for API endpoints
- Modernize Docker builds with multi-stage + pnpm deploy
- Migrate CI/CD from deployment to image-based pipeline
- Remove 60+ unused/dead files (~15K lines)
- Update color scheme from sky-blue to teal-cyan
- Move DB connection management to @bete/shared/database

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

services/discord-gateway/src/modules/voice-recording/mediaSource.ts

@@ -12,7 +12,6 @@ const logger = createChildLogger("media-source");
 export interface MediaInfo {
   title: string;
   duration: number;
-  uploader?: string;
   thumbnail?: string;
 }
 
@@ -284,7 +283,7 @@ export function resolveMediaUrl(
 }
 
 /**
- * Extract metadata (title, duration, uploader, thumbnail) from a media URL
+ * Extract metadata (title, duration, thumbnail) from a media URL
  * without downloading the audio stream.
  *
  * Uses `yt-dlp --dump-json` and parses the JSON output.
@@ -356,7 +355,6 @@ export async function extractMediaInfo(url: string): Promise<MediaInfo> {
         resolve({
           title: String(raw.title ?? url),
           duration: typeof raw.duration === "number" ? raw.duration : 0,
-          uploader: String(raw.uploader ?? raw.channel ?? "") || undefined,
           thumbnail: String(raw.thumbnail ?? "") || undefined,
         });
       } catch (parseErr) {

services/discord-gateway/src/modules/voice-recording/recorder.ts

@@ -213,7 +213,9 @@ export function stopRecording(guildId: string): void {
         status: snapshot.status,
         stopped_at: stoppedAt,
       })
-      .catch(() => {});
+      .catch((err) =>
+        logger.warn({ err }, "Failed to broadcast voice recording stopped"),
+      );
 
     // Auto-enqueue muxer job if there are multiple segments
     const segments = snapshot.segments;

services/discord-gateway/src/modules/voice-recording/recorder/metadata.ts

@@ -0,0 +1,107 @@
+import path from "node:path";
+import { createChildLogger } from "@bete/shared/logger";
+import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
+import { config } from "../../../shared/config/config.js";
+import type {
+  SegmentMetadata,
+  SegmentState,
+  UserMetadata,
+} from "../../message-capture/types.js";
+
+const logger = createChildLogger("voice-metadata");
+
+/** LRU-ish cache: userId -> UserMetadata. Avoids Discord API calls in hotpath. */
+const metadataCache = new Map<string, UserMetadata>();
+const METADATA_CACHE_MAX = 200;
+
+function cacheMetadata(userId: string, metadata: UserMetadata): void {
+  if (metadataCache.size >= METADATA_CACHE_MAX) {
+    // Evict oldest entry via Map iteration (Map preserves insertion order)
+    const firstKey = metadataCache.keys().next().value;
+    if (firstKey) metadataCache.delete(firstKey);
+  }
+  metadataCache.set(userId, metadata);
+}
+
+export async function collectUserMetadata(
+  client: Client,
+  userId: string,
+  channel: VoiceChannel,
+): Promise<UserMetadata> {
+  const cached = metadataCache.get(userId);
+  if (cached) return cached;
+
+  const user =
+    client.users.cache.get(userId) ||
+    (await client.users.fetch(userId).catch(() => {
+      logger.warn({ userId }, "Failed to fetch user");
+      return null;
+    }));
+  const member =
+    channel.guild.members.cache.get(userId) ||
+    (await channel.guild.members.fetch(userId).catch(() => {
+      logger.warn({ userId }, "Failed to fetch guild member");
+      return null;
+    }));
+  const username = user?.username ?? "Unknown User";
+  const roles =
+    member?.roles.cache
+      .filter((role) => role.id !== channel.guild.id)
+      .sort((a, b) => b.position - a.position)
+      .map((role) => ({
+        id: role.id,
+        name: role.name,
+        position: role.position,
+      })) ?? [];
+
+  const result: UserMetadata = {
+    userId,
+    username,
+    tag: user?.tag ?? "Unknown#0000",
+    displayName: member?.displayName ?? username,
+    avatarUrl:
+      user?.displayAvatarURL({
+        format: "png",
+        size: config.AVATAR_SIZE as
+          | 16
+          | 32
+          | 64
+          | 128
+          | 256
+          | 512
+          | 1024
+          | 2048
+          | 4096,
+      }) ?? "https://cdn.discordapp.com/embed/avatars/0.png",
+    bot: user?.bot ?? false,
+    roles,
+    highestRole: roles[0] ?? null,
+    joinedTimestamp: member?.joinedTimestamp ?? null,
+  };
+
+  cacheMetadata(userId, result);
+  return result;
+}
+
+export function createSegmentMetadata(
+  user: UserMetadata,
+  segment: SegmentState,
+  sessionId: string,
+  recordingSessionId: string,
+  sessionStartTime: number,
+  recordingSegmentMs: number,
+): SegmentMetadata {
+  const endTime = segment.endTime ?? Date.now();
+  return {
+    ...user,
+    sessionId,
+    recordingSessionId,
+    sessionStartTime,
+    segmentIndex: segment.index,
+    segmentMs: recordingSegmentMs,
+    startTime: segment.startTime,
+    endTime,
+    durationMs: endTime - segment.startTime,
+    filename: path.basename(segment.filename),
+  };
+}

services/discord-gateway/src/modules/voice-recording/recorder/segment.ts

@@ -1,102 +1,13 @@
-import fs, { promises as fsPromises } from "node:fs";
+import fs from "node:fs";
 import path from "node:path";
 import { createChildLogger } from "@bete/shared/logger";
-import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
 import * as prism from "prism-media";
-import { config } from "../../../shared/config/config.js";
-import type {
-  SegmentMetadata,
-  SegmentState,
-  UserMetadata,
-} from "../../message-capture/types.js";
-import type { RecordingSession } from "./sessionRecording.js";
-import { uploadRecordingSegment } from "./uploader.js";
-
-// ---------------------------------------------------------------------------
-// Logger & metadata cache
-// ---------------------------------------------------------------------------
+import type { SegmentState } from "../../message-capture/types.js";
 
 const logger = createChildLogger("voice-segment");
 
-/** LRU-ish cache: userId -> UserMetadata. Avoids Discord API calls in hotpath. */
-const metadataCache = new Map<string, UserMetadata>();
-const METADATA_CACHE_MAX = 200;
-
 // ---------------------------------------------------------------------------
-// collectUserMetadata (was metadata.ts)
-// ---------------------------------------------------------------------------
-
-export async function collectUserMetadata(
-  client: Client,
-  userId: string,
-  channel: VoiceChannel,
-): Promise<UserMetadata> {
-  const cached = metadataCache.get(userId);
-  if (cached) return cached;
-
-  const user =
-    client.users.cache.get(userId) ||
-    (await client.users.fetch(userId).catch(() => {
-      logger.warn({ userId }, "Failed to fetch user");
-      return null;
-    }));
-  const member =
-    channel.guild.members.cache.get(userId) ||
-    (await channel.guild.members.fetch(userId).catch(() => {
-      logger.warn({ userId }, "Failed to fetch guild member");
-      return null;
-    }));
-  const username = user?.username ?? "Unknown User";
-  const roles =
-    member?.roles.cache
-      .filter((role) => role.id !== channel.guild.id)
-      .sort((a, b) => b.position - a.position)
-      .map((role) => ({
-        id: role.id,
-        name: role.name,
-        position: role.position,
-      })) ?? [];
-
-  const result: UserMetadata = {
-    userId,
-    username,
-    tag: user?.tag ?? "Unknown#0000",
-    displayName: member?.displayName ?? username,
-    avatarUrl:
-      user?.displayAvatarURL({
-        format: "png",
-        size: config.AVATAR_SIZE as
-          | 16
-          | 32
-          | 64
-          | 128
-          | 256
-          | 512
-          | 1024
-          | 2048
-          | 4096,
-      }) ?? "https://cdn.discordapp.com/embed/avatars/0.png",
-    bot: user?.bot ?? false,
-    roles,
-    highestRole: roles[0] ?? null,
-    joinedTimestamp: member?.joinedTimestamp ?? null,
-  };
-
-  cacheMetadata(userId, result);
-  return result;
-}
-
-function cacheMetadata(userId: string, metadata: UserMetadata): void {
-  if (metadataCache.size >= METADATA_CACHE_MAX) {
-    // Evict oldest entry via Map iteration (Map preserves insertion order)
-    const firstKey = metadataCache.keys().next().value;
-    if (firstKey) metadataCache.delete(firstKey);
-  }
-  metadataCache.set(userId, metadata);
-}
-
-// ---------------------------------------------------------------------------
-// Path helpers (was segment.ts)
+// Path helpers
 // ---------------------------------------------------------------------------
 
 export function buildSegmentPaths(
@@ -118,7 +29,7 @@ export function shouldRotateSegment(
 }
 
 // ---------------------------------------------------------------------------
-// SegmentManager (was segment.ts)
+// SegmentManager
 // ---------------------------------------------------------------------------
 
 export class SegmentManager {
@@ -218,128 +129,3 @@ export class SegmentManager {
     return this.currentSegment;
   }
 }
-
-// ---------------------------------------------------------------------------
-// createSegmentMetadata (was metadata.ts)
-// ---------------------------------------------------------------------------
-
-export function createSegmentMetadata(
-  user: UserMetadata,
-  segment: SegmentState,
-  sessionId: string,
-  recordingSessionId: string,
-  sessionStartTime: number,
-  recordingSegmentMs: number,
-): SegmentMetadata {
-  const endTime = segment.endTime ?? Date.now();
-  return {
-    ...user,
-    sessionId,
-    recordingSessionId,
-    sessionStartTime,
-    segmentIndex: segment.index,
-    segmentMs: recordingSegmentMs,
-    startTime: segment.startTime,
-    endTime,
-    durationMs: endTime - segment.startTime,
-    filename: path.basename(segment.filename),
-  };
-}
-
-// ---------------------------------------------------------------------------
-// SegmentFinalizerInput (was segmentFinalizer.ts)
-// ---------------------------------------------------------------------------
-
-export interface SegmentFinalizerInput {
-  currentSegment: SegmentState;
-  userMetadata: UserMetadata;
-  activeSession: RecordingSession | undefined;
-  guildId: string;
-  channelId: string;
-  channelName: string;
-}
-
-// ---------------------------------------------------------------------------
-// finalizeSegment (was segmentFinalizer.ts)
-// ---------------------------------------------------------------------------
-
-/**
- * Handles the completion of an OGG segment:
- * - Logs the saved segment (if VERBOSE)
- * - Registers the segment with the active recording session
- * - Writes the metadata JSON file alongside the OGG file
- * - Triggers async upload of the segment to external storage
- *
- * This function is fire-and-forget for the metadata write and upload;
- * errors are caught and logged without throwing.
- */
-export function finalizeSegment(input: SegmentFinalizerInput): void {
-  const {
-    currentSegment,
-    userMetadata,
-    activeSession,
-    guildId,
-    channelId,
-    channelName,
-  } = input;
-
-  const endTime = currentSegment.endTime ?? Date.now();
-
-  if (config.VERBOSE) {
-    logger.info({ filename: currentSegment.filename }, "Segment saved");
-  }
-
-  // Register segment with the active recording session
-  if (activeSession) {
-    activeSession.registerSegment({
-      user: userMetadata,
-      oggPath: currentSegment.filename,
-      jsonPath: currentSegment.jsonFilename,
-      startTime: currentSegment.startTime,
-      endTime,
-    });
-  }
-
-  // Write metadata JSON (async, fire-and-forget)
-  const metadata = createSegmentMetadata(
-    userMetadata,
-    currentSegment,
-    activeSession?.sessionId ?? `${userMetadata.userId}-0`,
-    activeSession?.sessionId ?? `${guildId}-${channelId}-0`,
-    activeSession?.startTime ?? 0,
-    config.RECORDING_SEGMENT_MS,
-  );
-
-  fsPromises
-    .writeFile(currentSegment.jsonFilename, JSON.stringify(metadata, null, 2))
-    .then(() => {
-      if (config.VERBOSE) {
-        logger.info(
-          { jsonFile: currentSegment.jsonFilename },
-          "Metadata saved",
-        );
-      }
-    })
-    .catch((err: unknown) => {
-      logger.error(
-        { error: err instanceof Error ? err.message : String(err) },
-        "Failed to write segment metadata",
-      );
-    });
-
-  // Trigger async voice segment upload (fire-and-forget)
-  const segmentId = `${userMetadata.userId}-${currentSegment.startTime}`;
-  uploadRecordingSegment({
-    id: segmentId,
-    oggPath: currentSegment.filename,
-    userId: userMetadata.userId,
-    username: userMetadata.username,
-    avatarUrl: userMetadata.avatarUrl,
-    guildId,
-    channelId,
-    channelName,
-  }).catch((err: unknown) => {
-    const msg = err instanceof Error ? err.message : String(err);
-    logger.error({ segmentId, error: msg }, "Upload segment trigger failed");
-  });
-}

services/discord-gateway/src/modules/voice-recording/recorder/segmentFinalizer.ts

@@ -0,0 +1,110 @@
+import { promises as fsPromises } from "node:fs";
+import { createChildLogger } from "@bete/shared/logger";
+import { config } from "../../../shared/config/config.js";
+import type {
+  SegmentState,
+  UserMetadata,
+} from "../../message-capture/types.js";
+import { createSegmentMetadata } from "./metadata.js";
+import type { RecordingSession } from "./sessionRecording.js";
+import { uploadRecordingSegment } from "./uploader.js";
+
+const logger = createChildLogger("voice-segment-finalizer");
+
+// ---------------------------------------------------------------------------
+// SegmentFinalizerInput
+// ---------------------------------------------------------------------------
+
+export interface SegmentFinalizerInput {
+  currentSegment: SegmentState;
+  userMetadata: UserMetadata;
+  activeSession: RecordingSession | undefined;
+  guildId: string;
+  channelId: string;
+  channelName: string;
+}
+
+// ---------------------------------------------------------------------------
+// finalizeSegment
+// ---------------------------------------------------------------------------
+
+/**
+ * Handles the completion of an OGG segment:
+ * - Logs the saved segment (if VERBOSE)
+ * - Registers the segment with the active recording session
+ * - Writes the metadata JSON file alongside the OGG file
+ * - Triggers async upload of the segment to external storage
+ *
+ * This function is fire-and-forget for the metadata write and upload;
+ * errors are caught and logged without throwing.
+ */
+export function finalizeSegment(input: SegmentFinalizerInput): void {
+  const {
+    currentSegment,
+    userMetadata,
+    activeSession,
+    guildId,
+    channelId,
+    channelName,
+  } = input;
+
+  const endTime = currentSegment.endTime ?? Date.now();
+
+  if (config.VERBOSE) {
+    logger.info({ filename: currentSegment.filename }, "Segment saved");
+  }
+
+  // Register segment with the active recording session
+  if (activeSession) {
+    activeSession.registerSegment({
+      user: userMetadata,
+      oggPath: currentSegment.filename,
+      jsonPath: currentSegment.jsonFilename,
+      startTime: currentSegment.startTime,
+      endTime,
+    });
+  }
+
+  // Write metadata JSON (async, fire-and-forget)
+  const metadata = createSegmentMetadata(
+    userMetadata,
+    currentSegment,
+    activeSession?.sessionId ?? `${userMetadata.userId}-0`,
+    activeSession?.sessionId ?? `${guildId}-${channelId}-0`,
+    activeSession?.startTime ?? 0,
+    config.RECORDING_SEGMENT_MS,
+  );
+
+  fsPromises
+    .writeFile(currentSegment.jsonFilename, JSON.stringify(metadata, null, 2))
+    .then(() => {
+      if (config.VERBOSE) {
+        logger.info(
+          { jsonFile: currentSegment.jsonFilename },
+          "Metadata saved",
+        );
+      }
+    })
+    .catch((err: unknown) => {
+      logger.error(
+        { error: err instanceof Error ? err.message : String(err) },
+        "Failed to write segment metadata",
+      );
+    });
+
+  // Trigger async voice segment upload (fire-and-forget)
+  const segmentId = `${userMetadata.userId}-${currentSegment.startTime}`;
+  uploadRecordingSegment({
+    id: segmentId,
+    oggPath: currentSegment.filename,
+    userId: userMetadata.userId,
+    username: userMetadata.username,
+    avatarUrl: userMetadata.avatarUrl,
+    guildId,
+    channelId,
+    channelName,
+  }).catch((err: unknown) => {
+    const msg = err instanceof Error ? err.message : String(err);
+    logger.error({ segmentId, error: msg }, "Upload segment trigger failed");
+  });
+}

services/discord-gateway/src/modules/voice-recording/recorder/speakingHandler.ts

@@ -4,7 +4,8 @@ import { createChildLogger } from "@bete/shared/logger";
 import type { VoiceConnection } from "@discordjs/voice";
 import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
 import type { EventBroadcaster } from "../../event-broadcaster/eventBroadcaster.js";
-import { collectUserMetadata, finalizeSegment } from "./segment.js";
+import { collectUserMetadata } from "./metadata.js";
+import { finalizeSegment } from "./segmentFinalizer.js";
 import type { RecordingSession } from "./sessionRecording.js";
 import { setupUserStream } from "./streamSetup.js";
 

services/discord-gateway/src/modules/voice-recording/recorder/uploader.ts

@@ -8,7 +8,7 @@ import {
   updateVoiceRecordingAsUploaded,
   updateVoiceRecordingTranscription,
 } from "../../../shared/database/voiceRecordingRepo.js";
-import { uploadToTele } from "../teleUpload.js";
+import { uploadToTele } from "../../../shared/uploader.js";
 import { transcribeRecording } from "../voiceTranscriber.js";
 
 const logger = createChildLogger("recording-uploader");

services/discord-gateway/src/modules/voice-recording/teleUpload.ts

@@ -1,80 +0,0 @@
-import { createChildLogger } from "@bete/shared/logger";
-import { retryWithBackoff } from "@bete/shared/utils";
-
-const logger = createChildLogger("tele-upload");
-
-export interface TeleUploadResponse {
-  download_url: string;
-  public_id?: string;
-  file_name?: string;
-  size_bytes?: number;
-}
-
-export interface TeleUploadResult {
-  url: string;
-  publicId?: string;
-  filename?: string;
-  sizeBytes?: number;
-}
-
-export function parseTeleUploadResponse(
-  response: TeleUploadResponse,
-): TeleUploadResult {
-  if (!response.download_url) {
-    throw new Error("Missing download_url in response");
-  }
-
-  return {
-    url: response.download_url,
-    publicId: response.public_id,
-    filename: response.file_name,
-    sizeBytes: response.size_bytes,
-  };
-}
-
-export async function uploadToTele(input: {
-  buffer: Buffer;
-  filename: string;
-  contentType: string;
-  uploadUrl: string;
-  timeoutMs?: number;
-  retries: number;
-}): Promise<TeleUploadResult> {
-  const { buffer, filename, contentType, uploadUrl, timeoutMs, retries } =
-    input;
-
-  logger.debug({ filename, uploadUrl }, "Starting tele upload");
-
-  const response = await retryWithBackoff(
-    async () => {
-      const fileBlob = new Blob([new Uint8Array(buffer)], {
-        type: contentType,
-      });
-      const formData = new FormData();
-      formData.append("file", fileBlob, filename);
-      formData.append("fileName", filename);
-
-      const res = await fetch(uploadUrl, {
-        method: "POST",
-        headers: {
-          accept: "application/json",
-        },
-        body: formData,
-        ...(timeoutMs ? { signal: AbortSignal.timeout(timeoutMs) } : {}),
-      });
-
-      if (!res.ok) {
-        throw new Error(`Upload failed: Status ${res.status}`);
-      }
-
-      return (await res.json()) as TeleUploadResponse;
-    },
-    {
-      retries,
-      minTimeout: 1000,
-      maxTimeout: 10000,
-    },
-  );
-
-  return parseTeleUploadResponse(response);
-}

services/discord-gateway/src/modules/voice-recording/transmitter.ts

@@ -221,7 +221,11 @@ export class VoiceTransmitter {
 
       if (this.redisSub) {
         await this.redisSub.unsubscribe(this.TRANSMIT_CHANNEL);
-        this.redisSub.quit().catch(() => {});
+        this.redisSub
+          .quit()
+          .catch((err) =>
+            logger.warn({ err }, "Failed to quit Redis subscriber"),
+          );
         this.redisSub = null;
       }