From 07032ab521ac4abc99b607779b59fdebd01ccddd Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Tue, 9 Jun 2026 19:46:08 +0700 Subject: [PATCH] refactor: atomic, DRY, and logging improvements across codebase - Split llmModerationClient.ts (2170 lines) into 5 focused sub-modules - Split aiAnalyzer.ts (1282 lines) into 4 modular pipelines - Split messages.db.ts (826 lines) into 5 domain-specific modules - Moved shared schema to @bete/shared, eliminated backend duplication - Added createChildLogger to all voice-recording and AI moderation modules - Extracted tryCommandThenFallback, normalizeMediaState, DEFAULT_VOICE_STATUS - Created shared pagination.ts utility, eliminated 5+ cursor-pagination duplications - Created shared messageMapper.ts for row mapping - Standardized backend error handling with asyncHandler - Added frontend createLogger utility and useAsyncAction hook - Added structured logging to frontend hooks, socket, and API client Co-Authored-By: Claude Opus 4.8 --- packages/shared/package.json | 2 + packages/shared/src/database/schema.ts | 141 +++ packages/shared/src/index.ts | 1 + packages/shared/src/utils/index.ts | 2 + packages/shared/src/utils/pagination.ts | 65 + pnpm-lock.yaml | 3 + .../modules/analysis/analysis.repository.ts | 63 +- .../src/modules/health/health.controller.ts | 14 +- .../mascot-chat/mascot-chat.controller.ts | 43 +- .../modules/mascot-chat/mascot-chat.routes.ts | 7 +- .../src/modules/media/media.service.ts | 159 ++- .../modules/messages/messages.repository.ts | 202 +-- .../src/modules/voice/guilds.routes.ts | 33 +- .../src/modules/voice/voice.controller.ts | 93 +- .../src/modules/voice/voice.service.ts | 129 +- services/backend/src/shared/commandHelper.ts | 26 + .../backend/src/shared/utils/messageMapper.ts | 57 + services/backend/src/ws/broadcast.ts | 108 +- .../modules/ai-moderation/aiAnalysisWorker.ts | 25 +- .../src/modules/ai-moderation/aiAnalyzer.ts | 1120 +---------------- .../modules/ai-moderation/batchProcessor.ts | 399 ++++++ .../modules/ai-moderation/batchScheduler.ts | 140 +++ .../ai-moderation/channelCultureStore.ts | 13 + .../modules/ai-moderation/circuitBreaker.ts | 235 ++++ .../ai-moderation/concurrencyLimiter.ts | 41 +- .../ai-moderation/conversationContext.ts | 19 +- .../modules/ai-moderation/imageMimeSniffer.ts | 76 ++ .../individualFallbackProcessor.ts | 346 +++++ .../modules/ai-moderation/jsonExtractor.ts | 81 ++ .../ai-moderation/llmModerationClient.ts | 427 +------ .../ai-moderation/moderationResponseParser.ts | 178 +++ .../ai-moderation/moderationSchemas.ts | 41 + .../modules/ai-moderation/severityDeriver.ts | 64 + .../modules/ai-moderation/stickerPrompt.ts | 12 + .../ai-moderation/userReputationStore.ts | 33 + .../modules/message-capture/messageStore.ts | 4 +- .../message-capture/messages.analysis.ts | 413 ++++++ .../message-capture/messages.cleanup.ts | 86 ++ .../modules/message-capture/messages.crud.ts | 204 +++ .../modules/message-capture/messages.db.ts | 819 ++---------- .../message-capture/messages.pagination.ts | 100 ++ .../message-capture/messages.search.ts | 76 ++ .../message-capture/moderation-actions.db.ts | 21 +- .../src/modules/message-capture/pagination.ts | 21 +- .../src/modules/message-capture/reviews.db.ts | 21 +- .../modules/voice-recording/ffmpegProcess.ts | 8 + .../modules/voice-recording/packetFilter.ts | 8 + .../voice-recording/recorder/audioStream.ts | 15 +- .../voice-recording/recorder/metadata.ts | 16 +- .../voice-recording/recorder/segment.ts | 36 + .../recorder/sessionRecording.ts | 81 +- .../src/modules/voice-recording/teleUpload.ts | 5 + .../src/shared/database/schema.ts | 134 +- .../features/live/hooks/useMediaControl.ts | 102 +- .../features/live/hooks/useVoiceControl.ts | 66 +- .../features/messages/hooks/useMessages.ts | 24 +- services/frontend/src/shared/api/client.ts | 10 +- .../src/shared/hooks/useAsyncAction.ts | 37 + .../src/shared/hooks/useAudioPlayback.ts | 99 +- services/frontend/src/shared/lib/logger.ts | 22 + services/frontend/src/shared/ws/socket.ts | 25 +- 61 files changed, 3808 insertions(+), 3043 deletions(-) create mode 100644 packages/shared/src/database/schema.ts create mode 100644 packages/shared/src/utils/pagination.ts create mode 100644 services/backend/src/shared/commandHelper.ts create mode 100644 services/backend/src/shared/utils/messageMapper.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/batchProcessor.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/batchScheduler.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/circuitBreaker.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/imageMimeSniffer.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/individualFallbackProcessor.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/jsonExtractor.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/moderationResponseParser.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/moderationSchemas.ts create mode 100644 services/discord-gateway/src/modules/ai-moderation/severityDeriver.ts create mode 100644 services/discord-gateway/src/modules/message-capture/messages.analysis.ts create mode 100644 services/discord-gateway/src/modules/message-capture/messages.cleanup.ts create mode 100644 services/discord-gateway/src/modules/message-capture/messages.crud.ts create mode 100644 services/discord-gateway/src/modules/message-capture/messages.pagination.ts create mode 100644 services/discord-gateway/src/modules/message-capture/messages.search.ts create mode 100644 services/frontend/src/shared/hooks/useAsyncAction.ts create mode 100644 services/frontend/src/shared/lib/logger.ts diff --git a/packages/shared/package.json b/packages/shared/package.json index a992161..9e51b81 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -8,6 +8,7 @@ "exports": { ".": "./dist/index.js", "./config": "./dist/config/index.js", + "./database/schema": "./dist/database/schema.js", "./errors": "./dist/errors/index.js", "./logger": "./dist/logger/index.js", "./utils": "./dist/utils/index.js" @@ -17,6 +18,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "drizzle-orm": "^0.45.2", "pino": "^9.0.0", "zod": "^4.4.3" }, diff --git a/packages/shared/src/database/schema.ts b/packages/shared/src/database/schema.ts new file mode 100644 index 0000000..e1b032e --- /dev/null +++ b/packages/shared/src/database/schema.ts @@ -0,0 +1,141 @@ +import { + bigint as pgBigint, + foreignKey as pgForeignKey, + index as pgIndex, + integer as pgInteger, + real as pgReal, + pgTable, + text as pgText, +} from "drizzle-orm/pg-core"; + +/** + * Messages Table (PostgreSQL) + * Stores text messages with AI moderation analysis + */ +export const pgMessagesTable = pgTable( + "messages", + { + id: pgText("id").primaryKey(), + guild_id: pgText("guild_id").notNull(), + channel_id: pgText("channel_id").notNull(), + thread_id: pgText("thread_id"), + user_id: pgText("user_id").notNull(), + username: pgText("username").notNull(), + avatar_url: pgText("avatar_url"), + content: pgText("content").notNull(), + edited_content: pgText("edited_content"), + created_at: pgBigint("created_at", { mode: "number" }).notNull(), + edited_at: pgBigint("edited_at", { mode: "number" }), + deleted_at: pgBigint("deleted_at", { mode: "number" }), + type: pgText("type", { enum: ["text", "edited", "deleted"] }) + .notNull() + .default("text"), + metadata: pgText("metadata"), + ai_status: pgText("ai_status", { + enum: ["pending", "processing", "clean", "warn", "flagged", "error"], + }) + .notNull() + .default("pending"), + ai_moderation_flags: pgText("ai_moderation_flags"), + ai_moderation_score: pgReal("ai_moderation_score"), + ai_analysis: pgText("ai_analysis"), + ai_categories: pgText("ai_categories"), + ai_severity: pgText("ai_severity", { + enum: ["none", "low", "medium", "high", "critical"], + }), + ai_confidence: pgReal("ai_confidence"), + ai_recommended_action: pgText("ai_recommended_action", { + enum: ["none", "monitor", "warn", "review", "delete", "escalate"], + }), + ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }), + ai_error: pgText("ai_error"), + }, + (table) => ({ + channelIdx: pgIndex("idx_messages_channel").on(table.channel_id), + userIdx: pgIndex("idx_messages_user").on(table.user_id), + createdIdx: pgIndex("idx_messages_created").on(table.created_at), + threadIdx: pgIndex("idx_messages_thread").on(table.thread_id), + channelCreatedIdx: pgIndex("idx_messages_channel_created").on( + table.channel_id, + table.created_at, + table.id, + ), + threadCreatedIdx: pgIndex("idx_messages_thread_created").on( + table.thread_id, + table.created_at, + table.id, + ), + aiStatusCreatedIdx: pgIndex("idx_messages_ai_status_created").on( + table.ai_status, + table.created_at, + table.id, + ), + guildAiStatusCreatedIdx: pgIndex("idx_messages_guild_ai_status_created").on( + table.guild_id, + table.ai_status, + table.created_at, + table.id, + ), + guildCreatedDeletedIdx: pgIndex("idx_messages_guild_created_deleted").on( + table.guild_id, + table.created_at, + table.deleted_at, + table.id, + ), + channelAiStatusCreatedIdx: pgIndex( + "idx_messages_channel_ai_status_created", + ).on(table.channel_id, table.ai_status, table.created_at, table.id), + threadAiStatusCreatedIdx: pgIndex( + "idx_messages_thread_ai_status_created", + ).on(table.thread_id, table.ai_status, table.created_at, table.id), + }), +); + +/** + * Attachments Table (PostgreSQL) + * Stores attachment metadata with upload status tracking + */ +export const pgAttachmentsTable = pgTable( + "attachments", + { + id: pgText("id").primaryKey(), + message_id: pgText("message_id").notNull(), + guild_id: pgText("guild_id").notNull(), + channel_id: pgText("channel_id").notNull(), + thread_id: pgText("thread_id"), + user_id: pgText("user_id").notNull(), + filename: pgText("filename").notNull(), + size: pgInteger("size").notNull(), + type: pgText("type").notNull(), + discord_url: pgText("discord_url").notNull(), + uploaded_url: pgText("uploaded_url"), + upload_status: pgText("upload_status", { + enum: ["pending", "uploaded", "failed"], + }) + .notNull() + .default("pending"), + upload_error: pgText("upload_error"), + created_at: pgBigint("created_at", { mode: "number" }).notNull(), + uploaded_at: pgBigint("uploaded_at", { mode: "number" }), + }, + (table) => ({ + channelIdx: pgIndex("idx_attachments_channel").on(table.channel_id), + messageIdx: pgIndex("idx_attachments_message").on(table.message_id), + statusIdx: pgIndex("idx_attachments_status").on(table.upload_status), + channelCreatedIdx: pgIndex("idx_attachments_channel_created").on( + table.channel_id, + table.created_at, + table.id, + ), + threadCreatedIdx: pgIndex("idx_attachments_thread_created").on( + table.thread_id, + table.created_at, + table.id, + ), + messageFk: pgForeignKey({ + columns: [table.message_id], + foreignColumns: [pgMessagesTable.id], + name: "fk_attachments_message_id", + }).onDelete("cascade"), + }), +); diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 80db052..64a9fa9 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1,4 +1,5 @@ export * from "./config/index.js"; +export * from "./database/schema.js"; export * from "./errors/index.js"; export * from "./logger/index.js"; export * from "./moderation-types.js"; diff --git a/packages/shared/src/utils/index.ts b/packages/shared/src/utils/index.ts index 505255b..1204ede 100644 --- a/packages/shared/src/utils/index.ts +++ b/packages/shared/src/utils/index.ts @@ -4,6 +4,8 @@ export function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +export * from "./pagination.js"; + // --------------------------------------------------------------------------- // Retry with exponential backoff // --------------------------------------------------------------------------- diff --git a/packages/shared/src/utils/pagination.ts b/packages/shared/src/utils/pagination.ts new file mode 100644 index 0000000..d0686d9 --- /dev/null +++ b/packages/shared/src/utils/pagination.ts @@ -0,0 +1,65 @@ +// Shared cursor-based pagination utilities + +export interface CursorData { + created_at: number; + id: string; +} + +/** + * Encode a cursor to a base64 string. + */ +export function encodeCursor(data: CursorData): string { + return Buffer.from(JSON.stringify(data)).toString("base64"); +} + +/** + * Decode a cursor from a base64 string. Returns null on invalid input. + */ +export function decodeCursor(cursor?: string): CursorData | null { + if (!cursor) return null; + try { + const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8")); + if (typeof data.created_at === "number" && typeof data.id === "string") { + return data; + } + return null; + } catch { + return null; + } +} + +/** + * Build a `PageResult` from a slice of rows (limit + 1) using cursor-based pagination. + */ +export function pageResult( + rows: unknown[], + limit: number, +): { data: T[]; nextCursor: string | null } { + const hasMore = rows.length > limit; + const data = rows.slice(0, limit) as T[]; + const lastItem = data[data.length - 1]; + const nextCursor = + hasMore && lastItem + ? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id }) + : null; + + return { data, nextCursor }; +} + +/** + * Build a Drizzle cursor condition expression. + * Used in WHERE clauses: `(created_at < cursor.created_at OR (created_at = cursor.created_at AND id < cursor.id))` + * + * Returns the SQL expression or undefined when cursor is absent. + */ +import { type SQL, sql } from "drizzle-orm"; + +export function buildCursorCondition( + created_at_col: SQL | unknown, + id_col: SQL | unknown, + cursor?: string, +): SQL | undefined { + const data = decodeCursor(cursor); + if (!data) return undefined; + return sql`(${created_at_col} < ${data.created_at} or (${created_at_col} = ${data.created_at} and ${id_col} < ${data.id}))`; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f48dad6..2fc699f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -33,6 +33,9 @@ importers: packages/shared: dependencies: + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@opentelemetry/api@1.9.1)(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@11.10.0)(pg@8.21.0) pino: specifier: ^9.0.0 version: 9.14.0 diff --git a/services/backend/src/modules/analysis/analysis.repository.ts b/services/backend/src/modules/analysis/analysis.repository.ts index 86e0d41..19fddf8 100644 --- a/services/backend/src/modules/analysis/analysis.repository.ts +++ b/services/backend/src/modules/analysis/analysis.repository.ts @@ -1,5 +1,9 @@ import { createChildLogger } from "@bete/shared/logger"; import { getPool } from "../../shared/database/index.js"; +import { + type MappedMessage, + mapMessageRow, +} from "../../shared/utils/messageMapper.js"; const logger = createChildLogger("analysis.repository"); @@ -10,61 +14,8 @@ export interface AnalysisSearchQuery { limit?: number; } -export interface AnalysisSearchResult { - id: string; - guild_id: string; - channel_id: string; - thread_id: string | null; - user_id: string; - username: string; - avatar_url: string | null; - content: string; - edited_content: string | null; - created_at: number; - edited_at: number | null; - deleted_at: number | null; - type: string; - metadata: string | null; - ai_status: string | null; - ai_moderation_flags: string | null; - ai_moderation_score: number | null; - ai_analysis: string | null; - ai_categories: string | null; - ai_severity: string | null; - ai_confidence: number | null; - ai_recommended_action: string | null; - ai_analyzed_at: number | null; - ai_error: string | null; -} - -function mapSearchResult(row: Record): AnalysisSearchResult { - return { - id: String(row.id ?? ""), - guild_id: String(row.guild_id ?? ""), - channel_id: String(row.channel_id ?? ""), - thread_id: (row.thread_id as string | null) ?? null, - user_id: String(row.user_id ?? ""), - username: String(row.username ?? ""), - avatar_url: (row.avatar_url as string | null) ?? null, - content: String(row.content ?? ""), - edited_content: (row.edited_content as string | null) ?? null, - created_at: Number(row.created_at ?? 0), - edited_at: (row.edited_at as number | null) ?? null, - deleted_at: (row.deleted_at as number | null) ?? null, - type: String(row.type ?? "text"), - metadata: (row.metadata as string | null) ?? null, - ai_status: (row.ai_status as string | null) ?? null, - ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null, - ai_moderation_score: (row.ai_moderation_score as number | null) ?? null, - ai_analysis: (row.ai_analysis as string | null) ?? null, - ai_categories: (row.ai_categories as string | null) ?? null, - ai_severity: (row.ai_severity as string | null) ?? null, - ai_confidence: (row.ai_confidence as number | null) ?? null, - ai_recommended_action: (row.ai_recommended_action as string | null) ?? null, - ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null, - ai_error: (row.ai_error as string | null) ?? null, - }; -} +// AnalysisSearchResult is identical to MappedMessage — reuse the shared mapper +export type AnalysisSearchResult = MappedMessage; export class AnalysisRepository { async search(query: AnalysisSearchQuery): Promise { @@ -107,7 +58,7 @@ export class AnalysisRepository { [...params, limit], ); - return rows.map((r) => mapSearchResult(r as Record)); + return rows.map((r) => mapMessageRow(r as Record)); } } diff --git a/services/backend/src/modules/health/health.controller.ts b/services/backend/src/modules/health/health.controller.ts index 56aad0c..0f6a638 100644 --- a/services/backend/src/modules/health/health.controller.ts +++ b/services/backend/src/modules/health/health.controller.ts @@ -1,16 +1,12 @@ -import type { NextFunction, Request, Response } from "express"; +import type { Request, Response } from "express"; import { asyncHandler } from "../../shared/middlewares/index.js"; import { healthService } from "./health.service.js"; -export function handleHealthCheck( - req: Request, - res: Response, - next: NextFunction, -) { - return asyncHandler(async (req: Request, res: Response) => { +export const handleHealthCheck = asyncHandler( + async (req: Request, res: Response) => { const verbose = req.query.verbose === "true"; const result = await healthService.getHealth(verbose); const status = result.status === "healthy" ? 200 : 503; res.status(status).json(result); - })(req, res, next); -} + }, +); diff --git a/services/backend/src/modules/mascot-chat/mascot-chat.controller.ts b/services/backend/src/modules/mascot-chat/mascot-chat.controller.ts index 1e94189..737ecfd 100644 --- a/services/backend/src/modules/mascot-chat/mascot-chat.controller.ts +++ b/services/backend/src/modules/mascot-chat/mascot-chat.controller.ts @@ -1,5 +1,6 @@ import { createChildLogger } from "@bete/shared/logger"; import type { Request, Response } from "express"; +import { asyncHandler } from "../../shared/middlewares/index.js"; import { mascotChatService } from "./mascot-chat.service.js"; const logger = createChildLogger("mascot-chat.controller"); @@ -8,8 +9,8 @@ interface AuthenticatedRequest extends Request { userId?: string; } -export async function handleMascotChat(req: Request, res: Response) { - try { +export const handleMascotChat = asyncHandler( + async (req: Request, res: Response) => { const { message, context } = req.body; if (!message || typeof message !== "string") { @@ -49,17 +50,11 @@ export async function handleMascotChat(req: Request, res: Response) { response, timestamp: new Date().toISOString(), }); - } catch (error) { - logger.error({ error }, "Error processing mascot chat"); - res.status(500).json({ - error: "INTERNAL_SERVER_ERROR", - message: "Failed to process mascot chat", - }); - } -} + }, +); -export async function getMascotChatHistory(req: Request, res: Response) { - try { +export const getMascotChatHistory = asyncHandler( + async (req: Request, res: Response) => { const userId = (req as AuthenticatedRequest).userId || "anonymous"; const limit = Math.min(parseInt(req.query.limit as string) || 50, 100); @@ -69,17 +64,11 @@ export async function getMascotChatHistory(req: Request, res: Response) { history, total: history.length, }); - } catch (error) { - logger.error({ error }, "Error fetching chat history"); - res.status(500).json({ - error: "INTERNAL_SERVER_ERROR", - message: "Failed to fetch chat history", - }); - } -} + }, +); -export async function clearMascotChatHistory(req: Request, res: Response) { - try { +export const clearMascotChatHistory = asyncHandler( + async (req: Request, res: Response) => { const userId = (req as AuthenticatedRequest).userId || "anonymous"; await mascotChatService.clearChatHistory(userId); @@ -87,11 +76,5 @@ export async function clearMascotChatHistory(req: Request, res: Response) { res.status(200).json({ message: "Chat history cleared successfully", }); - } catch (error) { - logger.error({ error }, "Error clearing chat history"); - res.status(500).json({ - error: "INTERNAL_SERVER_ERROR", - message: "Failed to clear chat history", - }); - } -} + }, +); diff --git a/services/backend/src/modules/mascot-chat/mascot-chat.routes.ts b/services/backend/src/modules/mascot-chat/mascot-chat.routes.ts index b6a50f6..5de813a 100644 --- a/services/backend/src/modules/mascot-chat/mascot-chat.routes.ts +++ b/services/backend/src/modules/mascot-chat/mascot-chat.routes.ts @@ -1,5 +1,4 @@ import express, { type Router } from "express"; -import { asyncHandler } from "../../shared/middlewares/index.js"; import { clearMascotChatHistory, getMascotChatHistory, @@ -9,9 +8,9 @@ import { export function createMascotChatRouter(): Router { const router = express.Router(); - router.post("/mascot/chat", asyncHandler(handleMascotChat)); - router.get("/mascot/chat/history", asyncHandler(getMascotChatHistory)); - router.delete("/mascot/chat/history", asyncHandler(clearMascotChatHistory)); + router.post("/mascot/chat", handleMascotChat); + router.get("/mascot/chat/history", getMascotChatHistory); + router.delete("/mascot/chat/history", clearMascotChatHistory); return router; } diff --git a/services/backend/src/modules/media/media.service.ts b/services/backend/src/modules/media/media.service.ts index 362d1fb..a026433 100644 --- a/services/backend/src/modules/media/media.service.ts +++ b/services/backend/src/modules/media/media.service.ts @@ -5,7 +5,10 @@ import { COMMAND_MEDIA_VOLUME, MEDIA_STATUS_KEY, } from "@bete/shared"; -import { createChildLogger } from "@bete/shared/logger"; +import { + createChildLogger, + tryCommandThenFallback, +} from "../../shared/commandHelper.js"; import { publishCommand, readRedisStatus } from "../../shared/redis/index.js"; const logger = createChildLogger("media.service"); @@ -43,6 +46,35 @@ const DEFAULT_STATE: MediaState = { queue: [], }; +// --------------------------------------------------------------------------- +// Normalisation — handle both boolean (new) and string (legacy) playing values +// --------------------------------------------------------------------------- + +function normalizeMediaState(raw: Record): MediaState { + const rawPlaying = raw.playing; + const playing = + rawPlaying === true || + rawPlaying === "playing" || + rawPlaying === "buffering"; + return { + playing, + musicVolume: Number(raw.musicVolume ?? 1.0), + current: (raw.current as MediaItem | null) ?? null, + queue: (raw.queue as MediaItem[]) ?? [], + }; +} + +type MediaReplyData = Record | MediaState; + +function fromReply(data: MediaReplyData): MediaState { + return normalizeMediaState(data as Record); +} + +async function readStatusFallback(): Promise { + const cached = await readRedisStatus(MEDIA_STATUS_KEY); + return cached ? normalizeMediaState(cached) : DEFAULT_STATE; +} + // --------------------------------------------------------------------------- // Service methods // --------------------------------------------------------------------------- @@ -52,24 +84,7 @@ const DEFAULT_STATE: MediaState = { */ export async function getStatus(): Promise { logger.debug("getStatus called"); - const cached = await readRedisStatus(MEDIA_STATUS_KEY); - - if (cached) { - const rawPlaying = cached.playing; - // Handle both boolean (new) and string (legacy from String(discordPlayer.getStatus())) - const playing = - rawPlaying === true || - rawPlaying === "playing" || - rawPlaying === "buffering"; - return { - playing, - musicVolume: Number(cached.musicVolume ?? 1.0), - current: (cached.current as MediaItem | null) ?? null, - queue: (cached.queue as MediaItem[]) ?? [], - }; - } - - return DEFAULT_STATE; + return readStatusFallback(); } /** @@ -80,26 +95,16 @@ export async function queue( mode: "music" | "screen" = "music", ): Promise { logger.info({ source, mode }, "queue called"); - const reply = await publishCommand( - COMMAND_MEDIA_QUEUE, - { source, mode }, - DEFAULT_COMMAND_TIMEOUT_MS, + return tryCommandThenFallback( + () => + publishCommand( + COMMAND_MEDIA_QUEUE, + { source, mode }, + DEFAULT_COMMAND_TIMEOUT_MS, + ), + () => readStatusFallback(), + "queue", ); - - if (reply?.success && reply.data) { - return { - playing: reply.data.playing, - musicVolume: reply.data.musicVolume, - current: reply.data.current ?? null, - queue: reply.data.queue ?? [], - }; - } - - logger.warn( - { source, mode }, - "discord-gateway unreachable, returning current media status", - ); - return getStatus(); } /** @@ -107,23 +112,16 @@ export async function queue( */ export async function skip(): Promise { logger.info("skip called"); - const reply = await publishCommand( - COMMAND_MEDIA_SKIP, - {}, - DEFAULT_COMMAND_TIMEOUT_MS, + return tryCommandThenFallback( + () => + publishCommand( + COMMAND_MEDIA_SKIP, + {}, + DEFAULT_COMMAND_TIMEOUT_MS, + ), + () => readStatusFallback(), + "skip", ); - - if (reply?.success && reply.data) { - return { - playing: reply.data.playing, - musicVolume: reply.data.musicVolume, - current: reply.data.current ?? null, - queue: reply.data.queue ?? [], - }; - } - - logger.warn("discord-gateway unreachable, returning current media status"); - return getStatus(); } /** @@ -131,23 +129,16 @@ export async function skip(): Promise { */ export async function stop(): Promise { logger.info("stop called"); - const reply = await publishCommand( - COMMAND_MEDIA_STOP, - {}, - DEFAULT_COMMAND_TIMEOUT_MS, + return tryCommandThenFallback( + () => + publishCommand( + COMMAND_MEDIA_STOP, + {}, + DEFAULT_COMMAND_TIMEOUT_MS, + ), + () => readStatusFallback(), + "stop", ); - - if (reply?.success && reply.data) { - return { - playing: reply.data.playing, - musicVolume: reply.data.musicVolume, - current: reply.data.current ?? null, - queue: reply.data.queue ?? [], - }; - } - - logger.warn("discord-gateway unreachable, returning current media status"); - return getStatus(); } /** @@ -155,24 +146,14 @@ export async function stop(): Promise { */ export async function setVolume(volume: number): Promise { logger.info({ volume }, "setVolume called"); - const reply = await publishCommand( - COMMAND_MEDIA_VOLUME, - { volume }, - DEFAULT_COMMAND_TIMEOUT_MS, + return tryCommandThenFallback( + () => + publishCommand( + COMMAND_MEDIA_VOLUME, + { volume }, + DEFAULT_COMMAND_TIMEOUT_MS, + ), + () => readStatusFallback(), + "setVolume", ); - - if (reply?.success && reply.data) { - return { - playing: reply.data.playing, - musicVolume: reply.data.musicVolume, - current: reply.data.current ?? null, - queue: reply.data.queue ?? [], - }; - } - - logger.warn( - { volume }, - "discord-gateway unreachable, returning current media status", - ); - return getStatus(); } diff --git a/services/backend/src/modules/messages/messages.repository.ts b/services/backend/src/modules/messages/messages.repository.ts index 774913a..d99f780 100644 --- a/services/backend/src/modules/messages/messages.repository.ts +++ b/services/backend/src/modules/messages/messages.repository.ts @@ -1,14 +1,9 @@ import type { PageResult } from "@bete/shared"; +import { pgAttachmentsTable, pgMessagesTable } from "@bete/shared"; import { createChildLogger } from "@bete/shared/logger"; import { and, desc, eq, inArray, lt, ne, type SQL } from "drizzle-orm"; -import { - bigint as pgBigint, - integer as pgInteger, - real as pgReal, - pgTable, - text as pgText, -} from "drizzle-orm/pg-core"; import { getDatabase } from "../../shared/database/index.js"; +import { mapMessageRow } from "../../shared/utils/messageMapper.js"; import type { MessageCreate, MessageQuery, @@ -17,71 +12,6 @@ import type { const logger = createChildLogger("messages.repository"); -/** - * Local table definitions mirroring services/discord-gateway/src/shared/database/schema.ts. - * These are query-building references only — schema source of truth remains in discord-gateway. - */ -const messages = pgTable("messages", { - id: pgText("id").primaryKey(), - guild_id: pgText("guild_id").notNull(), - channel_id: pgText("channel_id").notNull(), - thread_id: pgText("thread_id"), - user_id: pgText("user_id").notNull(), - username: pgText("username").notNull(), - avatar_url: pgText("avatar_url"), - content: pgText("content").notNull(), - edited_content: pgText("edited_content"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - edited_at: pgBigint("edited_at", { mode: "number" }), - deleted_at: pgBigint("deleted_at", { mode: "number" }), - type: pgText("type", { - enum: ["text", "edited", "deleted"], - }) - .notNull() - .default("text"), - metadata: pgText("metadata"), - ai_status: pgText("ai_status", { - enum: ["pending", "processing", "clean", "warn", "flagged", "error"], - }) - .notNull() - .default("pending"), - ai_moderation_flags: pgText("ai_moderation_flags"), - ai_moderation_score: pgReal("ai_moderation_score"), - ai_analysis: pgText("ai_analysis"), - ai_categories: pgText("ai_categories"), - ai_severity: pgText("ai_severity", { - enum: ["none", "low", "medium", "high", "critical"], - }), - ai_confidence: pgReal("ai_confidence"), - ai_recommended_action: pgText("ai_recommended_action", { - enum: ["none", "monitor", "warn", "review", "delete", "escalate"], - }), - ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }), - ai_error: pgText("ai_error"), -}); - -const attachments = pgTable("attachments", { - id: pgText("id").primaryKey(), - message_id: pgText("message_id").notNull(), - guild_id: pgText("guild_id").notNull(), - channel_id: pgText("channel_id").notNull(), - thread_id: pgText("thread_id"), - user_id: pgText("user_id").notNull(), - filename: pgText("filename").notNull(), - size: pgInteger("size").notNull(), - type: pgText("type").notNull(), - discord_url: pgText("discord_url").notNull(), - uploaded_url: pgText("uploaded_url"), - upload_status: pgText("upload_status", { - enum: ["pending", "uploaded", "failed"], - }) - .notNull() - .default("pending"), - upload_error: pgText("upload_error"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - uploaded_at: pgBigint("uploaded_at", { mode: "number" }), -}); - export interface AttachmentResult { id: string; message_id: string; @@ -100,35 +30,6 @@ export interface AttachmentResult { uploaded_at: number | null; } -function mapMessageRow(row: Record) { - return { - id: String(row.id ?? ""), - guild_id: String(row.guild_id ?? ""), - channel_id: String(row.channel_id ?? ""), - thread_id: (row.thread_id as string | null) ?? null, - user_id: String(row.user_id ?? ""), - username: String(row.username ?? ""), - avatar_url: (row.avatar_url as string | null) ?? null, - content: String(row.content ?? ""), - edited_content: (row.edited_content as string | null) ?? null, - created_at: Number(row.created_at ?? 0), - edited_at: (row.edited_at as number | null) ?? null, - deleted_at: (row.deleted_at as number | null) ?? null, - type: String(row.type ?? "text"), - metadata: (row.metadata as string | null) ?? null, - ai_status: (row.ai_status as string | null) ?? null, - ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null, - ai_moderation_score: (row.ai_moderation_score as number | null) ?? null, - ai_analysis: (row.ai_analysis as string | null) ?? null, - ai_categories: (row.ai_categories as string | null) ?? null, - ai_severity: (row.ai_severity as string | null) ?? null, - ai_confidence: (row.ai_confidence as number | null) ?? null, - ai_recommended_action: (row.ai_recommended_action as string | null) ?? null, - ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null, - ai_error: (row.ai_error as string | null) ?? null, - }; -} - export class MessagesRepository { async findMany( query: MessageQuery, @@ -138,27 +39,27 @@ export class MessagesRepository { const conditions: SQL[] = []; if (query.guildId) { - conditions.push(eq(messages.guild_id, query.guildId)); + conditions.push(eq(pgMessagesTable.guild_id, query.guildId)); } if (query.channelId) { - conditions.push(eq(messages.channel_id, query.channelId)); + conditions.push(eq(pgMessagesTable.channel_id, query.channelId)); } if (query.userId) { - conditions.push(eq(messages.user_id, query.userId)); + conditions.push(eq(pgMessagesTable.user_id, query.userId)); } if (query.status) { - conditions.push(eq(messages.ai_status, query.status)); + conditions.push(eq(pgMessagesTable.ai_status, query.status)); } if (query.cursor) { - conditions.push(lt(messages.created_at, Number(query.cursor))); + conditions.push(lt(pgMessagesTable.created_at, Number(query.cursor))); } const where = conditions.length > 0 ? and(...conditions) : undefined; const rows = await db .select() - .from(messages) + .from(pgMessagesTable) .where(where) - .orderBy(desc(messages.created_at)) + .orderBy(desc(pgMessagesTable.created_at)) .limit(limit + 1); const data = rows @@ -175,8 +76,8 @@ export class MessagesRepository { const db = getDatabase(); const [row] = await db .select() - .from(messages) - .where(eq(messages.id, id)) + .from(pgMessagesTable) + .where(eq(pgMessagesTable.id, id)) .limit(1); if (!row) return null; @@ -189,17 +90,17 @@ export class MessagesRepository { ): Promise>> { const db = getDatabase(); const limit = query.limit ?? 50; - const conditions: SQL[] = [eq(messages.channel_id, channelId)]; + const conditions: SQL[] = [eq(pgMessagesTable.channel_id, channelId)]; if (query.cursor) { - conditions.push(lt(messages.created_at, Number(query.cursor))); + conditions.push(lt(pgMessagesTable.created_at, Number(query.cursor))); } const rows = await db .select() - .from(messages) + .from(pgMessagesTable) .where(and(...conditions)) - .orderBy(desc(messages.created_at)) + .orderBy(desc(pgMessagesTable.created_at)) .limit(limit + 1); const data = rows @@ -216,7 +117,7 @@ export class MessagesRepository { const id = crypto.randomUUID(); const [row] = await db - .insert(messages) + .insert(pgMessagesTable) .values({ id, guild_id: data.guildId, @@ -242,7 +143,7 @@ export class MessagesRepository { async update(id: string, data: MessageUpdate) { const db = getDatabase(); - const setData: Partial = {}; + const setData: Partial = {}; if (data.editedContent !== undefined) { setData.edited_content = data.editedContent; @@ -266,9 +167,9 @@ export class MessagesRepository { if (Object.keys(setData).length === 0) return this.findById(id); const [row] = await db - .update(messages) + .update(pgMessagesTable) .set(setData) - .where(eq(messages.id, id)) + .where(eq(pgMessagesTable.id, id)) .returning(); if (!row) return null; @@ -288,20 +189,20 @@ export class MessagesRepository { messageIds?: string[]; }): Promise { const db = getDatabase(); - const conditions: SQL[] = [eq(messages.ai_status, "error")]; + const conditions: SQL[] = [eq(pgMessagesTable.ai_status, "error")]; if (opts.messageIds && opts.messageIds.length > 0) { - conditions.push(inArray(messages.id, opts.messageIds)); + conditions.push(inArray(pgMessagesTable.id, opts.messageIds)); } if (opts.guildId) { - conditions.push(eq(messages.guild_id, opts.guildId)); + conditions.push(eq(pgMessagesTable.guild_id, opts.guildId)); } if (opts.channelId) { - conditions.push(eq(messages.channel_id, opts.channelId)); + conditions.push(eq(pgMessagesTable.channel_id, opts.channelId)); } const result = await db - .update(messages) + .update(pgMessagesTable) .set({ ai_status: "pending" }) .where(and(...conditions)); @@ -317,9 +218,14 @@ export class MessagesRepository { async markForReanalysis(id: string): Promise { const db = getDatabase(); await db - .update(messages) + .update(pgMessagesTable) .set({ ai_status: "pending" }) - .where(and(eq(messages.id, id), ne(messages.ai_status, "pending"))); + .where( + and( + eq(pgMessagesTable.id, id), + ne(pgMessagesTable.ai_status, "pending"), + ), + ); } /** @@ -332,32 +238,32 @@ export class MessagesRepository { ): Promise[]> { const db = getDatabase(); const conditions: SQL[] = [ - inArray(messages.ai_status, ["warn", "flagged"]), + inArray(pgMessagesTable.ai_status, ["warn", "flagged"]), ]; if (channelId) { - conditions.push(eq(messages.channel_id, channelId)); + conditions.push(eq(pgMessagesTable.channel_id, channelId)); } const rows = await db .select({ - id: messages.id, - guild_id: messages.guild_id, - channel_id: messages.channel_id, - user_id: messages.user_id, - username: messages.username, - avatar_url: messages.avatar_url, - content: messages.content, - type: messages.type, - created_at: messages.created_at, - ai_status: messages.ai_status, - ai_severity: messages.ai_severity, - ai_confidence: messages.ai_confidence, - ai_analysis: messages.ai_analysis, + id: pgMessagesTable.id, + guild_id: pgMessagesTable.guild_id, + channel_id: pgMessagesTable.channel_id, + user_id: pgMessagesTable.user_id, + username: pgMessagesTable.username, + avatar_url: pgMessagesTable.avatar_url, + content: pgMessagesTable.content, + type: pgMessagesTable.type, + created_at: pgMessagesTable.created_at, + ai_status: pgMessagesTable.ai_status, + ai_severity: pgMessagesTable.ai_severity, + ai_confidence: pgMessagesTable.ai_confidence, + ai_analysis: pgMessagesTable.ai_analysis, }) - .from(messages) + .from(pgMessagesTable) .where(and(...conditions)) - .orderBy(desc(messages.created_at)) + .orderBy(desc(pgMessagesTable.created_at)) .limit(limit); return rows as unknown as Record[]; @@ -365,7 +271,9 @@ export class MessagesRepository { async delete(id: string): Promise { const db = getDatabase(); - const result = await db.delete(messages).where(eq(messages.id, id)); + const result = await db + .delete(pgMessagesTable) + .where(eq(pgMessagesTable.id, id)); return (result.rowCount ?? 0) > 0; } @@ -376,17 +284,17 @@ export class MessagesRepository { ): Promise> { const db = getDatabase(); const limit = query.limit ?? 50; - const conditions: SQL[] = [eq(attachments.channel_id, channelId)]; + const conditions: SQL[] = [eq(pgAttachmentsTable.channel_id, channelId)]; if (query.cursor) { - conditions.push(lt(attachments.created_at, Number(query.cursor))); + conditions.push(lt(pgAttachmentsTable.created_at, Number(query.cursor))); } const rows = await db .select() - .from(attachments) + .from(pgAttachmentsTable) .where(and(...conditions)) - .orderBy(desc(attachments.created_at)) + .orderBy(desc(pgAttachmentsTable.created_at)) .limit(limit + 1); const data = rows.map((r) => ({ diff --git a/services/backend/src/modules/voice/guilds.routes.ts b/services/backend/src/modules/voice/guilds.routes.ts index 5f6ca1b..cfc20ca 100644 --- a/services/backend/src/modules/voice/guilds.routes.ts +++ b/services/backend/src/modules/voice/guilds.routes.ts @@ -1,21 +1,36 @@ -import type { Router } from "express"; +import { createChildLogger } from "@bete/shared/logger"; +import type { Request, Response, Router } from "express"; import express from "express"; +import { asyncHandler } from "../../shared/middlewares/index.js"; import { getGuilds, getTextChannels } from "./voice.service.js"; +const logger = createChildLogger("guilds.routes"); + export function createGuildsRouter(): Router { const router = express.Router(); // GET /api/guilds - router.get("/", async (_req, res) => { - const guilds = await getGuilds(); - res.json(guilds); - }); + router.get( + "/", + asyncHandler(async (_req: Request, res: Response) => { + logger.debug("Fetching guilds"); + const guilds = await getGuilds(); + res.json(guilds); + }), + ); // GET /api/guilds/:guildId/channels - router.get("/:guildId/channels", async (req, res) => { - const channels = await getTextChannels(req.params.guildId); - res.json(channels); - }); + router.get( + "/:guildId/channels", + asyncHandler(async (req: Request, res: Response) => { + const guildId = Array.isArray(req.params.guildId) + ? req.params.guildId[0] + : req.params.guildId; + logger.debug({ guildId }, "Fetching text channels"); + const channels = await getTextChannels(guildId); + res.json(channels); + }), + ); return router; } diff --git a/services/backend/src/modules/voice/voice.controller.ts b/services/backend/src/modules/voice/voice.controller.ts index 7f4e5d8..d03a9d5 100644 --- a/services/backend/src/modules/voice/voice.controller.ts +++ b/services/backend/src/modules/voice/voice.controller.ts @@ -1,4 +1,6 @@ +import { createChildLogger } from "@bete/shared/logger"; import type { Request, Response } from "express"; +import { asyncHandler } from "../../shared/middlewares/index.js"; import { publishCommandNoReply } from "../../shared/redis/index.js"; import { connectVoice, @@ -7,10 +9,14 @@ import { getVoiceStatus, } from "./voice.service.js"; -export async function handleGetVoiceStatus(_req: Request, res: Response) { - const status = await getVoiceStatus(); - res.json(status); -} +const logger = createChildLogger("voice.controller"); + +export const handleGetVoiceStatus = asyncHandler( + async (_req: Request, res: Response) => { + const status = await getVoiceStatus(); + res.json(status); + }, +); /** Safely extract a string value that may be a single string or string array. */ function asString(val: unknown): string { @@ -18,47 +24,52 @@ function asString(val: unknown): string { return String(val ?? ""); } -export async function handleConnectVoice(req: Request, res: Response) { - const guildId = asString(req.body.guildId); - const channelId = asString(req.body.channelId); - if (!guildId || !channelId) { - return res.status(400).json({ - error: "VALIDATION_ERROR", - message: "guildId and channelId are required", - }); - } - const status = await connectVoice(guildId, channelId); - res.json(status); -} +export const handleConnectVoice = asyncHandler( + async (req: Request, res: Response) => { + const guildId = asString(req.body.guildId); + const channelId = asString(req.body.channelId); + if (!guildId || !channelId) { + return res.status(400).json({ + error: "VALIDATION_ERROR", + message: "guildId and channelId are required", + }); + } + logger.debug({ guildId, channelId }, "Connecting to voice channel"); + const status = await connectVoice(guildId, channelId); + res.json(status); + }, +); -export async function handleDisconnectVoice(_req: Request, res: Response) { - const status = await disconnectVoice(); - res.json(status); -} +export const handleDisconnectVoice = asyncHandler( + async (_req: Request, res: Response) => { + logger.debug("Disconnecting from voice"); + const status = await disconnectVoice(); + res.json(status); + }, +); -export async function handleGetVoiceChannels(req: Request, res: Response) { - const guildId = asString(req.params.guildId); - const channels = await getVoiceChannels(guildId); - res.json(channels); -} +export const handleGetVoiceChannels = asyncHandler( + async (req: Request, res: Response) => { + const guildId = asString(req.params.guildId); + logger.debug({ guildId }, "Fetching voice channels"); + const channels = await getVoiceChannels(guildId); + res.json(channels); + }, +); -export async function handleVoiceCommand(req: Request, res: Response) { - const command = asString(req.body.command); +export const handleVoiceCommand = asyncHandler( + async (req: Request, res: Response) => { + const command = asString(req.body.command); - if (!command) { - return res.status(400).json({ - error: "VALIDATION_ERROR", - message: "command is required", - }); - } + if (!command) { + return res.status(400).json({ + error: "VALIDATION_ERROR", + message: "command is required", + }); + } - try { + logger.debug({ command }, "Publishing voice command"); await publishCommandNoReply(command); res.json({ success: true, command }); - } catch (err) { - res.status(500).json({ - error: "COMMAND_FAILED", - message: err instanceof Error ? err.message : "Unknown error", - }); - } -} + }, +); diff --git a/services/backend/src/modules/voice/voice.service.ts b/services/backend/src/modules/voice/voice.service.ts index 138e3ad..5500f42 100644 --- a/services/backend/src/modules/voice/voice.service.ts +++ b/services/backend/src/modules/voice/voice.service.ts @@ -6,7 +6,10 @@ import { COMMAND_VOICE_DISCONNECT, VOICE_STATUS_KEY, } from "@bete/shared"; -import { createChildLogger } from "@bete/shared/logger"; +import { + createChildLogger, + tryCommandThenFallback, +} from "../../shared/commandHelper.js"; import { getPool } from "../../shared/database/index.js"; import { publishCommand, readRedisStatus } from "../../shared/redis/index.js"; @@ -31,29 +34,40 @@ export interface VoiceStatus { activeChannelName: string | null; } +export const DEFAULT_VOICE_STATUS: VoiceStatus = { + connected: false, + activeGuildId: null, + activeChannelId: null, + activeChannelName: null, +}; + +function readVoiceStatusFallback(): Promise { + return readRedisStatus(VOICE_STATUS_KEY).then( + (cached) => (cached as unknown as VoiceStatus) ?? DEFAULT_VOICE_STATUS, + ); +} + /** * Get guilds — query from discord-gateway via Redis command for real names. * Falls back to database (distinct guild_id from messages) if gateway unreachable. */ export async function getGuilds(): Promise { logger.info("getGuilds called"); - const reply = await publishCommand(COMMAND_GUILDS_LIST, {}); - if (reply?.success && reply.data && reply.data.length > 0) return reply.data; - - // Fallback: Postgres with synthetic names - logger.warn( - "discord-gateway unreachable, falling back to Postgres for guilds", + return tryCommandThenFallback( + () => publishCommand(COMMAND_GUILDS_LIST, {}), + async () => { + const pool = getPool(); + const { rows } = await pool.query( + `SELECT DISTINCT guild_id FROM messages ORDER BY guild_id`, + ); + return rows.map((row: Record) => ({ + id: String(row.guild_id ?? ""), + name: `Guild ${String(row.guild_id).slice(0, 8)}`, + icon: null, + })); + }, + "getGuilds", ); - const pool = getPool(); - const { rows } = await pool.query( - `SELECT DISTINCT guild_id FROM messages ORDER BY guild_id`, - ); - - return rows.map((row: Record) => ({ - id: String(row.guild_id ?? ""), - name: `Guild ${String(row.guild_id).slice(0, 8)}`, - icon: null, - })); } /** @@ -62,27 +76,22 @@ export async function getGuilds(): Promise { */ export async function getTextChannels(guildId: string): Promise { logger.info({ guildId }, "getTextChannels called"); - const reply = await publishCommand(COMMAND_GUILDS_TEXT_CHANNELS, { - guildId, - }); - if (reply?.success && reply.data && reply.data.length > 0) return reply.data; - - // Fallback: Postgres with synthetic names - logger.warn( - { guildId }, - "discord-gateway unreachable, falling back to Postgres for text channels", + return tryCommandThenFallback( + () => publishCommand(COMMAND_GUILDS_TEXT_CHANNELS, { guildId }), + async () => { + const pool = getPool(); + const { rows } = await pool.query( + `SELECT DISTINCT channel_id FROM messages WHERE guild_id = $1 ORDER BY channel_id`, + [guildId], + ); + return rows.map((row: Record) => ({ + id: String(row.channel_id ?? ""), + name: `Channel ${String(row.channel_id).slice(0, 8)}`, + type: "text" as const, + })); + }, + "getTextChannels", ); - const pool = getPool(); - const { rows } = await pool.query( - `SELECT DISTINCT channel_id FROM messages WHERE guild_id = $1 ORDER BY channel_id`, - [guildId], - ); - - return rows.map((row: Record) => ({ - id: String(row.channel_id ?? ""), - name: `Channel ${String(row.channel_id).slice(0, 8)}`, - type: "text" as const, - })); } /** @@ -102,13 +111,7 @@ export async function getVoiceChannels(guildId: string): Promise { export async function getVoiceStatus(): Promise { logger.debug("getVoiceStatus called"); const cached = await readRedisStatus(VOICE_STATUS_KEY); - if (cached) return cached as unknown as VoiceStatus; - return { - connected: false, - activeGuildId: null, - activeChannelId: null, - activeChannelName: null, - }; + return (cached as unknown as VoiceStatus) ?? DEFAULT_VOICE_STATUS; } /** @@ -119,21 +122,14 @@ export async function connectVoice( channelId: string, ): Promise { logger.info({ guildId, channelId }, "connectVoice called"); - const reply = await publishCommand(COMMAND_VOICE_CONNECT, { - guildId, - channelId, - }); - if (reply?.success && reply.data) return reply.data; - - // Fallback: read from Redis status key - const cached = await readRedisStatus(VOICE_STATUS_KEY); - return ( - (cached as unknown as VoiceStatus) ?? { - connected: false, - activeGuildId: null, - activeChannelId: null, - activeChannelName: null, - } + return tryCommandThenFallback( + () => + publishCommand(COMMAND_VOICE_CONNECT, { + guildId, + channelId, + }), + () => readVoiceStatusFallback(), + "connectVoice", ); } @@ -142,16 +138,9 @@ export async function connectVoice( */ export async function disconnectVoice(): Promise { logger.info("disconnectVoice called"); - const reply = await publishCommand(COMMAND_VOICE_DISCONNECT, {}); - if (reply?.success && reply.data) return reply.data; - - const cached = await readRedisStatus(VOICE_STATUS_KEY); - return ( - (cached as unknown as VoiceStatus) ?? { - connected: false, - activeGuildId: null, - activeChannelId: null, - activeChannelName: null, - } + return tryCommandThenFallback( + () => publishCommand(COMMAND_VOICE_DISCONNECT, {}), + () => readVoiceStatusFallback(), + "disconnectVoice", ); } diff --git a/services/backend/src/shared/commandHelper.ts b/services/backend/src/shared/commandHelper.ts new file mode 100644 index 0000000..35a3e1f --- /dev/null +++ b/services/backend/src/shared/commandHelper.ts @@ -0,0 +1,26 @@ +import type { CommandReply } from "@bete/shared"; +import { createChildLogger } from "@bete/shared/logger"; +import { publishCommand, readRedisStatus } from "./redis/index.js"; + +export { createChildLogger }; + +/** + * Attempt a Redis command first; if it fails or times out, fall back. + * + * @param commandFn - Function that issues the publishCommand and returns the reply. + * @param fallbackFn - Async fallback, typically reads from Redis status key. + * @param commandLabel - Label used for logging (e.g. "voice:connect"). + */ +export async function tryCommandThenFallback( + commandFn: () => Promise | null>, + fallbackFn: () => Promise, + commandLabel: string, +): Promise { + const logger = createChildLogger(`command-helper:${commandLabel}`); + const reply = await commandFn(); + if (reply?.success && reply.data !== undefined && reply.data !== null) { + return reply.data; + } + logger.warn("discord-gateway unreachable, falling back"); + return fallbackFn(); +} diff --git a/services/backend/src/shared/utils/messageMapper.ts b/services/backend/src/shared/utils/messageMapper.ts new file mode 100644 index 0000000..97fce6f --- /dev/null +++ b/services/backend/src/shared/utils/messageMapper.ts @@ -0,0 +1,57 @@ +// Shared message row mapper for backend repository modules + +export interface MappedMessage { + id: string; + guild_id: string; + channel_id: string; + thread_id: string | null; + user_id: string; + username: string; + avatar_url: string | null; + content: string; + edited_content: string | null; + created_at: number; + edited_at: number | null; + deleted_at: number | null; + type: string; + metadata: string | null; + ai_status: string | null; + ai_moderation_flags: string | null; + ai_moderation_score: number | null; + ai_analysis: string | null; + ai_categories: string | null; + ai_severity: string | null; + ai_confidence: number | null; + ai_recommended_action: string | null; + ai_analyzed_at: number | null; + ai_error: string | null; +} + +export function mapMessageRow(row: Record): MappedMessage { + return { + id: String(row.id ?? ""), + guild_id: String(row.guild_id ?? ""), + channel_id: String(row.channel_id ?? ""), + thread_id: (row.thread_id as string | null) ?? null, + user_id: String(row.user_id ?? ""), + username: String(row.username ?? ""), + avatar_url: (row.avatar_url as string | null) ?? null, + content: String(row.content ?? ""), + edited_content: (row.edited_content as string | null) ?? null, + created_at: Number(row.created_at ?? 0), + edited_at: (row.edited_at as number | null) ?? null, + deleted_at: (row.deleted_at as number | null) ?? null, + type: String(row.type ?? "text"), + metadata: (row.metadata as string | null) ?? null, + ai_status: (row.ai_status as string | null) ?? null, + ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null, + ai_moderation_score: (row.ai_moderation_score as number | null) ?? null, + ai_analysis: (row.ai_analysis as string | null) ?? null, + ai_categories: (row.ai_categories as string | null) ?? null, + ai_severity: (row.ai_severity as string | null) ?? null, + ai_confidence: (row.ai_confidence as number | null) ?? null, + ai_recommended_action: (row.ai_recommended_action as string | null) ?? null, + ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null, + ai_error: (row.ai_error as string | null) ?? null, + }; +} diff --git a/services/backend/src/ws/broadcast.ts b/services/backend/src/ws/broadcast.ts index 7b19950..4d7e2ed 100644 --- a/services/backend/src/ws/broadcast.ts +++ b/services/backend/src/ws/broadcast.ts @@ -9,6 +9,10 @@ * broadcastMessageCreated(messageData); */ +import { createChildLogger } from "@bete/shared/logger"; + +const logger = createChildLogger("broadcast"); + type BroadcastFn = (data: unknown) => void; type BroadcastRawFn = (type: string, data: unknown) => void; type BroadcastBinaryFn = (data: Buffer) => void; @@ -30,63 +34,103 @@ export interface BroadcastFunctions { binary: BroadcastBinaryFn; } -const noop: BroadcastFn = () => {}; -const noopRaw: BroadcastRawFn = () => {}; -const noopBinary: BroadcastBinaryFn = () => {}; - let _fns: BroadcastFunctions | null = null; +let _enabled = true; + +/** Enable or disable broadcast logging (disabled by default to reduce noise). */ +export function setBroadcastLogging(enabled: boolean): void { + _enabled = enabled; +} + /** * Inject broadcast functions from the WebSocket server initializer. * Must be called once during server startup before any broadcast is used. */ export function setBroadcastFunctions(fns: BroadcastFunctions): void { _fns = fns; + logger.info("Broadcast functions initialized"); } /** Clear injected functions (used during cleanup). */ export function clearBroadcastFunctions(): void { _fns = null; + logger.info("Broadcast functions cleared"); } -export const broadcastMessageCreated: BroadcastFn = (data) => - (_fns?.messageCreated ?? noop)(data); +function logBroadcast(name: string, data: unknown): void { + if (!_enabled) return; + // Avoid logging binary or PCM data due to volume + if (name === "voice_pcm_data" || name === "binary") return; + logger.debug({ event: name }, "Broadcasting event"); +} -export const broadcastMessageUpdated: BroadcastFn = (data) => - (_fns?.messageUpdated ?? noop)(data); +export const broadcastMessageCreated: BroadcastFn = (data) => { + logBroadcast("message_created", data); + _fns?.messageCreated?.(data); +}; -export const broadcastMessageDeleted: BroadcastFn = (data) => - (_fns?.messageDeleted ?? noop)(data); +export const broadcastMessageUpdated: BroadcastFn = (data) => { + logBroadcast("message_updated", data); + _fns?.messageUpdated?.(data); +}; -export const broadcastAttachmentCreated: BroadcastFn = (data) => - (_fns?.attachmentCreated ?? noop)(data); +export const broadcastMessageDeleted: BroadcastFn = (data) => { + logBroadcast("message_deleted", data); + _fns?.messageDeleted?.(data); +}; -export const broadcastAttachmentUploaded: BroadcastFn = (data) => - (_fns?.attachmentUploaded ?? noop)(data); +export const broadcastAttachmentCreated: BroadcastFn = (data) => { + logBroadcast("attachment_created", data); + _fns?.attachmentCreated?.(data); +}; -export const broadcastMessageAnalyzed: BroadcastFn = (data) => - (_fns?.messageAnalyzed ?? noop)(data); +export const broadcastAttachmentUploaded: BroadcastFn = (data) => { + logBroadcast("attachment_uploaded", data); + _fns?.attachmentUploaded?.(data); +}; -export const broadcastVoiceRecordingStarted: BroadcastFn = (data) => - (_fns?.voiceRecordingStarted ?? noop)(data); +export const broadcastMessageAnalyzed: BroadcastFn = (data) => { + logBroadcast("message_analyzed", data); + _fns?.messageAnalyzed?.(data); +}; -export const broadcastVoiceRecordingStopped: BroadcastFn = (data) => - (_fns?.voiceRecordingStopped ?? noop)(data); +export const broadcastVoiceRecordingStarted: BroadcastFn = (data) => { + logBroadcast("voice_recording_started", data); + _fns?.voiceRecordingStarted?.(data); +}; -export const broadcastVoiceRecordingUploaded: BroadcastFn = (data) => - (_fns?.voiceRecordingUploaded ?? noop)(data); +export const broadcastVoiceRecordingStopped: BroadcastFn = (data) => { + logBroadcast("voice_recording_stopped", data); + _fns?.voiceRecordingStopped?.(data); +}; -export const broadcastVoicePcmData: BroadcastFn = (data) => - (_fns?.voicePcmData ?? noop)(data); +export const broadcastVoiceRecordingUploaded: BroadcastFn = (data) => { + logBroadcast("voice_recording_uploaded", data); + _fns?.voiceRecordingUploaded?.(data); +}; -export const broadcastVoiceActiveUser: BroadcastFn = (data) => - (_fns?.voiceActiveUser ?? noop)(data); +export const broadcastVoicePcmData: BroadcastFn = (data) => { + // PCM data is high-volume; logging is skipped unconditionally + _fns?.voicePcmData?.(data); +}; -export const broadcastAnalysisQueueStatus: BroadcastFn = (data) => - (_fns?.analysisQueueStatus ?? noop)(data); +export const broadcastVoiceActiveUser: BroadcastFn = (data) => { + logBroadcast("voice_active_user", data); + _fns?.voiceActiveUser?.(data); +}; -export const broadcastRaw: BroadcastRawFn = (type, data) => - (_fns?.raw ?? noopRaw)(type, data); +export const broadcastAnalysisQueueStatus: BroadcastFn = (data) => { + logBroadcast("analysis_queue_status", data); + _fns?.analysisQueueStatus?.(data); +}; -export const broadcastBinary: BroadcastBinaryFn = (data) => - (_fns?.binary ?? noopBinary)(data); +export const broadcastRaw: BroadcastRawFn = (type, data) => { + logBroadcast(type, data); + _fns?.raw?.(type, data); +}; + +export const broadcastBinary: BroadcastBinaryFn = (data) => { + // Binary data is high-volume; logging is skipped unconditionally + _fns?.binary?.(data); +}; diff --git a/services/discord-gateway/src/modules/ai-moderation/aiAnalysisWorker.ts b/services/discord-gateway/src/modules/ai-moderation/aiAnalysisWorker.ts index 76f0aae..bc20770 100644 --- a/services/discord-gateway/src/modules/ai-moderation/aiAnalysisWorker.ts +++ b/services/discord-gateway/src/modules/ai-moderation/aiAnalysisWorker.ts @@ -1,3 +1,4 @@ +import { createChildLogger } from "@bete/shared/logger"; import { config } from "../../shared/config/config.js"; import { initializeDatabase } from "../../shared/database/drizzle.js"; import { @@ -15,6 +16,8 @@ import { runSimpleTextFallback, } from "./llmModerationClient.js"; +const logger = createChildLogger("aiAnalysisWorker"); + let dbInitialized = false; let dbInitPromise: Promise | null = null; @@ -70,13 +73,9 @@ export default async function workerRouter( if (!config.AI_LLM_API_KEY) { const errorMsg = "AI_LLM_API_KEY is missing from environment. Worker cannot process moderation requests without credentials."; - console.error( - JSON.stringify({ - level: "ERROR", - context: "aiAnalysisWorker", - error: errorMsg, - timestamp: new Date().toISOString(), - }), + logger.error( + { error: errorMsg }, + "AI_LLM_API_KEY is missing from environment", ); if (job.type === "batch") { @@ -113,15 +112,9 @@ export default async function workerRouter( } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); const errorStack = error instanceof Error ? error.stack : undefined; - console.error( - JSON.stringify({ - level: "ERROR", - context: "aiAnalysisWorker", - type: job.type, - error: errorMessage, - stack: errorStack, - timestamp: new Date().toISOString(), - }), + logger.error( + { type: job.type, error: errorMessage, stack: errorStack }, + "Worker job failed", ); if (job.type === "batch") { return { diff --git a/services/discord-gateway/src/modules/ai-moderation/aiAnalyzer.ts b/services/discord-gateway/src/modules/ai-moderation/aiAnalyzer.ts index 53cb6c4..4c4878d 100644 --- a/services/discord-gateway/src/modules/ai-moderation/aiAnalyzer.ts +++ b/services/discord-gateway/src/modules/ai-moderation/aiAnalyzer.ts @@ -1,1087 +1,53 @@ -import { existsSync } from "node:fs"; -import { availableParallelism } from "node:os"; -import { fileURLToPath } from "node:url"; import { createChildLogger } from "@bete/shared/logger"; import type { Client } from "discord.js-selfbot-v13"; -import { LRUCache } from "lru-cache"; -import { Piscina } from "piscina"; import { config } from "../../shared/config/config.js"; import type { EventBroadcaster } from "../event-broadcaster/index.js"; -import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js"; import { getConversationKeysWithIncompleteAnalysis, getIncompleteMessagesByConversation, getMessageById, getPendingConversationKeys, - getPendingMessagesByConversation, revertStuckProcessingMessages, updateMessageAIAnalysis, - updateMessagesAIAnalysisBulk, } from "../message-capture/messageStore.js"; -import type { - AnalysisQueueStatus, - AnalysisResult, - MessageRecord, -} from "../message-capture/types.js"; -import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js"; -import { estimateTokens } from "./conversationContext.js"; -import { logModerationError } from "./responseLogger.js"; +import type { AnalysisQueueStatus } from "../message-capture/types.js"; +import { + activeRequests, + buildAgeRestrictedSkipResult, + isAgeRestrictedMessage, + skipAgeRestrictedMessages, +} from "./batchProcessor.js"; +import { scheduleConversationAnalysis } from "./batchScheduler.js"; +import { + _redisEventBroadcaster, + broadcastAnalysisCompleted, + conversationConsecutiveErrors, + conversationDebounceTimers, + conversationErrorCooldown, + conversationProcessing, + getConversationKey, + isConversationProcessingLocked, + LAST_ERROR, + setModerationClient, + setSharedEventBroadcaster, +} from "./circuitBreaker.js"; +import { + activeIndividualRequests, + enqueueIndividualFallbacks, + individualCooldownUntil, + individualInFlight, + individualInFlightByConversation, + individualInFlightLastTouched, +} from "./individualFallbackProcessor.js"; const logger = createChildLogger("ai-analyzer"); -// Redis EventBroadcaster — set by startPendingAIAnalysisWorker. -// Used to publish analysis completion events so the backend -// redis-bridge can forward them to frontend WebSocket clients. -let _redisEventBroadcaster: EventBroadcaster | undefined; - -function broadcastAnalysisCompleted(row: MessageRecord): void { - // Redis pub/sub broadcast → backend → frontend WebSocket - if (_redisEventBroadcaster) { - _redisEventBroadcaster.messageAnalyzed(row).catch((err: unknown) => - logger.warn( - { - messageId: row.id, - error: err instanceof Error ? err.message : String(err), - }, - "Failed to publish message_analyzed via Redis EventBroadcaster", - ), - ); - } -} - -function scheduleAutoDelete(row: MessageRecord): void { - if (row.ai_status !== "flagged" && row.ai_status !== "warn") return; - - // Idempotency guard: if a concurrent path (batch + individual fallback) both - // produce a result for the same message, only the first call proceeds. - if (autoDeleteInFlight.has(row.id)) { - logger.debug( - { messageId: row.id }, - "Auto-delete skipped: already in-flight for this message", - ); - return; - } - autoDeleteInFlight.set(row.id, true); - - const run = () => { - attemptAutoDeleteFlaggedMessage(moderationClient, row) - .catch((error: unknown) => { - logger.error( - { - messageId: row.id, - error: error instanceof Error ? error.message : String(error), - }, - "Unexpected auto-delete error", - ); - }) - .finally(() => { - autoDeleteInFlight.delete(row.id); - }); - }; - - if (config.AUTO_DELETE_FLAGGED_DELAY_MS > 0) { - setTimeout(run, config.AUTO_DELETE_FLAGGED_DELAY_MS); - return; - } - setImmediate(run); -} - -function isAgeRestrictedMessage(message: MessageRecord): boolean { - return isAgeRestrictedMetadata(message.metadata); -} - -function buildAgeRestrictedSkipResult(): { - status: "clean"; - flags: string | null; - score: number; - analysis: string; - categories: string[]; - severity: "none"; - confidence: number; - recommendedAction: "none"; - analyzedAt: number; - error: null; -} { - return { - status: "clean", - flags: JSON.stringify(["age_restricted"]), - score: 0, - analysis: "Skipped moderation for age-restricted content.", - categories: ["age_restricted"], - severity: "none", - confidence: 1, - recommendedAction: "none", - analyzedAt: Date.now(), - error: null, - }; -} - -async function skipAgeRestrictedMessages( - messages: MessageRecord[], -): Promise { - const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage); - if (ageRestrictedMessages.length === 0) { - return messages; - } - - const skippedRows = await updateMessagesAIAnalysisBulk( - ageRestrictedMessages.map((message) => ({ - messageId: message.id, - result: buildAgeRestrictedSkipResult(), - })), - ); - - for (const row of skippedRows) { - broadcastAnalysisCompleted(row); - } - - const skippedIds = new Set( - ageRestrictedMessages.map((message) => message.id), - ); - return messages.filter((message) => !skippedIds.has(message.id)); -} - // --------------------------------------------------------------------------- -// Batch pipeline state (with LRU eviction to prevent unbounded memory growth) +// Re-exports from sub-modules (preserving original public API) // --------------------------------------------------------------------------- -/** Debounce timer handle per conversation key. */ -const conversationDebounceTimers = new LRUCache({ - max: 10000, - dispose: (value) => { - clearTimeout(value); - }, -}); -/** Timestamp of when processing started per conversation key. */ -const conversationProcessing = new LRUCache({ max: 10000 }); -/** Cooldown expiry timestamp per conversation key after an error. */ -const conversationErrorCooldown = new LRUCache({ max: 10000 }); - -/** - * Per-message in-flight guard for the auto-delete side-effect. - * `scheduleAutoDelete` is called from both `processBatch` (on batch success) - * and `processIndividualFallback` (on individual success). For a message that - * races through both paths, without this guard two concurrent - * `attemptAutoDeleteFlaggedMessage` calls would be launched — producing a - * duplicate moderation-action log and an unnecessary Discord 10008 error. - * (LRU-backed to prevent unbounded growth from message IDs accumulating forever) - */ -const autoDeleteInFlight = new LRUCache({ max: 10000 }); - -let activeRequests = 0; -let lastError: string | null = null; -let moderationClient: Client | undefined; - -// Batch circuit breaker (LRU-backed to prevent unbounded growth) -const conversationConsecutiveErrors = new LRUCache({ - max: 10000, -}); -const MAX_CONSECUTIVE_ERRORS = 5; -const CONVERSATION_CB_COOLDOWN_MS = 60000; - -/** Alert sinks — called when circuit breakers or sustained errors fire. */ -type CircuitBreakerAlert = { - type: "conversation_cb" | "individual_cb" | "sustained_error"; - conversationKey?: string; - consecutiveErrors: number; - message: string; - lastError?: string | null; -}; - -/** Registered alert handlers */ -const alertHandlers: Array<(alert: CircuitBreakerAlert) => void> = []; - -/** - * Register an alert handler (e.g., for webhook integration). - */ -export function onCircuitBreakerAlert( - handler: (alert: CircuitBreakerAlert) => void, -): void { - alertHandlers.push(handler); -} - -function fireAlert(alert: CircuitBreakerAlert): void { - logger.warn(alert, `CB Alert: ${alert.type} — ${alert.message}`); - for (const handler of alertHandlers) { - try { - handler(alert); - } catch { - // handler errors are non-critical - } - } -} - -function recordConversationBatchFailure(conversationKey: string): void { - const nextCount = - (conversationConsecutiveErrors.get(conversationKey) ?? 0) + 1; - conversationConsecutiveErrors.set(conversationKey, nextCount); - - if (nextCount >= MAX_CONSECUTIVE_ERRORS) { - conversationErrorCooldown.set( - conversationKey, - Date.now() + CONVERSATION_CB_COOLDOWN_MS, - ); - fireAlert({ - type: "conversation_cb", - conversationKey, - consecutiveErrors: nextCount, - message: `Conversation ${conversationKey} circuit breaker triggered after ${nextCount} consecutive errors`, - lastError, - }); - conversationConsecutiveErrors.set(conversationKey, 0); - } -} - -function resetConversationBatchFailures(conversationKey: string): void { - conversationConsecutiveErrors.delete(conversationKey); -} - -// --------------------------------------------------------------------------- -// Individual fallback queue — runs PARALLEL to the batch pipeline. -// -// Design guarantees: -// • Concurrency is capped at config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT. -// • A flat Set de-duplicates so the same message can't be -// in-flight twice (Discord snowflakes are globally unique, but be safe). -// • A Map lets the recovery worker skip conversations -// that already have individual work in progress (#4 fix). -// • A separate circuit breaker prevents a cascade of individual failures -// from hammering a down/rate-limited LLM endpoint (#1+#5 fix). -// • All collections use LRU eviction to prevent unbounded memory growth. -// --------------------------------------------------------------------------- - -/** IDs currently being processed one-by-one (LRU-backed, max 10k entries). */ -const individualInFlight = new LRUCache({ max: 10000 }); - -/** - * Per-conversation count of in-flight individual messages. - * Used by the recovery worker to avoid re-scheduling a conversation that - * already has individual fallback work running for it. - * (LRU-backed to prevent unbounded growth) - */ -const individualInFlightByConversation = new LRUCache({ - max: 10000, -}); - -/** Last-touched timestamp for pruning stale entries (LRU-backed). */ -const individualInFlightLastTouched = new LRUCache({ - max: 10000, -}); - -/** Counter for observability. */ -let activeIndividualRequests = 0; - -// Individual fallback circuit breaker (independent of batch CB) -let individualConsecutiveErrors = 0; -let individualCooldownUntil = 0; -const INDIVIDUAL_COOLDOWN_MS = 60000; - -// --------------------------------------------------------------------------- -// Piscina worker pool (batch path only) -// --------------------------------------------------------------------------- - -function getAnalysisWorkerUrl(): URL { - const candidates = [ - new URL("./aiAnalysisWorker.js", import.meta.url), - new URL("../aiAnalysisWorker.js", import.meta.url), - new URL("./aiAnalysisWorker.ts", import.meta.url), - ]; - - for (const candidate of candidates) { - if (existsSync(fileURLToPath(candidate))) { - return candidate; - } - } - - return candidates[2]; -} - -const workerPool = new Piscina({ - filename: fileURLToPath(getAnalysisWorkerUrl()), - execArgv: process.execArgv, - maxThreads: config.PISCINA_MAX_THREADS ?? availableParallelism(), -}); - -interface AnalysisWorkerResponse { - ok: boolean; - conversationKey: string; - rows: MessageRecord[]; - error?: string; -} - -// --------------------------------------------------------------------------- -// Exported helpers -// --------------------------------------------------------------------------- - -/** - * Gets the conversation key for a message (thread_id or channel_id). - */ -export function getConversationKey(message: MessageRecord): string { - return message.thread_id || message.channel_id; -} - -/** - * Picks a batch of messages within a token budget. - * `tokensPerMessage` accounts for JSON structure overhead around each entry. - * Uses a rough character-based token estimate (avoids async formatMessageForPrompt - * since this function runs in a synchronous promise chain). - */ -export function pickBatchWithinBudget( - messages: MessageRecord[], - maxTokens: number, - tokensPerMessage: number, -): MessageRecord[] { - const batch: MessageRecord[] = []; - let usedTokens = 0; - - for (const msg of messages) { - const content = msg.edited_content ?? msg.content; - // Accurate token count via tiktoken (+ overhead for JSON structure) - const msgTokens = estimateTokens(content) + tokensPerMessage; - - if (usedTokens + msgTokens <= maxTokens) { - batch.push(msg); - usedTokens += msgTokens; - } - } - - return batch; -} - -// --------------------------------------------------------------------------- -// Conversation lock helpers -// --------------------------------------------------------------------------- - -function isConversationProcessingLocked(conversationKey: string): boolean { - const startedAt = conversationProcessing.get(conversationKey); - // FIX #7: use configurable timeout that exceeds (LLM timeout × max retries). - // Old hardcoded value was 30 000 ms — shorter than a single LLM call under retries. - return Boolean( - startedAt && - Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS, - ); -} - -// --------------------------------------------------------------------------- -// Individual fallback pipeline -// --------------------------------------------------------------------------- - -/** - * Processes a single message via the Piscina worker pool (offloaded from - * main thread to avoid blocking the event loop). - * - * The worker handles: - * 1. DB initialization - * 2. Context fetching + conversation building - * 3. Attachment fetching - * 4. LLM analysis (normal or simple fallback) - * - * The main thread handles: - * - DB writes (updateMessagesAIAnalysisBulk) - * - WebSocket/Redis broadcast - * - Analytics cache invalidation - * - Auto-delete scheduling - * - * Infinite-loop prevention: if the LLM consistently drops the single target - * message across all retries (analysis_incomplete), we write a terminal flag - * 'individual_analysis_exhausted' to DB instead of 'analysis_incomplete'. - * The recovery worker only queries for 'analysis_incomplete', so exhausted - * messages are permanently excluded from the reprocessing loop. - * Transient failures (network/parse/DB) are NOT written as exhausted — they - * stay as 'analysis_incomplete' so the circuit-breaker-throttled recovery - * cycle can retry them later. - */ -async function processIndividualFallback( - message: MessageRecord, -): Promise { - const { id: messageId } = message; - const conversationKey = getConversationKey(message); - - activeIndividualRequests++; - individualInFlightByConversation.set( - conversationKey, - (individualInFlightByConversation.get(conversationKey) ?? 0) + 1, - ); - individualInFlightLastTouched.set(conversationKey, Date.now()); - - let exhaustedOnIncomplete = false; - - try { - // ── Run the LLM-heavy work in the worker thread ── - // Try normal analysis first. The worker handles retries internally. - const workerResult = (await workerPool.run({ - type: "individual", - message, - skipNormalAnalysis: false, - } as unknown)) as - | { ok: true; results: AnalysisResult[] } - | { ok: false; results: AnalysisResult[]; error: string }; - - let analysisResult: { results: AnalysisResult[] } | null = null; - let usedSimpleFallback = false; - - if (workerResult.ok) { - const stillIncomplete = workerResult.results.some((r) => - r.flags.includes("analysis_incomplete"), - ); - if (stillIncomplete) { - exhaustedOnIncomplete = true; - analysisResult = null; - } else { - analysisResult = workerResult; - } - } - - // ── Step 2: If normal analysis failed, try SIMPLE fallback via worker ── - if (!analysisResult) { - logger.info( - { messageId }, - "Normal analysis failed (or incomplete) — trying simple text fallback via worker", - ); - - const simpleResult = (await workerPool.run({ - type: "individual", - message, - skipNormalAnalysis: true, - } as unknown)) as - | { ok: true; results: AnalysisResult[] } - | { ok: false; results: AnalysisResult[]; error: string }; - - if (simpleResult.ok) { - analysisResult = simpleResult; - usedSimpleFallback = true; - exhaustedOnIncomplete = false; - } - } - - // If both failed, throw to go to the catch block - if (!analysisResult) { - throw new Error( - `Both normal and simple analysis failed for message ${messageId}`, - ); - } - - if (usedSimpleFallback) { - logger.info( - { messageId, status: analysisResult.results[0]?.status }, - "Used simple text fallback for individual message (via worker)", - ); - } - - // ── Main thread: DB writes + broadcast (non-blocking work) ── - const updates = analysisResult.results.map((r) => ({ - messageId: r.messageId, - result: { - status: r.status, - flags: JSON.stringify(r.flags), - score: r.score, - analysis: r.analysis, - categories: r.categories, - severity: r.severity, - confidence: r.confidence, - recommendedAction: r.recommendedAction, - analyzedAt: Date.now(), - error: null, - }, - })); - - const rows = await updateMessagesAIAnalysisBulk(updates); - for (const row of rows) { - broadcastAnalysisCompleted(row); - scheduleAutoDelete(row); - - // Update reputation autonomously (Belajar & Kebijaksanaan) - if (row.ai_status === "clean") { - import("./userReputationStore.js") - .then((store) => store.recordCleanMessage(row.user_id, row.guild_id)) - .catch((e) => - logger.error( - { error: e }, - "Failed to record clean message streak in fallback", - ), - ); - } else if (row.ai_status === "flagged" && row.ai_severity !== "none") { - import("./userReputationStore.js") - .then((store) => - store.recordInfraction( - row.user_id, - row.guild_id, - row.ai_severity as "low" | "medium" | "high" | "critical", - ), - ) - .catch((e) => - logger.error( - { error: e }, - "Failed to record infraction penalty in fallback", - ), - ); - } - } - - const resultSummary = analysisResult.results[0]; - logModerationError([messageId], config.AI_LLM_MODEL, new Error("Success"), { - phase: "individual_fallback", - status: resultSummary?.status, - flags: resultSummary?.flags, - severity: resultSummary?.severity, - confidence: resultSummary?.confidence, - }); - - individualConsecutiveErrors = 0; - - logger.debug( - { messageId, status: analysisResult.results[0]?.status }, - "Individual fallback analysis complete (via worker)", - ); - } catch (error) { - individualConsecutiveErrors++; - if ( - individualConsecutiveErrors >= config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD - ) { - individualCooldownUntil = Date.now() + INDIVIDUAL_COOLDOWN_MS; - fireAlert({ - type: "individual_cb", - consecutiveErrors: individualConsecutiveErrors, - message: `Individual fallback circuit breaker triggered after ${individualConsecutiveErrors} consecutive errors`, - lastError, - }); - } - - lastError = error instanceof Error ? error.message : String(error); - - logModerationError( - [messageId], - config.AI_LLM_MODEL, - error as Error | string, - { - phase: "individual_fallback", - conversationKey, - exhaustedOnIncomplete, - }, - ); - - if (exhaustedOnIncomplete) { - await updateMessagesAIAnalysisBulk([ - { - messageId, - result: { - status: "error", - flags: JSON.stringify(["individual_analysis_exhausted"]), - score: 0, - analysis: - "Individual fallback exhausted all retries: LLM consistently dropped this message even in single-target mode", - categories: ["individual_analysis_exhausted"], - severity: "none", - confidence: 0, - recommendedAction: "review", - analyzedAt: Date.now(), - error: lastError, - }, - }, - ]).catch((dbErr: unknown) => { - logger.error( - { messageId, error: String(dbErr) }, - "Failed to write terminal exhausted status", - ); - }); - logger.warn( - { messageId }, - "Individual fallback exhausted — marked as individual_analysis_exhausted", - ); - } else { - logger.error( - { - messageId, - error: lastError, - stack: error instanceof Error ? error.stack : undefined, - }, - "Individual fallback analysis failed (transient) — will be retried", - ); - } - } finally { - activeIndividualRequests--; - individualInFlight.delete(messageId); - - const prev = individualInFlightByConversation.get(conversationKey) ?? 1; - if (prev <= 1) { - individualInFlightByConversation.delete(conversationKey); - individualInFlightLastTouched.delete(conversationKey); - } else { - individualInFlightByConversation.set(conversationKey, prev - 1); - individualInFlightLastTouched.set(conversationKey, Date.now()); - } - } -} - -/** - * Fans out message records to the individual fallback queue. - * - * FIX #1: Checks concurrency cap before admitting new work. - * FIX #5: Checks individual circuit breaker before admitting new work. - * Messages that cannot be admitted remain as `error/analysis_incomplete` in - * the DB and will be picked up by the recovery worker on the next interval. - */ -function enqueueIndividualFallbacks(messages: MessageRecord[]): void { - // FIX #5: Honour the individual circuit breaker. - if (Date.now() < individualCooldownUntil) { - logger.warn( - { - until: new Date(individualCooldownUntil).toISOString(), - skipped: messages.length, - }, - "Individual fallback circuit breaker active — messages will be recovered later", - ); - return; - } - - // FIX #5: Enforce concurrency cap — do not admit more individual fallbacks - // than the configured limit. Excess messages stay as error/analysis_incomplete - // and will be recovered on the next worker tick. - const maxConcurrent = config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT ?? 50; - const availableSlots = Math.max(0, maxConcurrent - activeIndividualRequests); - if (availableSlots <= 0) { - logger.debug( - { maxConcurrent, active: activeIndividualRequests }, - "Individual fallback concurrency cap reached — messages will be recovered later", - ); - return; - } - - const newMessages = messages - .filter((m) => !individualInFlight.has(m.id)) - .slice(0, availableSlots); - if (newMessages.length === 0) return; - - logger.debug( - { - count: newMessages.length, - messageIds: newMessages.map((m) => m.id), - }, - "Enqueueing individual fallback analysis for batch-incomplete messages", - ); - - for (const msg of newMessages) { - individualInFlight.set(msg.id, true); - // Fire-and-forget: processIndividualFallback handles all errors internally. - processIndividualFallback(msg).catch((err: unknown) => { - // Belt-and-suspenders guard — should never reach here. - logger.error( - { messageId: msg.id, error: String(err) }, - "Unexpected uncaught error escaping processIndividualFallback", - ); - individualInFlight.delete(msg.id); - const ck = getConversationKey(msg); - const prev = individualInFlightByConversation.get(ck) ?? 1; - if (prev <= 1) { - individualInFlightByConversation.delete(ck); - individualInFlightLastTouched.delete(ck); - } else { - individualInFlightByConversation.set(ck, prev - 1); - individualInFlightLastTouched.set(ck, Date.now()); - } - }); - } -} - -// --------------------------------------------------------------------------- -// Batch pipeline -// --------------------------------------------------------------------------- - -async function processBatch( - conversationKey: string, - messages: MessageRecord[], - processingStartedAt: number, -): Promise { - if (messages.length === 0) { - if (conversationProcessing.get(conversationKey) === processingStartedAt) { - conversationProcessing.delete(conversationKey); - } - return; - } - const cooldownUntil = conversationErrorCooldown.get(conversationKey) ?? 0; - if (Date.now() < cooldownUntil) { - if (conversationProcessing.get(conversationKey) === processingStartedAt) { - conversationProcessing.delete(conversationKey); - } - return; - } - - activeRequests++; - let shouldScheduleNext = false; - try { - const result = (await workerPool.run({ - type: "batch", - conversationKey, - messages, - })) as AnalysisWorkerResponse; - - // Do not broadcast or auto-delete if it's an API failure that will be reverted. - // We check the flags to see if it's an API failure. - for (const row of result.rows) { - let isApiFailure = false; - if (row.ai_status === "error") { - try { - const flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[]; - isApiFailure = flags.includes("analysis_api_failed"); - } catch {} - } - - if (!isApiFailure) { - broadcastAnalysisCompleted(row); - scheduleAutoDelete(row); - - // Update reputation autonomously (Belajar & Kebijaksanaan) - if (row.ai_status === "clean") { - import("./userReputationStore.js") - .then((store) => - store.recordCleanMessage(row.user_id, row.guild_id), - ) - .catch((e) => - logger.error( - { error: e }, - "Failed to record clean message streak", - ), - ); - } else if (row.ai_status === "flagged" && row.ai_severity !== "none") { - import("./userReputationStore.js") - .then((store) => - store.recordInfraction( - row.user_id, - row.guild_id, - row.ai_severity as "low" | "medium" | "high" | "critical", - ), - ) - .catch((e) => - logger.error({ error: e }, "Failed to record infraction penalty"), - ); - } - } - } - - if (!result.ok) { - recordConversationBatchFailure(conversationKey); - - // Batch failed entirely — fall back all messages to individual queue - // so no message is permanently lost behind a cooldown. - logger.warn( - { - conversationKey, - messageCount: messages.length, - error: result.error, - }, - "Batch failed entirely — routing all messages to individual fallback queue", - ); - enqueueIndividualFallbacks(messages); - - lastError = result.error ?? "Analysis worker failed"; - conversationErrorCooldown.set( - conversationKey, - Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, - ); - logger.error( - { - conversationKey, - error: lastError, - messageCount: messages.length, - messageIds: messages.map((m) => m.id), - cooldownUntil: new Date( - Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, - ).toISOString(), - timestamp: new Date().toISOString(), - }, - "Batch analysis failed, will retry after cooldown", - ); - return; - } - - // Batch succeeded — but check for messages the LLM silently dropped or failed to parse/API. - // Rows with flag "analysis_incomplete", "analysis_parse_failed", or "analysis_api_failed" - // were produced by the client as synthetic errors. - const incompleteMessages: MessageRecord[] = []; - const parseFailedMessages: MessageRecord[] = []; - const apiFailedMessages: MessageRecord[] = []; - - for (const msg of messages) { - const row = result.rows.find((r) => r.id === msg.id); - if (!row) { - incompleteMessages.push(msg); - continue; - } - if (row.ai_status === "error") { - let flags: string[] = []; - try { - flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[]; - } catch {} - - if (flags.includes("analysis_incomplete")) { - incompleteMessages.push(msg); - } else if (flags.includes("analysis_parse_failed")) { - parseFailedMessages.push(msg); - } else if (flags.includes("analysis_api_failed")) { - apiFailedMessages.push(msg); - } - } - } - - const messagesForIndividualQueue = [ - ...incompleteMessages, - ...parseFailedMessages, - ]; - - if (messagesForIndividualQueue.length > 0) { - logger.warn( - { - conversationKey, - count: messagesForIndividualQueue.length, - ids: messagesForIndividualQueue.map((m) => m.id), - totalBatchSize: messages.length, - }, - "Batch returned incomplete or unparseable results — fanning out to individual fallback queue", - ); - enqueueIndividualFallbacks(messagesForIndividualQueue); - } - - if (apiFailedMessages.length > 0) { - logger.warn( - { - conversationKey, - count: apiFailedMessages.length, - ids: apiFailedMessages.map((m) => m.id), - }, - "Batch returned API failures — reverting to pending to put back in queue", - ); - - // Revert to pending so they are picked up again - const revertedRows = await updateMessagesAIAnalysisBulk( - apiFailedMessages.map((msg) => ({ - messageId: msg.id, - result: { - status: "pending", - flags: null, - score: null, - analysis: null, - categories: null, - severity: null, - confidence: null, - recommendedAction: null, - analyzedAt: null, - error: null, - }, - })), - ).catch((err) => { - logger.error( - { error: String(err) }, - "Failed to revert API failures to pending", - ); - return []; - }); - - for (const row of revertedRows) { - // Broadcast the pending status so the UI knows it's back in queue - broadcastAnalysisCompleted(row); - } - - // Trigger conversation cooldown so we don't tight loop the API - recordConversationBatchFailure(conversationKey); - // Preserve the longer cooldown: circuit breaker (via recordConversationBatchFailure) - // may have set a 60s cooldown; don't let the shorter config value overwrite it. - const existingCooldown = - conversationErrorCooldown.get(conversationKey) ?? 0; - const newCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS; - if (newCooldown > existingCooldown) { - conversationErrorCooldown.set(conversationKey, newCooldown); - } - - // FIX: Release the processing lock immediately so the cooldown timer - // (not the processing-timeout expiry) controls when this conversation - // is next eligible. Without this the lock would hold for the full - // AI_ANALYSIS_PROCESSING_TIMEOUT_MS before the recovery worker could - // pick the reverted-pending messages back up. - if (conversationProcessing.get(conversationKey) === processingStartedAt) { - conversationProcessing.delete(conversationKey); - } - - // FIX: Do NOT set shouldScheduleNext = true here. The reverted messages - // are now 'pending' again. scheduleConversationAnalysis would race with - // the recovery worker and schedule the same conversation twice — once - // immediately (via shouldScheduleNext) and once after the cooldown - // (via recovery worker). Let the cooldown gate the next attempt. - shouldScheduleNext = false; - } - - if (apiFailedMessages.length === 0) { - resetConversationBatchFailures(conversationKey); - conversationErrorCooldown.delete(conversationKey); - } - shouldScheduleNext = true; - } catch (error) { - recordConversationBatchFailure(conversationKey); - - // Unhandled exception — route everything to individual fallback. - logger.warn( - { conversationKey, messageCount: messages.length }, - "Batch threw exception — routing all messages to individual fallback queue", - ); - enqueueIndividualFallbacks(messages); - - lastError = error instanceof Error ? error.message : String(error); - const errorStack = error instanceof Error ? error.stack : undefined; - // Preserve the longer cooldown: circuit breaker (via recordConversationBatchFailure) - // may have set a 60s cooldown; don't let the shorter config value overwrite it. - const existingCatchCooldown = - conversationErrorCooldown.get(conversationKey) ?? 0; - const newCatchCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS; - if (newCatchCooldown > existingCatchCooldown) { - conversationErrorCooldown.set(conversationKey, newCatchCooldown); - } - logger.error( - { - conversationKey, - error: lastError, - stack: errorStack, - messageCount: messages.length, - messageIds: messages.map((m) => m.id), - cooldownUntil: new Date( - Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, - ).toISOString(), - timestamp: new Date().toISOString(), - }, - "Analysis worker failed, will retry after cooldown", - ); - } finally { - activeRequests--; - if (conversationProcessing.get(conversationKey) === processingStartedAt) { - conversationProcessing.delete(conversationKey); - } - if (shouldScheduleNext) { - setImmediate(() => scheduleConversationAnalysis(conversationKey)); - } - } -} - -// --------------------------------------------------------------------------- -// Scheduling -// --------------------------------------------------------------------------- - -/** - * Schedules a debounced analysis run for a conversation. - * - * FIX #3: The async work inside setTimeout is now wrapped in an explicit - * .catch() so DB errors don't produce unhandled promise rejections. - * FIX #6: Calls pickBatchWithinBudget after fetching messages so token budget - * is respected before handing the batch to the LLM. - * FIX #7: Unified single-timer path — always clear-and-reset one timer per - * conversation key regardless of whether a cooldown is active. The delay is - * simply max(cooldownRemainder+500, debounce) so the same timer serves both - * the "throttled by error cooldown" and "normal debounce" cases, eliminating - * the previous two-path logic that could leave both timers live simultaneously. - */ -function scheduleConversationAnalysis(conversationKey: string): void { - if (isConversationProcessingLocked(conversationKey)) { - return; - } - - const convoCooldown = conversationErrorCooldown.get(conversationKey) ?? 0; - const convoErrors = conversationConsecutiveErrors.get(conversationKey) ?? 0; - - // Hard-block: circuit breaker threshold reached AND cooldown still active. - if (convoErrors >= MAX_CONSECUTIVE_ERRORS && Date.now() < convoCooldown) { - return; - } - - // Unified delay: honour the cooldown window if active, otherwise use the - // normal debounce interval. Always clear-and-reset so only ONE timer is - // ever pending per conversation key regardless of call source. - const now = Date.now(); - const delayMs = - convoCooldown > now - ? convoCooldown - now + 500 - : config.AI_ANALYSIS_DEBOUNCE_MS; - - const existingTimer = conversationDebounceTimers.get(conversationKey); - if (existingTimer) { - clearTimeout(existingTimer); - } - - const timer = setTimeout(() => { - conversationDebounceTimers.delete(conversationKey); - - // FIX TOCTOU: Set lock synchronously BEFORE the async DB fetch starts - if (isConversationProcessingLocked(conversationKey)) { - return; - } - const processingStartedAt = Date.now(); - conversationProcessing.set(conversationKey, processingStartedAt); - - // FIX #3: explicit .catch() — no async arrow function to avoid unhandled rejection. - getPendingMessagesByConversation( - conversationKey, - config.AI_ANALYSIS_MAX_BATCH_SIZE, - ) - .then(async (messages) => { - if (messages.length === 0) { - if ( - conversationProcessing.get(conversationKey) === processingStartedAt - ) { - conversationProcessing.delete(conversationKey); - } - return; - } - - const processableMessages = await skipAgeRestrictedMessages(messages); - if (processableMessages.length === 0) { - if ( - conversationProcessing.get(conversationKey) === processingStartedAt - ) { - conversationProcessing.delete(conversationKey); - } - return; - } - - // FIX #6: trim to token budget before sending to LLM. - // 50 tokens overhead accounts for JSON structure + id/username fields. - let trimmed = pickBatchWithinBudget( - processableMessages, - config.AI_ANALYSIS_MAX_TARGET_TOKENS, - 50, - ); - - // FIX #10: if every message individually exceeds the token budget, - // pickBatchWithinBudget returns [] — which would leave them permanently - // stuck as `pending`. Fall back to the first message alone so at - // least one makes progress; the rest will be processed in later ticks. - if (trimmed.length === 0 && processableMessages.length > 0) { - trimmed = processableMessages.slice(0, 1); - logger.warn( - { - conversationKey, - messageId: processableMessages[0]?.id, - tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS, - }, - "All messages exceed token budget — processing first message alone to avoid stuck-pending deadlock", - ); - } - - return processBatch(conversationKey, trimmed, processingStartedAt); - }) - .catch((err: unknown) => { - if ( - conversationProcessing.get(conversationKey) === processingStartedAt - ) { - conversationProcessing.delete(conversationKey); - } - logger.error( - { - conversationKey, - error: err instanceof Error ? err.message : String(err), - }, - "Failed to fetch or dispatch pending messages for scheduled analysis", - ); - }); - }, delayMs); - - conversationDebounceTimers.set(conversationKey, timer); -} +export { pickBatchWithinBudget } from "./batchProcessor.js"; +export { getConversationKey, onCircuitBreakerAlert } from "./circuitBreaker.js"; // --------------------------------------------------------------------------- // Public API @@ -1145,7 +111,7 @@ export function getAnalysisQueueStatus(): AnalysisQueueStatus { activeIndividualRequests, individualInFlightCount: individualInFlight.size, individualCircuitBreakerActive: Date.now() < individualCooldownUntil, - lastError, + lastError: LAST_ERROR.value, }; } @@ -1160,8 +126,8 @@ export function startPendingAIAnalysisWorker( client?: Client, eventBroadcaster?: EventBroadcaster, ): void { - moderationClient = client; - _redisEventBroadcaster = eventBroadcaster; + setModerationClient(client); + setSharedEventBroadcaster(eventBroadcaster); if (!config.AI_ANALYSIS_ENABLED) return; import("./cultureLearner.js") @@ -1176,7 +142,7 @@ export function startPendingAIAnalysisWorker( ); }); - // FIX #3 pattern: no async arrow — chain promises explicitly. + // FIX #3 pattern: no async arrow -- chain promises explicitly. Promise.all([ getPendingConversationKeys(500), getConversationKeysWithIncompleteAnalysis(200), @@ -1196,7 +162,7 @@ export function startPendingAIAnalysisWorker( } // FIX #7: Prune stale in-flight counters for conversations that have - // been idle longer than the processing timeout — prevents permanent + // been idle longer than the processing timeout -- prevents permanent // blocking if a decrement was missed due to an uncaught exception. const staleThreshold = config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS * 2; for (const [key, lastTouched] of individualInFlightLastTouched) { @@ -1220,32 +186,30 @@ export function startPendingAIAnalysisWorker( } // FIX #8: Build a set of keys already targeted for individual recovery - // so the batch loop below skips them, preventing a race where batch - // scheduling and individual scheduling collide on the same conversation. + // so the batch loop below skips them. const incompleteKeySet = new Set(incompleteKeys); - // --- Batch recovery for `pending` messages --- + // --- Batch recovery for pending messages --- for (const key of pendingKeys) { if (conversationDebounceTimers.has(key)) continue; if (isConversationProcessingLocked(key)) continue; // FIX #4: skip if individual fallback already running for this conversation. if (individualInFlightByConversation.has(key)) continue; - // FIX #8: skip if this conversation also needs individual recovery - // (batch processing would conflict with in-flight individual work). + // FIX #8: skip if this conversation also needs individual recovery. if (incompleteKeySet.has(key)) continue; const cooldownUntil = conversationErrorCooldown.get(key); if (cooldownUntil && now < cooldownUntil) continue; scheduleConversationAnalysis(key); } - // --- Individual recovery for `error/analysis_incomplete` messages --- + // --- Individual recovery for error/analysis_incomplete messages --- // Circuit breaker check: no point iterating if individual CB is active. if (now >= individualCooldownUntil) { const promises: Promise[] = []; for (const key of incompleteKeys) { // Skip if individual work is already running for this conversation. if (individualInFlightByConversation.has(key)) continue; - // Skip if batch processing is running (it will fan-out if it finds more incomplete). + // Skip if batch processing is running. if (isConversationProcessingLocked(key)) continue; promises.push( diff --git a/services/discord-gateway/src/modules/ai-moderation/batchProcessor.ts b/services/discord-gateway/src/modules/ai-moderation/batchProcessor.ts new file mode 100644 index 0000000..9a32c36 --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/batchProcessor.ts @@ -0,0 +1,399 @@ +import { createChildLogger } from "@bete/shared/logger"; +import { config } from "../../shared/config/config.js"; +import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js"; +import { updateMessagesAIAnalysisBulk } from "../message-capture/messageStore.js"; +import type { MessageRecord } from "../message-capture/types.js"; +import { + broadcastAnalysisCompleted, + conversationErrorCooldown, + conversationProcessing, + LAST_ERROR, + recordConversationBatchFailure, + resetConversationBatchFailures, + scheduleAutoDelete, + workerPool, +} from "./circuitBreaker.js"; +import { estimateTokens } from "./conversationContext.js"; +import { enqueueIndividualFallbacks } from "./individualFallbackProcessor.js"; + +const logger = createChildLogger("batch-processor"); + +export interface AnalysisWorkerResponse { + ok: boolean; + conversationKey: string; + rows: MessageRecord[]; + error?: string; +} + +// --------------------------------------------------------------------------- +// Observability +// --------------------------------------------------------------------------- + +export let activeRequests = 0; + +// --------------------------------------------------------------------------- +// Exported helpers +// --------------------------------------------------------------------------- + +/** + * Picks a batch of messages within a token budget. + * `tokensPerMessage` accounts for JSON structure overhead around each entry. + * Uses a rough character-based token estimate (avoids async formatMessageForPrompt + * since this function runs in a synchronous promise chain). + */ +export function pickBatchWithinBudget( + messages: MessageRecord[], + maxTokens: number, + tokensPerMessage: number, +): MessageRecord[] { + const batch: MessageRecord[] = []; + let usedTokens = 0; + + for (const msg of messages) { + const content = msg.edited_content ?? msg.content; + // Accurate token count via tiktoken (+ overhead for JSON structure) + const msgTokens = estimateTokens(content) + tokensPerMessage; + + if (usedTokens + msgTokens <= maxTokens) { + batch.push(msg); + usedTokens += msgTokens; + } + } + + return batch; +} + +// --------------------------------------------------------------------------- +// Age-restricted message helpers +// --------------------------------------------------------------------------- + +export function isAgeRestrictedMessage(message: MessageRecord): boolean { + return isAgeRestrictedMetadata(message.metadata); +} + +export function buildAgeRestrictedSkipResult(): { + status: "clean"; + flags: string | null; + score: number; + analysis: string; + categories: string[]; + severity: "none"; + confidence: number; + recommendedAction: "none"; + analyzedAt: number; + error: null; +} { + return { + status: "clean", + flags: JSON.stringify(["age_restricted"]), + score: 0, + analysis: "Skipped moderation for age-restricted content.", + categories: ["age_restricted"], + severity: "none", + confidence: 1, + recommendedAction: "none", + analyzedAt: Date.now(), + error: null, + }; +} + +export async function skipAgeRestrictedMessages( + messages: MessageRecord[], +): Promise { + const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage); + if (ageRestrictedMessages.length === 0) { + return messages; + } + + const skippedRows = await updateMessagesAIAnalysisBulk( + ageRestrictedMessages.map((message) => ({ + messageId: message.id, + result: buildAgeRestrictedSkipResult(), + })), + ); + + for (const row of skippedRows) { + broadcastAnalysisCompleted(row); + } + + const skippedIds = new Set( + ageRestrictedMessages.map((message) => message.id), + ); + return messages.filter((message) => !skippedIds.has(message.id)); +} + +// --------------------------------------------------------------------------- +// Batch pipeline +// --------------------------------------------------------------------------- + +async function postBatchReputationUpdate(rows: MessageRecord[]): Promise { + for (const row of rows) { + if (row.ai_status === "clean") { + import("./userReputationStore.js") + .then((store) => store.recordCleanMessage(row.user_id, row.guild_id)) + .catch((e) => + logger.error({ error: e }, "Failed to record clean message streak"), + ); + } else if (row.ai_status === "flagged" && row.ai_severity !== "none") { + import("./userReputationStore.js") + .then((store) => + store.recordInfraction( + row.user_id, + row.guild_id, + row.ai_severity as "low" | "medium" | "high" | "critical", + ), + ) + .catch((e) => + logger.error({ error: e }, "Failed to record infraction penalty"), + ); + } + } +} + +export async function processBatch( + conversationKey: string, + messages: MessageRecord[], + processingStartedAt: number, +): Promise { + if (messages.length === 0) { + if (conversationProcessing.get(conversationKey) === processingStartedAt) { + conversationProcessing.delete(conversationKey); + } + return; + } + const cooldownUntil = conversationErrorCooldown.get(conversationKey) ?? 0; + if (Date.now() < cooldownUntil) { + if (conversationProcessing.get(conversationKey) === processingStartedAt) { + conversationProcessing.delete(conversationKey); + } + return; + } + + activeRequests++; + let shouldScheduleNext = false; + try { + const result = (await workerPool.run({ + type: "batch", + conversationKey, + messages, + })) as AnalysisWorkerResponse; + + // Do not broadcast or auto-delete if it's an API failure that will be reverted. + for (const row of result.rows) { + let isApiFailure = false; + if (row.ai_status === "error") { + try { + const flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[]; + isApiFailure = flags.includes("analysis_api_failed"); + } catch {} + } + + if (!isApiFailure) { + broadcastAnalysisCompleted(row); + scheduleAutoDelete(row); + } + } + + // Post-batch reputation updates (fire-and-forget) + postBatchReputationUpdate( + result.rows.filter((r) => { + if (r.ai_status === "error") { + try { + const flags = JSON.parse(r.ai_moderation_flags ?? "[]") as string[]; + return !flags.includes("analysis_api_failed"); + } catch { + return false; + } + } + return true; + }), + ); + + if (!result.ok) { + recordConversationBatchFailure(conversationKey); + + // Batch failed entirely -- fall back all messages to individual queue + logger.warn( + { + conversationKey, + messageCount: messages.length, + error: result.error, + }, + "Batch failed entirely -- routing all messages to individual fallback queue", + ); + enqueueIndividualFallbacks(messages); + + LAST_ERROR.value = result.error ?? "Analysis worker failed"; + conversationErrorCooldown.set( + conversationKey, + Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, + ); + logger.error( + { + conversationKey, + error: LAST_ERROR.value, + messageCount: messages.length, + messageIds: messages.map((m) => m.id), + cooldownUntil: new Date( + Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, + ).toISOString(), + timestamp: new Date().toISOString(), + }, + "Batch analysis failed, will retry after cooldown", + ); + return; + } + + // Batch succeeded -- check for messages the LLM silently dropped or failed + const incompleteMessages: MessageRecord[] = []; + const parseFailedMessages: MessageRecord[] = []; + const apiFailedMessages: MessageRecord[] = []; + + for (const msg of messages) { + const row = result.rows.find((r) => r.id === msg.id); + if (!row) { + incompleteMessages.push(msg); + continue; + } + if (row.ai_status === "error") { + let flags: string[] = []; + try { + flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[]; + } catch {} + + if (flags.includes("analysis_incomplete")) { + incompleteMessages.push(msg); + } else if (flags.includes("analysis_parse_failed")) { + parseFailedMessages.push(msg); + } else if (flags.includes("analysis_api_failed")) { + apiFailedMessages.push(msg); + } + } + } + + const messagesForIndividualQueue = [ + ...incompleteMessages, + ...parseFailedMessages, + ]; + + if (messagesForIndividualQueue.length > 0) { + logger.warn( + { + conversationKey, + count: messagesForIndividualQueue.length, + ids: messagesForIndividualQueue.map((m) => m.id), + totalBatchSize: messages.length, + }, + "Batch returned incomplete or unparseable results -- fanning out to individual fallback queue", + ); + enqueueIndividualFallbacks(messagesForIndividualQueue); + } + + if (apiFailedMessages.length > 0) { + logger.warn( + { + conversationKey, + count: apiFailedMessages.length, + ids: apiFailedMessages.map((m) => m.id), + }, + "Batch returned API failures -- reverting to pending to put back in queue", + ); + + // Revert to pending so they are picked up again + const revertedRows = await updateMessagesAIAnalysisBulk( + apiFailedMessages.map((msg) => ({ + messageId: msg.id, + result: { + status: "pending", + flags: null, + score: null, + analysis: null, + categories: null, + severity: null, + confidence: null, + recommendedAction: null, + analyzedAt: null, + error: null, + }, + })), + ).catch((err) => { + logger.error( + { error: String(err) }, + "Failed to revert API failures to pending", + ); + return []; + }); + + for (const row of revertedRows) { + broadcastAnalysisCompleted(row); + } + + // Trigger conversation cooldown + recordConversationBatchFailure(conversationKey); + const existingCooldown = + conversationErrorCooldown.get(conversationKey) ?? 0; + const newCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS; + if (newCooldown > existingCooldown) { + conversationErrorCooldown.set(conversationKey, newCooldown); + } + + // Release the processing lock immediately so the cooldown timer controls retry + if (conversationProcessing.get(conversationKey) === processingStartedAt) { + conversationProcessing.delete(conversationKey); + } + + // Do NOT schedule next -- let the cooldown gate it + shouldScheduleNext = false; + } + + if (apiFailedMessages.length === 0) { + resetConversationBatchFailures(conversationKey); + conversationErrorCooldown.delete(conversationKey); + } + shouldScheduleNext = true; + } catch (error) { + recordConversationBatchFailure(conversationKey); + + logger.warn( + { conversationKey, messageCount: messages.length }, + "Batch threw exception -- routing all messages to individual fallback queue", + ); + enqueueIndividualFallbacks(messages); + + LAST_ERROR.value = error instanceof Error ? error.message : String(error); + const errorStack = error instanceof Error ? error.stack : undefined; + const existingCatchCooldown = + conversationErrorCooldown.get(conversationKey) ?? 0; + const newCatchCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS; + if (newCatchCooldown > existingCatchCooldown) { + conversationErrorCooldown.set(conversationKey, newCatchCooldown); + } + logger.error( + { + conversationKey, + error: LAST_ERROR.value, + stack: errorStack, + messageCount: messages.length, + messageIds: messages.map((m) => m.id), + cooldownUntil: new Date( + Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS, + ).toISOString(), + timestamp: new Date().toISOString(), + }, + "Analysis worker failed, will retry after cooldown", + ); + } finally { + activeRequests--; + if (conversationProcessing.get(conversationKey) === processingStartedAt) { + conversationProcessing.delete(conversationKey); + } + if (shouldScheduleNext) { + setImmediate(() => { + // Dynamic import to avoid circular dependency at module scope + import("./batchScheduler.js").then((m) => + m.scheduleConversationAnalysis(conversationKey), + ); + }); + } + } +} diff --git a/services/discord-gateway/src/modules/ai-moderation/batchScheduler.ts b/services/discord-gateway/src/modules/ai-moderation/batchScheduler.ts new file mode 100644 index 0000000..cd4f93e --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/batchScheduler.ts @@ -0,0 +1,140 @@ +import { createChildLogger } from "@bete/shared/logger"; +import { config } from "../../shared/config/config.js"; +import { getPendingMessagesByConversation } from "../message-capture/messageStore.js"; +import type { MessageRecord } from "../message-capture/types.js"; +import { + pickBatchWithinBudget, + processBatch, + skipAgeRestrictedMessages, +} from "./batchProcessor.js"; +import { + conversationConsecutiveErrors, + conversationDebounceTimers, + conversationErrorCooldown, + conversationProcessing, + isConversationProcessingLocked, + MAX_CONSECUTIVE_ERRORS, +} from "./circuitBreaker.js"; + +const logger = createChildLogger("batch-scheduler"); + +// --------------------------------------------------------------------------- +// Scheduling +// --------------------------------------------------------------------------- + +/** + * Schedules a debounced analysis run for a conversation. + * + * FIX #3: The async work inside setTimeout is now wrapped in an explicit + * .catch() so DB errors don't produce unhandled promise rejections. + * FIX #6: Calls pickBatchWithinBudget after fetching messages so token budget + * is respected before handing the batch to the LLM. + * FIX #7: Unified single-timer path -- always clear-and-reset one timer per + * conversation key regardless of whether a cooldown is active. The delay is + * simply max(cooldownRemainder+500, debounce) so the same timer serves both + * the "throttled by error cooldown" and "normal debounce" cases, eliminating + * the previous two-path logic that could leave both timers live simultaneously. + */ +export function scheduleConversationAnalysis(conversationKey: string): void { + if (isConversationProcessingLocked(conversationKey)) { + return; + } + + const convoCooldown = conversationErrorCooldown.get(conversationKey) ?? 0; + const convoErrors = conversationConsecutiveErrors.get(conversationKey) ?? 0; + + // Hard-block: circuit breaker threshold reached AND cooldown still active. + if (convoErrors >= MAX_CONSECUTIVE_ERRORS && Date.now() < convoCooldown) { + return; + } + + // Unified delay: honour the cooldown window if active, otherwise use the + // normal debounce interval. Always clear-and-reset so only ONE timer is + // ever pending per conversation key regardless of call source. + const now = Date.now(); + const delayMs = + convoCooldown > now + ? convoCooldown - now + 500 + : config.AI_ANALYSIS_DEBOUNCE_MS; + + const existingTimer = conversationDebounceTimers.get(conversationKey); + if (existingTimer) { + clearTimeout(existingTimer); + } + + const timer = setTimeout(() => { + conversationDebounceTimers.delete(conversationKey); + + // FIX TOCTOU: Set lock synchronously BEFORE the async DB fetch starts + if (isConversationProcessingLocked(conversationKey)) { + return; + } + const processingStartedAt = Date.now(); + conversationProcessing.set(conversationKey, processingStartedAt); + + // FIX #3: explicit .catch() -- no async arrow function to avoid unhandled rejection. + getPendingMessagesByConversation( + conversationKey, + config.AI_ANALYSIS_MAX_BATCH_SIZE, + ) + .then(async (messages: MessageRecord[]) => { + if (messages.length === 0) { + if ( + conversationProcessing.get(conversationKey) === processingStartedAt + ) { + conversationProcessing.delete(conversationKey); + } + return; + } + + const processableMessages = await skipAgeRestrictedMessages(messages); + if (processableMessages.length === 0) { + if ( + conversationProcessing.get(conversationKey) === processingStartedAt + ) { + conversationProcessing.delete(conversationKey); + } + return; + } + + // FIX #6: trim to token budget before sending to LLM. + let trimmed = pickBatchWithinBudget( + processableMessages, + config.AI_ANALYSIS_MAX_TARGET_TOKENS, + 50, + ); + + // FIX #10: if every message individually exceeds the token budget, + // fall back to the first message alone. + if (trimmed.length === 0 && processableMessages.length > 0) { + trimmed = processableMessages.slice(0, 1); + logger.warn( + { + conversationKey, + messageId: processableMessages[0]?.id, + tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS, + }, + "All messages exceed token budget -- processing first message alone to avoid stuck-pending deadlock", + ); + } + + return processBatch(conversationKey, trimmed, processingStartedAt); + }) + .catch((err: unknown) => { + if ( + conversationProcessing.get(conversationKey) === processingStartedAt + ) { + conversationProcessing.delete(conversationKey); + } + logger.error( + { + conversationKey, + error: err instanceof Error ? err.message : String(err), + }, + "Failed to fetch or dispatch pending messages for scheduled analysis", + ); + }); + }, delayMs); + + conversationDebounceTimers.set(conversationKey, timer); +} diff --git a/services/discord-gateway/src/modules/ai-moderation/channelCultureStore.ts b/services/discord-gateway/src/modules/ai-moderation/channelCultureStore.ts index 9f91b9a..faecfd7 100644 --- a/services/discord-gateway/src/modules/ai-moderation/channelCultureStore.ts +++ b/services/discord-gateway/src/modules/ai-moderation/channelCultureStore.ts @@ -1,3 +1,4 @@ +import { createChildLogger } from "@bete/shared/logger"; import { eq } from "drizzle-orm"; import { getDatabase } from "../../shared/database/drizzle.js"; import { @@ -5,6 +6,8 @@ import { channelCulturesTable, } from "../../shared/database/schema.js"; +const logger = createChildLogger("channelCultureStore"); + /** * Fetch the AI-generated culture summary for a channel. */ @@ -18,6 +21,11 @@ export async function getChannelCulture( .where(eq(channelCulturesTable.channel_id, channelId)) .limit(1); + if (existing[0]) { + logger.debug({ channelId }, "Channel culture lookup: found"); + } else { + logger.debug({ channelId }, "Channel culture lookup: not found"); + } return existing[0] || null; } @@ -46,4 +54,9 @@ export async function updateChannelCulture( last_analyzed_at: Date.now(), }, }); + + logger.debug( + { channelId, guildId, cultureSummary }, + "Channel culture updated", + ); } diff --git a/services/discord-gateway/src/modules/ai-moderation/circuitBreaker.ts b/services/discord-gateway/src/modules/ai-moderation/circuitBreaker.ts new file mode 100644 index 0000000..db388dc --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/circuitBreaker.ts @@ -0,0 +1,235 @@ +import { existsSync } from "node:fs"; +import { availableParallelism } from "node:os"; +import { fileURLToPath } from "node:url"; +import { createChildLogger } from "@bete/shared/logger"; +import type { Client } from "discord.js-selfbot-v13"; +import { LRUCache } from "lru-cache"; +import { Piscina } from "piscina"; +import { config } from "../../shared/config/config.js"; +import type { EventBroadcaster } from "../event-broadcaster/index.js"; +import type { MessageRecord } from "../message-capture/types.js"; +import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js"; + +const logger = createChildLogger("circuit-breaker"); + +// --------------------------------------------------------------------------- +// Piscina worker pool (shared by batch + individual pipelines) +// --------------------------------------------------------------------------- + +function getAnalysisWorkerUrl(): URL { + const candidates = [ + new URL("./aiAnalysisWorker.js", import.meta.url), + new URL("../aiAnalysisWorker.js", import.meta.url), + new URL("./aiAnalysisWorker.ts", import.meta.url), + ]; + + for (const candidate of candidates) { + if (existsSync(fileURLToPath(candidate))) { + return candidate; + } + } + + return candidates[2]; +} + +export const workerPool = new Piscina({ + filename: fileURLToPath(getAnalysisWorkerUrl()), + execArgv: process.execArgv, + maxThreads: config.PISCINA_MAX_THREADS ?? availableParallelism(), +}); + +/** + * Gets the conversation key for a message (thread_id or channel_id). + */ +export function getConversationKey(message: MessageRecord): string { + return message.thread_id || message.channel_id; +} + +// --------------------------------------------------------------------------- +// Shared observable state +// --------------------------------------------------------------------------- + +/** Redis EventBroadcaster -- set externally so sub-modules can publish events. */ +export let _redisEventBroadcaster: EventBroadcaster | undefined; + +/** Discord client reference -- needed for auto-delete actions. */ +export let moderationClient: Client | undefined; + +export function setSharedEventBroadcaster( + eb: EventBroadcaster | undefined, +): void { + _redisEventBroadcaster = eb; +} + +export function setModerationClient(mc: Client | undefined): void { + moderationClient = mc; +} + +/** + * Per-message in-flight guard for the auto-delete side-effect. + * (LRU-backed to prevent unbounded growth) + */ +export const autoDeleteInFlight = new LRUCache({ max: 10000 }); + +/** Last recorded error across all pipelines. */ +export const LAST_ERROR: { value: string | null } = { value: null }; + +// --------------------------------------------------------------------------- +// Batch circuit breaker state +// --------------------------------------------------------------------------- + +export const conversationConsecutiveErrors = new LRUCache({ + max: 10000, +}); +export const MAX_CONSECUTIVE_ERRORS = 5; +export const CONVERSATION_CB_COOLDOWN_MS = 60000; +export const conversationErrorCooldown = new LRUCache({ + max: 10000, +}); + +// --------------------------------------------------------------------------- +// Scheduling / timing state (shared so sub-modules can access without cycles) +// --------------------------------------------------------------------------- + +/** Debounce timer handle per conversation key. */ +export const conversationDebounceTimers = new LRUCache({ + max: 10000, + dispose: (value) => { + clearTimeout(value); + }, +}); + +/** Timestamp of when processing started per conversation key. */ +export const conversationProcessing = new LRUCache({ + max: 10000, +}); + +// --------------------------------------------------------------------------- +// Conversation lock helper +// --------------------------------------------------------------------------- + +export function isConversationProcessingLocked( + conversationKey: string, +): boolean { + const startedAt = conversationProcessing.get(conversationKey); + return Boolean( + startedAt && + Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS, + ); +} + +// --------------------------------------------------------------------------- +// Alert system +// --------------------------------------------------------------------------- + +export type CircuitBreakerAlert = { + type: "conversation_cb" | "individual_cb" | "sustained_error"; + conversationKey?: string; + consecutiveErrors: number; + message: string; + lastError?: string | null; +}; + +const alertHandlers: Array<(alert: CircuitBreakerAlert) => void> = []; + +/** + * Register an alert handler (e.g., for webhook integration). + */ +export function onCircuitBreakerAlert( + handler: (alert: CircuitBreakerAlert) => void, +): void { + alertHandlers.push(handler); +} + +export function fireAlert(alert: CircuitBreakerAlert): void { + logger.warn(alert, `CB Alert: ${alert.type} -- ${alert.message}`); + for (const handler of alertHandlers) { + try { + handler(alert); + } catch { + // handler errors are non-critical + } + } +} + +// --------------------------------------------------------------------------- +// Circuit breaker helpers +// --------------------------------------------------------------------------- + +export function recordConversationBatchFailure(conversationKey: string): void { + const nextCount = + (conversationConsecutiveErrors.get(conversationKey) ?? 0) + 1; + conversationConsecutiveErrors.set(conversationKey, nextCount); + + if (nextCount >= MAX_CONSECUTIVE_ERRORS) { + conversationErrorCooldown.set( + conversationKey, + Date.now() + CONVERSATION_CB_COOLDOWN_MS, + ); + fireAlert({ + type: "conversation_cb", + conversationKey, + consecutiveErrors: nextCount, + message: `Conversation ${conversationKey} circuit breaker triggered after ${nextCount} consecutive errors`, + lastError: LAST_ERROR.value, + }); + conversationConsecutiveErrors.set(conversationKey, 0); + } +} + +export function resetConversationBatchFailures(conversationKey: string): void { + conversationConsecutiveErrors.delete(conversationKey); +} + +// --------------------------------------------------------------------------- +// Broadcast & auto-delete helpers +// --------------------------------------------------------------------------- + +export function broadcastAnalysisCompleted(row: MessageRecord): void { + if (_redisEventBroadcaster) { + _redisEventBroadcaster.messageAnalyzed(row).catch((err: unknown) => + logger.warn( + { + messageId: row.id, + error: err instanceof Error ? err.message : String(err), + }, + "Failed to publish message_analyzed via Redis EventBroadcaster", + ), + ); + } +} + +export function scheduleAutoDelete(row: MessageRecord): void { + if (row.ai_status !== "flagged" && row.ai_status !== "warn") return; + + if (autoDeleteInFlight.has(row.id)) { + logger.debug( + { messageId: row.id }, + "Auto-delete skipped: already in-flight for this message", + ); + return; + } + autoDeleteInFlight.set(row.id, true); + + const run = () => { + attemptAutoDeleteFlaggedMessage(moderationClient, row) + .catch((error: unknown) => { + logger.error( + { + messageId: row.id, + error: error instanceof Error ? error.message : String(error), + }, + "Unexpected auto-delete error", + ); + }) + .finally(() => { + autoDeleteInFlight.delete(row.id); + }); + }; + + if (config.AUTO_DELETE_FLAGGED_DELAY_MS > 0) { + setTimeout(run, config.AUTO_DELETE_FLAGGED_DELAY_MS); + return; + } + setImmediate(run); +} diff --git a/services/discord-gateway/src/modules/ai-moderation/concurrencyLimiter.ts b/services/discord-gateway/src/modules/ai-moderation/concurrencyLimiter.ts index 1be7834..9bdaf1e 100644 --- a/services/discord-gateway/src/modules/ai-moderation/concurrencyLimiter.ts +++ b/services/discord-gateway/src/modules/ai-moderation/concurrencyLimiter.ts @@ -1,6 +1,9 @@ +import { createChildLogger } from "@bete/shared/logger"; import pLimit from "p-limit"; import { config } from "../../shared/config/config.js"; +const logger = createChildLogger("concurrencyLimiter"); + /** * Concurrency limiter for LLM API calls. * @@ -9,6 +12,42 @@ import { config } from "../../shared/config/config.js"; */ const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5); +let activeCount = 0; +let pendingCount = 0; + +// Track queue state changes for logging +function updateCounts(): void { + // p-limit exposes queueSize and activeCount via constructor internals, + // but we track via our wrapper to avoid depending on internals. +} + export async function withLlmConcurrency(fn: () => Promise): Promise { - return llmSemaphore(fn); + const queuedAt = activeCount + pendingCount; + pendingCount++; + logger.debug( + { activeCount, pendingCount, maxConcurrent: config.AI_LLM_MAX_CONCURRENT }, + "Queuing LLM request", + ); + + return llmSemaphore(async () => { + pendingCount--; + activeCount++; + + if (activeCount >= (config.AI_LLM_MAX_CONCURRENT ?? 5)) { + logger.warn( + { activeCount, maxConcurrent: config.AI_LLM_MAX_CONCURRENT }, + "LLM concurrency limit reached", + ); + } + + try { + return await fn(); + } finally { + activeCount--; + logger.debug( + { activeCount, pendingCount }, + "LLM request completed, concurrency slot released", + ); + } + }); } diff --git a/services/discord-gateway/src/modules/ai-moderation/conversationContext.ts b/services/discord-gateway/src/modules/ai-moderation/conversationContext.ts index b4476c9..5fba64e 100644 --- a/services/discord-gateway/src/modules/ai-moderation/conversationContext.ts +++ b/services/discord-gateway/src/modules/ai-moderation/conversationContext.ts @@ -1,7 +1,10 @@ +import { createChildLogger } from "@bete/shared/logger"; import { encoding_for_model as encodingForModel } from "tiktoken"; import { formatMediaEvidenceForPrompt } from "../message-capture/messageMetadata.js"; import type { MessageRecord } from "../message-capture/types.js"; +const logger = createChildLogger("conversationContext"); + export interface ConversationContextInput { contextBefore: MessageRecord[]; targets: MessageRecord[]; @@ -29,7 +32,12 @@ function formatTimestamp(ms: number): string { */ export function estimateTokens(text: string): number { // Use tiktoken for accurate token counting (+15 overhead for JSON structure) - return getEncoder().encode(text).length + 15; + const tokens = getEncoder().encode(text).length + 15; + logger.debug( + { tokenEstimate: tokens, textLength: text.length }, + "Estimated tokens for text", + ); + return tokens; } /** @@ -81,5 +89,14 @@ export function buildConversationContext( } } + logger.debug( + { + targetCount: targets.length, + contextCount: selectedContextLines.length, + usedTokens, + maxTokens, + }, + "Conversation context built", + ); return selectedContextLines; } diff --git a/services/discord-gateway/src/modules/ai-moderation/imageMimeSniffer.ts b/services/discord-gateway/src/modules/ai-moderation/imageMimeSniffer.ts new file mode 100644 index 0000000..b543c39 --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/imageMimeSniffer.ts @@ -0,0 +1,76 @@ +import { createChildLogger } from "@bete/shared/logger"; + +const log = createChildLogger("imageMimeSniffer"); + +/** + * Sniff the first bytes of a buffer to determine if it is a supported image + * format. Returns the canonical MIME type string on success, or null if the + * bytes are not a recognizable image. + */ +export function sniffImageMimeType(buf: Buffer): string | null { + if (buf.length < 12) return null; + + if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) { + return "image/jpeg"; + } + + if ( + buf[0] === 0x89 && + buf[1] === 0x50 && + buf[2] === 0x4e && + buf[3] === 0x47 && + buf[4] === 0x0d && + buf[5] === 0x0a && + buf[6] === 0x1a && + buf[7] === 0x0a + ) { + return "image/png"; + } + + if ( + buf[0] === 0x47 && + buf[1] === 0x49 && + buf[2] === 0x46 && + buf[3] === 0x38 + ) { + return "image/gif"; + } + + if ( + buf[0] === 0x52 && + buf[1] === 0x49 && + buf[2] === 0x46 && + buf[3] === 0x46 && + buf[8] === 0x57 && + buf[9] === 0x45 && + buf[10] === 0x42 && + buf[11] === 0x50 + ) { + return "image/webp"; + } + + if ( + buf.length >= 12 && + buf[4] === 0x66 && + buf[5] === 0x74 && + buf[6] === 0x79 && + buf[7] === 0x70 + ) { + const brand = buf.subarray(8, 12).toString("ascii"); + if (brand.startsWith("avif") || brand.startsWith("avis")) { + return "image/avif"; + } + if ( + brand.startsWith("mif1") || + brand.startsWith("heic") || + brand.startsWith("heis") + ) { + return "image/heic"; + } + } + + return null; +} + +// Keep log referenced so TS does not tree-shake the logger init +log.debug("imageMimeSniffer loaded"); diff --git a/services/discord-gateway/src/modules/ai-moderation/individualFallbackProcessor.ts b/services/discord-gateway/src/modules/ai-moderation/individualFallbackProcessor.ts new file mode 100644 index 0000000..184cd9d --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/individualFallbackProcessor.ts @@ -0,0 +1,346 @@ +import { createChildLogger } from "@bete/shared/logger"; +import { LRUCache } from "lru-cache"; +import { config } from "../../shared/config/config.js"; +import { updateMessagesAIAnalysisBulk } from "../message-capture/messageStore.js"; +import type { + AnalysisResult, + MessageRecord, +} from "../message-capture/types.js"; +import { + broadcastAnalysisCompleted, + fireAlert, + getConversationKey, + LAST_ERROR, + scheduleAutoDelete, + workerPool, +} from "./circuitBreaker.js"; +import { logModerationError } from "./responseLogger.js"; + +const logger = createChildLogger("individual-fallback"); + +// --------------------------------------------------------------------------- +// Individual fallback queue state +// --------------------------------------------------------------------------- + +/** IDs currently being processed one-by-one (LRU-backed, max 10k entries). */ +export const individualInFlight = new LRUCache({ max: 10000 }); + +/** + * Per-conversation count of in-flight individual messages. + * (LRU-backed to prevent unbounded growth) + */ +export const individualInFlightByConversation = new LRUCache({ + max: 10000, +}); + +/** Last-touched timestamp for pruning stale entries (LRU-backed). */ +export const individualInFlightLastTouched = new LRUCache({ + max: 10000, +}); + +/** Counter for observability. */ +export let activeIndividualRequests = 0; + +// --------------------------------------------------------------------------- +// Individual fallback circuit breaker (independent of batch CB) +// --------------------------------------------------------------------------- + +let individualConsecutiveErrors = 0; +export let individualCooldownUntil = 0; +const INDIVIDUAL_COOLDOWN_MS = 60000; + +// --------------------------------------------------------------------------- +// Individual fallback pipeline +// --------------------------------------------------------------------------- + +/** + * Processes a single message via the Piscina worker pool (offloaded from + * main thread to avoid blocking the event loop). + */ +async function processIndividualFallback( + message: MessageRecord, +): Promise { + const { id: messageId } = message; + const conversationKey = getConversationKey(message); + + activeIndividualRequests++; + individualInFlightByConversation.set( + conversationKey, + (individualInFlightByConversation.get(conversationKey) ?? 0) + 1, + ); + individualInFlightLastTouched.set(conversationKey, Date.now()); + + let exhaustedOnIncomplete = false; + + try { + // Run the LLM-heavy work in the worker thread + const workerResult = (await workerPool.run({ + type: "individual", + message, + skipNormalAnalysis: false, + } as unknown)) as + | { ok: true; results: AnalysisResult[] } + | { ok: false; results: AnalysisResult[]; error: string }; + + let analysisResult: { results: AnalysisResult[] } | null = null; + let usedSimpleFallback = false; + + if (workerResult.ok) { + const stillIncomplete = workerResult.results.some((r) => + r.flags.includes("analysis_incomplete"), + ); + if (stillIncomplete) { + exhaustedOnIncomplete = true; + analysisResult = null; + } else { + analysisResult = workerResult; + } + } + + // Step 2: If normal analysis failed, try SIMPLE fallback via worker + if (!analysisResult) { + logger.info( + { messageId }, + "Normal analysis failed -- trying simple text fallback via worker", + ); + + const simpleResult = (await workerPool.run({ + type: "individual", + message, + skipNormalAnalysis: true, + } as unknown)) as + | { ok: true; results: AnalysisResult[] } + | { ok: false; results: AnalysisResult[]; error: string }; + + if (simpleResult.ok) { + analysisResult = simpleResult; + usedSimpleFallback = true; + exhaustedOnIncomplete = false; + } + } + + if (!analysisResult) { + throw new Error( + `Both normal and simple analysis failed for message ${messageId}`, + ); + } + + if (usedSimpleFallback) { + logger.info( + { messageId, status: analysisResult.results[0]?.status }, + "Used simple text fallback for individual message (via worker)", + ); + } + + // Main thread: DB writes + broadcast + const updates = analysisResult.results.map((r) => ({ + messageId: r.messageId, + result: { + status: r.status, + flags: JSON.stringify(r.flags), + score: r.score, + analysis: r.analysis, + categories: r.categories, + severity: r.severity, + confidence: r.confidence, + recommendedAction: r.recommendedAction, + analyzedAt: Date.now(), + error: null, + }, + })); + + const rows = await updateMessagesAIAnalysisBulk(updates); + for (const row of rows) { + broadcastAnalysisCompleted(row); + scheduleAutoDelete(row); + + // Update reputation autonomously + if (row.ai_status === "clean") { + import("./userReputationStore.js") + .then((store) => store.recordCleanMessage(row.user_id, row.guild_id)) + .catch((e) => + logger.error( + { error: e }, + "Failed to record clean message streak in fallback", + ), + ); + } else if (row.ai_status === "flagged" && row.ai_severity !== "none") { + import("./userReputationStore.js") + .then((store) => + store.recordInfraction( + row.user_id, + row.guild_id, + row.ai_severity as "low" | "medium" | "high" | "critical", + ), + ) + .catch((e) => + logger.error( + { error: e }, + "Failed to record infraction penalty in fallback", + ), + ); + } + } + + const resultSummary = analysisResult.results[0]; + logModerationError([messageId], config.AI_LLM_MODEL, new Error("Success"), { + phase: "individual_fallback", + status: resultSummary?.status, + flags: resultSummary?.flags, + severity: resultSummary?.severity, + confidence: resultSummary?.confidence, + }); + + individualConsecutiveErrors = 0; + + logger.debug( + { messageId, status: analysisResult.results[0]?.status }, + "Individual fallback analysis complete (via worker)", + ); + } catch (error) { + individualConsecutiveErrors++; + if ( + individualConsecutiveErrors >= config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD + ) { + individualCooldownUntil = Date.now() + INDIVIDUAL_COOLDOWN_MS; + fireAlert({ + type: "individual_cb", + consecutiveErrors: individualConsecutiveErrors, + message: `Individual fallback circuit breaker triggered after ${individualConsecutiveErrors} consecutive errors`, + lastError: LAST_ERROR.value, + }); + } + + LAST_ERROR.value = error instanceof Error ? error.message : String(error); + + logModerationError( + [messageId], + config.AI_LLM_MODEL, + error as Error | string, + { + phase: "individual_fallback", + conversationKey, + exhaustedOnIncomplete, + }, + ); + + if (exhaustedOnIncomplete) { + await updateMessagesAIAnalysisBulk([ + { + messageId, + result: { + status: "error", + flags: JSON.stringify(["individual_analysis_exhausted"]), + score: 0, + analysis: + "Individual fallback exhausted all retries: LLM consistently dropped this message even in single-target mode", + categories: ["individual_analysis_exhausted"], + severity: "none", + confidence: 0, + recommendedAction: "review", + analyzedAt: Date.now(), + error: LAST_ERROR.value, + }, + }, + ]).catch((dbErr: unknown) => { + logger.error( + { messageId, error: String(dbErr) }, + "Failed to write terminal exhausted status", + ); + }); + logger.warn( + { messageId }, + "Individual fallback exhausted -- marked as individual_analysis_exhausted", + ); + } else { + logger.error( + { + messageId, + error: LAST_ERROR.value, + stack: error instanceof Error ? error.stack : undefined, + }, + "Individual fallback analysis failed (transient) -- will be retried", + ); + } + } finally { + activeIndividualRequests--; + individualInFlight.delete(messageId); + + const prev = individualInFlightByConversation.get(conversationKey) ?? 1; + if (prev <= 1) { + individualInFlightByConversation.delete(conversationKey); + individualInFlightLastTouched.delete(conversationKey); + } else { + individualInFlightByConversation.set(conversationKey, prev - 1); + individualInFlightLastTouched.set(conversationKey, Date.now()); + } + } +} + +// --------------------------------------------------------------------------- +// Enqueue individual fallbacks +// --------------------------------------------------------------------------- + +/** + * Fans out message records to the individual fallback queue. + * + * FIX #1: Checks concurrency cap before admitting new work. + * FIX #5: Checks individual circuit breaker before admitting new work. + */ +export function enqueueIndividualFallbacks(messages: MessageRecord[]): void { + // FIX #5: Honour the individual circuit breaker. + if (Date.now() < individualCooldownUntil) { + logger.warn( + { + until: new Date(individualCooldownUntil).toISOString(), + skipped: messages.length, + }, + "Individual fallback circuit breaker active -- messages will be recovered later", + ); + return; + } + + // FIX #5: Enforce concurrency cap + const maxConcurrent = config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT ?? 50; + const availableSlots = Math.max(0, maxConcurrent - activeIndividualRequests); + if (availableSlots <= 0) { + logger.debug( + { maxConcurrent, active: activeIndividualRequests }, + "Individual fallback concurrency cap reached -- messages will be recovered later", + ); + return; + } + + const newMessages = messages + .filter((m) => !individualInFlight.has(m.id)) + .slice(0, availableSlots); + if (newMessages.length === 0) return; + + logger.debug( + { + count: newMessages.length, + messageIds: newMessages.map((m) => m.id), + }, + "Enqueueing individual fallback analysis for batch-incomplete messages", + ); + + for (const msg of newMessages) { + individualInFlight.set(msg.id, true); + processIndividualFallback(msg).catch((err: unknown) => { + logger.error( + { messageId: msg.id, error: String(err) }, + "Unexpected uncaught error escaping processIndividualFallback", + ); + individualInFlight.delete(msg.id); + const ck = getConversationKey(msg); + const prev = individualInFlightByConversation.get(ck) ?? 1; + if (prev <= 1) { + individualInFlightByConversation.delete(ck); + individualInFlightLastTouched.delete(ck); + } else { + individualInFlightByConversation.set(ck, prev - 1); + individualInFlightLastTouched.set(ck, Date.now()); + } + }); + } +} diff --git a/services/discord-gateway/src/modules/ai-moderation/jsonExtractor.ts b/services/discord-gateway/src/modules/ai-moderation/jsonExtractor.ts new file mode 100644 index 0000000..999b744 --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/jsonExtractor.ts @@ -0,0 +1,81 @@ +import { createChildLogger } from "@bete/shared/logger"; + +const log = createChildLogger("jsonExtractor"); + +/** + * Helper to extract JSON from a potentially conversational or markdown-wrapped string. + */ +export function extractJson(content: string): unknown { + const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g; + const matches = content.matchAll(codeBlockRegex); + for (const match of matches) { + const codeContent = match[1].trim(); + try { + const parsed = JSON.parse(codeContent); + if (parsed && typeof parsed === "object") { + return parsed; + } + } catch (err) { + log.debug( + { err: err instanceof Error ? err.message : String(err) }, + "Failed to parse JSON from code block — trying next block", + ); + } + } + + for (let start = 0; start < content.length; start++) { + const firstChar = content[start]; + if (firstChar !== "{" && firstChar !== "[") continue; + + const stack = [firstChar]; + let inString = false; + let escaped = false; + + for (let i = start + 1; i < content.length; i++) { + const char = content[i]; + + if (inString) { + if (escaped) { + escaped = false; + } else if (char === "\\") { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + + if (char === '"') { + inString = true; + continue; + } + + if (char === "{" || char === "[") { + stack.push(char); + continue; + } + + const last = stack[stack.length - 1]; + if ((char === "}" && last === "{") || (char === "]" && last === "[")) { + stack.pop(); + if (stack.length === 0) { + const candidate = content.slice(start, i + 1); + try { + const parsed = JSON.parse(candidate); + if (parsed && typeof parsed === "object") { + return parsed; + } + } catch (err) { + log.debug( + { err: err instanceof Error ? err.message : String(err) }, + "Failed to parse JSON candidate — trying next position", + ); + } + break; + } + } + } + } + + throw new Error("No JSON object found in response"); +} diff --git a/services/discord-gateway/src/modules/ai-moderation/llmModerationClient.ts b/services/discord-gateway/src/modules/ai-moderation/llmModerationClient.ts index 712f692..bf71758 100644 --- a/services/discord-gateway/src/modules/ai-moderation/llmModerationClient.ts +++ b/services/discord-gateway/src/modules/ai-moderation/llmModerationClient.ts @@ -2,7 +2,6 @@ import { createChildLogger } from "@bete/shared/logger"; import { delay, retryWithBackoff } from "@bete/shared/utils"; import { LRUCache } from "lru-cache"; import type { ChatCompletion } from "openai/resources/chat/completions"; -import { z } from "zod"; import { config } from "../../shared/config/config.js"; import { resizeImageForVision } from "../attachment-upload/imageResizer.js"; import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js"; @@ -46,33 +45,31 @@ import { import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js"; import { initializeUserReputation } from "./userReputationStore.js"; -const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]); -const RecommendedActionSchema = z.enum([ - "none", - "monitor", - "warn", - "review", - "delete", - "escalate", -]); +export { sniffImageMimeType } from "./imageMimeSniffer.js"; +export { extractJson } from "./jsonExtractor.js"; +export { + parseModerationResponse, + sanitizeErrorMessage, +} from "./moderationResponseParser.js"; +// Re-export all symbols from sub-modules to preserve public API +export { + ModerationResponseSchema, + RecommendedActionSchema, + ResultItemSchema, + SeveritySchema, +} from "./moderationSchemas.js"; +export { + clampScore, + DEFERRAL_ANALYSIS_PATTERN, + DEFERRAL_EXCEPTION_PATTERN, + deriveRecommendedAction, + deriveSeverity, + hasDeferralAnalysis, +} from "./severityDeriver.js"; -const ResultItemSchema = z.object({ - message_id: z.union([z.string(), z.number()]).transform(String), - status: z.enum(["clean", "warn", "flagged"]), - flags: z.array(z.string()).optional(), - score: z.number(), - analysis: z.string().nullable().optional(), - categories: z.array(z.string()).optional(), - severity: SeveritySchema.optional(), - confidence: z.number().optional(), - recommended_action: RecommendedActionSchema.optional(), - policy_version: z.string().optional(), - evidence: z.array(z.string()).optional(), -}); - -const ModerationResponseSchema = z.object({ - results: z.array(ResultItemSchema), -}); +import { sniffImageMimeType } from "./imageMimeSniffer.js"; +// Internal imports for functions used locally in the facade +import { parseModerationResponse } from "./moderationResponseParser.js"; const log = createChildLogger("llmModerationClient"); @@ -112,371 +109,6 @@ async function buildCorrectedFewShotExamples(): Promise { } } -/** - * Enhanced deferral detection pattern (R9). - * - * Only matches patterns where the model explicitly states it cannot make - * a decision and needs human review. Removed overly broad patterns that - * caused false positives: - * - "admin (perlu|harus|sebaiknya)" → common in regular sentences - * - "bisa (berpotensi|mengandung)" → decisive statements, not deferral - * - "maaf|sorry" → opinions/apologies, not deferral - * - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral - */ -const DEFERRAL_ANALYSIS_PATTERN = - /(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i; - -/** - * Exceptions: patterns that look like deferral but are actually decisive. - * Expanded to catch more variations where the model gives a clear verdict. - */ -const DEFERRAL_EXCEPTION_PATTERN = - /tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i; - -function hasDeferralAnalysis(analysis: string): boolean { - if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false; - return DEFERRAL_ANALYSIS_PATTERN.test(analysis); -} - -function clampScore(value: number | undefined, fallback = 0): number { - return Math.max( - 0, - Math.min(1, Number.isFinite(value) ? (value as number) : fallback), - ); -} - -function deriveSeverity( - status: "clean" | "warn" | "flagged", - score: number, -): z.infer { - if (status === "clean") return "none"; - if (status === "warn") return score >= 0.65 ? "medium" : "low"; - if (score >= 0.9) return "critical"; - return score >= 0.75 ? "high" : "medium"; -} - -function deriveRecommendedAction( - status: "clean" | "warn" | "flagged", - severity: z.infer, -): z.infer { - if (status === "clean") return "none"; - if (status === "warn") return severity === "medium" ? "review" : "warn"; - if (severity === "critical") return "escalate"; - if (severity === "high") return "delete"; - return "review"; -} - -/** - * Helper to extract JSON from a potentially conversational or markdown-wrapped string. - */ -export function extractJson(content: string): unknown { - const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g; - const matches = content.matchAll(codeBlockRegex); - for (const match of matches) { - const codeContent = match[1].trim(); - try { - const parsed = JSON.parse(codeContent); - if (parsed && typeof parsed === "object") { - return parsed; - } - } catch (err) { - log.debug( - { err: err instanceof Error ? err.message : String(err) }, - "Failed to parse JSON from code block — trying next block", - ); - } - } - - for (let start = 0; start < content.length; start++) { - const firstChar = content[start]; - if (firstChar !== "{" && firstChar !== "[") continue; - - const stack = [firstChar]; - let inString = false; - let escaped = false; - - for (let i = start + 1; i < content.length; i++) { - const char = content[i]; - - if (inString) { - if (escaped) { - escaped = false; - } else if (char === "\\") { - escaped = true; - } else if (char === '"') { - inString = false; - } - continue; - } - - if (char === '"') { - inString = true; - continue; - } - - if (char === "{" || char === "[") { - stack.push(char); - continue; - } - - const last = stack[stack.length - 1]; - if ((char === "}" && last === "{") || (char === "]" && last === "[")) { - stack.pop(); - if (stack.length === 0) { - const candidate = content.slice(start, i + 1); - try { - const parsed = JSON.parse(candidate); - if (parsed && typeof parsed === "object") { - return parsed; - } - } catch (err) { - log.debug( - { err: err instanceof Error ? err.message : String(err) }, - "Failed to parse JSON candidate — trying next position", - ); - } - break; - } - } - } - } - - throw new Error("No JSON object found in response"); -} - -/** - * Sanitize error messages for client-facing output (R10). - * Internal details are logged but the caller gets a generic message. - */ -function sanitizeErrorMessage(internalMsg: string, messageId: string): string { - // Log the full error for debugging - log.warn( - { messageId, internalError: internalMsg }, - "Internal moderation error (sanitized for client)", - ); - // Return generic message without internal details - return `Analisis gagal dan memerlukan pemeriksaan manual. Error code: MOD_${Date.now().toString(36).slice(0, 6)}`; -} - -export function parseModerationResponse( - content: string, - targetIds: string[], -): AnalysisResult[] { - let parsed: any; - try { - parsed = JSON.parse(content); - } catch (e) { - parsed = extractJson(content); - } - - if (Array.isArray(parsed)) { - parsed = { results: parsed }; - } else if (parsed && typeof parsed === "object" && !("results" in parsed)) { - if ("message_id" in parsed) { - parsed = { results: [parsed] }; - } else { - const arrayKey = Object.keys(parsed).find((key) => { - const val = parsed[key]; - return ( - Array.isArray(val) && - val.length > 0 && - val.every( - (item: unknown) => - typeof item === "object" && - item !== null && - "message_id" in (item as Record), - ) - ); - }); - if (arrayKey) { - parsed.results = parsed[arrayKey]; - } else { - parsed = { results: [parsed] }; - } - } - } - - const parseResult = ModerationResponseSchema.safeParse(parsed); - if (!parseResult.success) { - throw new Error(`Zod validation failed: ${parseResult.error.message}`); - } - - const response = parseResult.data; - const foundIds = new Set(); - const targetIdSet = new Set(targetIds); - - const results: (AnalysisResult | null)[] = response.results.map((result) => { - const { - message_id, - status, - flags, - score, - analysis, - categories, - severity, - confidence, - recommended_action, - policy_version, - evidence, - } = result; - const finalId = message_id.trim(); - - if (!targetIdSet.has(finalId)) { - return null; - } - - if (foundIds.has(finalId)) { - throw new Error( - `Duplicate message_id in moderation response: ${finalId}`, - ); - } - - foundIds.add(finalId); - - const coalescedAnalysis = analysis ?? ""; - - if (hasDeferralAnalysis(coalescedAnalysis)) { - throw new Error( - `Deferral analysis is not allowed for message ${finalId}; return a direct moderation decision`, - ); - } - - const normalizedScore = clampScore(score); - const normalizedConfidence = clampScore(confidence, normalizedScore); - const normalizedSeverity = - severity ?? deriveSeverity(status, normalizedScore); - - return { - messageId: finalId, - status: status as "clean" | "warn" | "flagged", - flags: flags ?? [], - score: normalizedScore, - analysis: coalescedAnalysis, - categories: categories ?? flags ?? [], - severity: normalizedSeverity, - confidence: normalizedConfidence, - recommendedAction: - recommended_action ?? - deriveRecommendedAction(status, normalizedSeverity), - policyVersion: policy_version ?? "default-2026-05-30", - evidence: evidence ?? [], - }; - }); - - const filteredResults = results.filter( - (r): r is AnalysisResult => r !== null, - ); - - const missingIds = targetIds.filter((id) => !foundIds.has(id)); - if (missingIds.length > 0) { - log.warn( - { missingIds, foundCount: foundIds.size, totalCount: targetIds.length }, - "Some target IDs missing in response - marking as incomplete", - ); - for (const missingId of missingIds) { - filteredResults.push({ - messageId: missingId, - status: "error", - flags: ["analysis_incomplete"], - score: 0, - analysis: sanitizeErrorMessage( - "Analysis incomplete - LLM did not process this message", - missingId, - ), - categories: ["analysis_incomplete"], - severity: "none", - confidence: 0, - recommendedAction: "review", - policyVersion: "default-2026-05-30", - evidence: [], - }); - } - } - - return filteredResults; -} - -interface ModerationInput { - targets: MessageRecord[]; - contextText: string; - attachments?: AttachmentRecord[]; -} - -interface ModerationOutput { - results: AnalysisResult[]; - raw: unknown; -} - -/** - * Sniff the first bytes of a buffer to determine if it is a supported image - * format. Returns the canonical MIME type string on success, or null if the - * bytes are not a recognizable image. - */ -function sniffImageMimeType(buf: Buffer): string | null { - if (buf.length < 12) return null; - - if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) { - return "image/jpeg"; - } - - if ( - buf[0] === 0x89 && - buf[1] === 0x50 && - buf[2] === 0x4e && - buf[3] === 0x47 && - buf[4] === 0x0d && - buf[5] === 0x0a && - buf[6] === 0x1a && - buf[7] === 0x0a - ) { - return "image/png"; - } - - if ( - buf[0] === 0x47 && - buf[1] === 0x49 && - buf[2] === 0x46 && - buf[3] === 0x38 - ) { - return "image/gif"; - } - - if ( - buf[0] === 0x52 && - buf[1] === 0x49 && - buf[2] === 0x46 && - buf[3] === 0x46 && - buf[8] === 0x57 && - buf[9] === 0x45 && - buf[10] === 0x42 && - buf[11] === 0x50 - ) { - return "image/webp"; - } - - if ( - buf.length >= 12 && - buf[4] === 0x66 && - buf[5] === 0x74 && - buf[6] === 0x79 && - buf[7] === 0x70 - ) { - const brand = buf.subarray(8, 12).toString("ascii"); - if (brand.startsWith("avif") || brand.startsWith("avis")) { - return "image/avif"; - } - if ( - brand.startsWith("mif1") || - brand.startsWith("heic") || - brand.startsWith("heis") - ) { - return "image/heic"; - } - } - - return null; -} - // --------------------------------------------------------------------------- // Shared types for image resolution // --------------------------------------------------------------------------- @@ -1495,6 +1127,17 @@ async function runMediaBatch( // Main entry point — splits text-only vs media, runs both paths in parallel // --------------------------------------------------------------------------- +interface ModerationInput { + targets: MessageRecord[]; + contextText: string; + attachments?: AttachmentRecord[]; +} + +interface ModerationOutput { + results: AnalysisResult[]; + raw: unknown; +} + /** * Runs LLM-based moderation analysis on messages. * diff --git a/services/discord-gateway/src/modules/ai-moderation/moderationResponseParser.ts b/services/discord-gateway/src/modules/ai-moderation/moderationResponseParser.ts new file mode 100644 index 0000000..7d427a1 --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/moderationResponseParser.ts @@ -0,0 +1,178 @@ +import { createChildLogger } from "@bete/shared/logger"; +import type { AnalysisResult } from "../message-capture/types.js"; +import { extractJson } from "./jsonExtractor.js"; +import { ModerationResponseSchema } from "./moderationSchemas.js"; +import { + clampScore, + DEFERRAL_ANALYSIS_PATTERN, + DEFERRAL_EXCEPTION_PATTERN, + deriveRecommendedAction, + deriveSeverity, + hasDeferralAnalysis, +} from "./severityDeriver.js"; + +const log = createChildLogger("moderationResponseParser"); + +/** + * Re-export deferral patterns for backward compatibility. + * See severityDeriver.ts for the full regex definitions. + */ +export { + DEFERRAL_ANALYSIS_PATTERN, + DEFERRAL_EXCEPTION_PATTERN, +} from "./severityDeriver.js"; + +/** + * Sanitize error messages for client-facing output (R10). + * Internal details are logged but the caller gets a generic message. + */ +export function sanitizeErrorMessage( + internalMsg: string, + messageId: string, +): string { + // Log the full error for debugging + log.warn( + { messageId, internalError: internalMsg }, + "Internal moderation error (sanitized for client)", + ); + // Return generic message without internal details + return `Analisis gagal dan memerlukan pemeriksaan manual. Error code: MOD_${Date.now().toString(36).slice(0, 6)}`; +} + +export function parseModerationResponse( + content: string, + targetIds: string[], +): AnalysisResult[] { + let parsed: any; + try { + parsed = JSON.parse(content); + } catch (e) { + parsed = extractJson(content); + } + + if (Array.isArray(parsed)) { + parsed = { results: parsed }; + } else if (parsed && typeof parsed === "object" && !("results" in parsed)) { + if ("message_id" in parsed) { + parsed = { results: [parsed] }; + } else { + const arrayKey = Object.keys(parsed).find((key) => { + const val = parsed[key]; + return ( + Array.isArray(val) && + val.length > 0 && + val.every( + (item: unknown) => + typeof item === "object" && + item !== null && + "message_id" in (item as Record), + ) + ); + }); + if (arrayKey) { + parsed.results = parsed[arrayKey]; + } else { + parsed = { results: [parsed] }; + } + } + } + + const parseResult = ModerationResponseSchema.safeParse(parsed); + if (!parseResult.success) { + throw new Error(`Zod validation failed: ${parseResult.error.message}`); + } + + const response = parseResult.data; + const foundIds = new Set(); + const targetIdSet = new Set(targetIds); + + const results: (AnalysisResult | null)[] = response.results.map((result) => { + const { + message_id, + status, + flags, + score, + analysis, + categories, + severity, + confidence, + recommended_action, + policy_version, + evidence, + } = result; + const finalId = message_id.trim(); + + if (!targetIdSet.has(finalId)) { + return null; + } + + if (foundIds.has(finalId)) { + throw new Error( + `Duplicate message_id in moderation response: ${finalId}`, + ); + } + + foundIds.add(finalId); + + const coalescedAnalysis = analysis ?? ""; + + if (hasDeferralAnalysis(coalescedAnalysis)) { + throw new Error( + `Deferral analysis is not allowed for message ${finalId}; return a direct moderation decision`, + ); + } + + const normalizedScore = clampScore(score); + const normalizedConfidence = clampScore(confidence, normalizedScore); + const normalizedSeverity = + severity ?? deriveSeverity(status, normalizedScore); + + return { + messageId: finalId, + status: status as "clean" | "warn" | "flagged", + flags: flags ?? [], + score: normalizedScore, + analysis: coalescedAnalysis, + categories: categories ?? flags ?? [], + severity: normalizedSeverity, + confidence: normalizedConfidence, + recommendedAction: + recommended_action ?? + deriveRecommendedAction(status, normalizedSeverity), + policyVersion: policy_version ?? "default-2026-05-30", + evidence: evidence ?? [], + }; + }); + + const filteredResults = results.filter( + (r): r is AnalysisResult => r !== null, + ); + + const missingIds = targetIds.filter((id) => !foundIds.has(id)); + if (missingIds.length > 0) { + log.warn( + { missingIds, foundCount: foundIds.size, totalCount: targetIds.length }, + "Some target IDs missing in response - marking as incomplete", + ); + for (const missingId of missingIds) { + filteredResults.push({ + messageId: missingId, + status: "error", + flags: ["analysis_incomplete"], + score: 0, + analysis: sanitizeErrorMessage( + "Analysis incomplete - LLM did not process this message", + missingId, + ), + categories: ["analysis_incomplete"], + severity: "none", + confidence: 0, + recommendedAction: "review", + policyVersion: "default-2026-05-30", + evidence: [], + }); + } + } + + return filteredResults; +} diff --git a/services/discord-gateway/src/modules/ai-moderation/moderationSchemas.ts b/services/discord-gateway/src/modules/ai-moderation/moderationSchemas.ts new file mode 100644 index 0000000..a7b4bf4 --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/moderationSchemas.ts @@ -0,0 +1,41 @@ +import { createChildLogger } from "@bete/shared/logger"; +import { z } from "zod"; + +const log = createChildLogger("moderationSchemas"); + +export const SeveritySchema = z.enum([ + "none", + "low", + "medium", + "high", + "critical", +]); +export const RecommendedActionSchema = z.enum([ + "none", + "monitor", + "warn", + "review", + "delete", + "escalate", +]); + +export const ResultItemSchema = z.object({ + message_id: z.union([z.string(), z.number()]).transform(String), + status: z.enum(["clean", "warn", "flagged"]), + flags: z.array(z.string()).optional(), + score: z.number(), + analysis: z.string().nullable().optional(), + categories: z.array(z.string()).optional(), + severity: SeveritySchema.optional(), + confidence: z.number().optional(), + recommended_action: RecommendedActionSchema.optional(), + policy_version: z.string().optional(), + evidence: z.array(z.string()).optional(), +}); + +export const ModerationResponseSchema = z.object({ + results: z.array(ResultItemSchema), +}); + +// Keep log referenced so TS does not tree-shake the logger init +log.debug("moderationSchemas loaded"); diff --git a/services/discord-gateway/src/modules/ai-moderation/severityDeriver.ts b/services/discord-gateway/src/modules/ai-moderation/severityDeriver.ts new file mode 100644 index 0000000..c2d0e5d --- /dev/null +++ b/services/discord-gateway/src/modules/ai-moderation/severityDeriver.ts @@ -0,0 +1,64 @@ +import { createChildLogger } from "@bete/shared/logger"; +import type { z } from "zod"; +import { + RecommendedActionSchema, + SeveritySchema, +} from "./moderationSchemas.js"; + +const log = createChildLogger("severityDeriver"); + +/** + * Enhanced deferral detection pattern (R9). + * + * Only matches patterns where the model explicitly states it cannot make + * a decision and needs human review. Removed overly broad patterns that + * caused false positives: + * - "admin (perlu|harus|sebaiknya)" → common in regular sentences + * - "bisa (berpotensi|mengandung)" → decisive statements, not deferral + * - "maaf|sorry" → opinions/apologies, not deferral + * - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral + */ +export const DEFERRAL_ANALYSIS_PATTERN = + /(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i; + +/** + * Exceptions: patterns that look like deferral but are actually decisive. + * Expanded to catch more variations where the model gives a clear verdict. + */ +export const DEFERRAL_EXCEPTION_PATTERN = + /tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i; + +export function hasDeferralAnalysis(analysis: string): boolean { + if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false; + return DEFERRAL_ANALYSIS_PATTERN.test(analysis); +} + +export function clampScore(value: number | undefined, fallback = 0): number { + return Math.max( + 0, + Math.min(1, Number.isFinite(value) ? (value as number) : fallback), + ); +} + +export function deriveSeverity( + status: "clean" | "warn" | "flagged", + score: number, +): z.infer { + if (status === "clean") return "none"; + if (status === "warn") return score >= 0.65 ? "medium" : "low"; + if (score >= 0.9) return "critical"; + return score >= 0.75 ? "high" : "medium"; +} + +export function deriveRecommendedAction( + status: "clean" | "warn" | "flagged", + severity: z.infer, +): z.infer { + if (status === "clean") return "none"; + if (status === "warn") return severity === "medium" ? "review" : "warn"; + if (severity === "critical") return "escalate"; + if (severity === "high") return "delete"; + return "review"; +} + +log.debug("severityDeriver loaded"); diff --git a/services/discord-gateway/src/modules/ai-moderation/stickerPrompt.ts b/services/discord-gateway/src/modules/ai-moderation/stickerPrompt.ts index 33b859d..0fd233a 100644 --- a/services/discord-gateway/src/modules/ai-moderation/stickerPrompt.ts +++ b/services/discord-gateway/src/modules/ai-moderation/stickerPrompt.ts @@ -1,3 +1,7 @@ +import { createChildLogger } from "@bete/shared/logger"; + +const logger = createChildLogger("stickerPrompt"); + /** * Sticker-specific prompt templates for AI moderation. * @@ -17,6 +21,7 @@ export function buildStickerVisionPrompt( stickerName: string, messageId: string, ): string { + logger.debug({ stickerName, messageId }, "Building sticker vision prompt"); return [ `Analisis sticker Discord berikut sebagai evidence moderasi.`, `Sticker "${stickerName}" berasal dari pesan id=${messageId}.`, @@ -49,6 +54,10 @@ export function buildStickerTextOnlyWarning( stickerName: string, stickerUrl: string, ): string { + logger.debug( + { stickerName, stickerUrl }, + "Building sticker text-only warning", + ); return ( `[sticker: "${stickerName}" (${stickerUrl}) — GAMBAR GAGAL DIUNDUH. ` + `"${stickerName}" adalah sticker kartun/meme Discord. ` + @@ -68,6 +77,7 @@ export function buildCustomEmojiVisionPrompt( emojiName: string, messageId: string, ): string { + logger.debug({ emojiName, messageId }, "Building custom emoji vision prompt"); return [ `Analisis custom emoji Discord berikut sebagai evidence moderasi.`, `Emoji "${emojiName}" berasal dari pesan id=${messageId}.`, @@ -87,6 +97,7 @@ export function buildCustomEmojiVisionPrompt( * Fallback text for when a custom emoji image failed to download. */ export function buildCustomEmojiTextOnlyFallback(emojiName: string): string { + logger.debug({ emojiName }, "Building custom emoji text-only fallback"); return ( `[custom_emoji: "${emojiName}" — GAMBAR GAGAL DIUNDUH. ` + `"${emojiName}" adalah custom emoji Discord (ikon kecil). ` + @@ -105,6 +116,7 @@ export function buildGeneralImageVisionPrompt( sourceLabel: string, _messageId: string, ): string { + logger.debug({ sourceLabel }, "Building general image vision prompt"); return [ `Deskripsikan gambar ini secara objektif dan spesifik.`, `${sourceLabel}`, diff --git a/services/discord-gateway/src/modules/ai-moderation/userReputationStore.ts b/services/discord-gateway/src/modules/ai-moderation/userReputationStore.ts index c54d86a..c37067b 100644 --- a/services/discord-gateway/src/modules/ai-moderation/userReputationStore.ts +++ b/services/discord-gateway/src/modules/ai-moderation/userReputationStore.ts @@ -1,3 +1,4 @@ +import { createChildLogger } from "@bete/shared/logger"; import { and, desc, eq } from "drizzle-orm"; import { getDatabase } from "../../shared/database/drizzle.js"; import { @@ -6,6 +7,8 @@ import { userReputationsTable, } from "../../shared/database/schema.js"; +const logger = createChildLogger("userReputationStore"); + /** * Ensures a user reputation record exists. */ @@ -21,6 +24,7 @@ export async function initializeUserReputation( .limit(1); if (existing.length > 0) { + logger.debug({ userId }, "Reputation record already exists"); return existing[0]; } @@ -40,6 +44,7 @@ export async function initializeUserReputation( if (!inserted) { // If concurrent insert happened + logger.debug({ userId }, "Concurrent reputation insert detected, retrying"); const retry = await db .select() .from(userReputationsTable) @@ -48,6 +53,10 @@ export async function initializeUserReputation( return retry[0]; } + logger.debug( + { userId, trustScore: inserted.trust_score }, + "Initialized user reputation", + ); return inserted; } @@ -64,6 +73,14 @@ export async function getUserReputation( .where(eq(userReputationsTable.user_id, userId)) .limit(1); + if (existing[0]) { + logger.debug( + { userId, trustScore: existing[0].trust_score }, + "Fetched user reputation", + ); + } else { + logger.debug({ userId }, "No reputation record found, returning null"); + } return existing[0] || null; } @@ -93,6 +110,11 @@ export async function recordCleanMessage( updated_at: Date.now(), }) .where(eq(userReputationsTable.user_id, userId)); + + logger.debug( + { userId, previousScore: rep.trust_score, newScore, newStreak }, + "Clean message recorded, reputation updated", + ); } /** @@ -133,6 +155,17 @@ export async function recordInfraction( updated_at: Date.now(), }) .where(eq(userReputationsTable.user_id, userId)); + + logger.info( + { + userId, + severity, + penalty, + newScore, + totalInfractions: rep.total_infractions + 1, + }, + "Infraction recorded", + ); } /** diff --git a/services/discord-gateway/src/modules/message-capture/messageStore.ts b/services/discord-gateway/src/modules/message-capture/messageStore.ts index 7aa21a5..1557586 100644 --- a/services/discord-gateway/src/modules/message-capture/messageStore.ts +++ b/services/discord-gateway/src/modules/message-capture/messageStore.ts @@ -1,8 +1,8 @@ +import { decodeCursor, encodeCursor } from "@bete/shared"; import { createChildLogger, type Logger } from "@bete/shared/logger"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import { getDatabase } from "../../shared/database/drizzle.js"; import type * as schema from "../../shared/database/schema.js"; -import { decodeCursor, encodeCursor } from "../message-capture/pagination.js"; import type { AttachmentRecord, MessageQuery, @@ -18,7 +18,7 @@ import { ModerationActionsDb } from "./moderation-actions.db.js"; import { RetentionDb } from "./retention.db.js"; import { ReviewsDb } from "./reviews.db.js"; -export { decodeCursor, encodeCursor } from "../message-capture/pagination.js"; +export { decodeCursor, encodeCursor } from "@bete/shared"; export type { AIAnalysisUpdate } from "./messages.db.js"; // ─── Lazy singleton ──────────────────────────────────────────────────────── diff --git a/services/discord-gateway/src/modules/message-capture/messages.analysis.ts b/services/discord-gateway/src/modules/message-capture/messages.analysis.ts new file mode 100644 index 0000000..ccdf5a1 --- /dev/null +++ b/services/discord-gateway/src/modules/message-capture/messages.analysis.ts @@ -0,0 +1,413 @@ +import { createChildLogger, type Logger } from "@bete/shared/logger"; +import { + and, + asc, + desc, + eq, + inArray, + isNull, + or, + type SQL, + sql, +} from "drizzle-orm"; +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; +import type * as schema from "../../shared/database/schema.js"; +import { messagesTable } from "../../shared/database/schema.js"; +import type { MessageRecord } from "../message-capture/types.js"; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function stringifyAIList( + value: string[] | string | null | undefined, +): string | null { + if (value == null) return null; + return Array.isArray(value) ? JSON.stringify(value) : value; +} + +// ─── AIAnalysisUpdate interface ────────────────────────────────────────────── + +export interface AIAnalysisUpdate { + status: "pending" | "processing" | "clean" | "warn" | "flagged" | "error"; + flags?: string | null; + score?: number | null; + analysis?: string | null; + categories?: string[] | string | null; + severity?: MessageRecord["ai_severity"] | null; + confidence?: number | null; + recommendedAction?: MessageRecord["ai_recommended_action"] | null; + analyzedAt?: number | null; + error?: string | null; +} + +// ─── MessagesAnalysis Class ─────────────────────────────────────────────────── + +export class MessagesAnalysis { + protected logger: Logger; + + constructor( + protected db: NodePgDatabase, + _parentLogger?: Logger, + ) { + this.logger = createChildLogger("messages-analysis"); + } + + // ── AI Analysis Updates ───────────────────────────────────────────────────── + + async updateMessageAIAnalysis( + messageId: string, + result: AIAnalysisUpdate, + ): Promise { + this.logger.debug({ messageId }, "updateMessageAIAnalysis entry"); + try { + await this.db + .update(messagesTable) + .set({ + ai_status: result.status, + ai_moderation_flags: result.flags ?? null, + ai_moderation_score: result.score ?? null, + ai_analysis: result.analysis ?? null, + ai_categories: stringifyAIList(result.categories), + ai_severity: result.severity ?? null, + ai_confidence: result.confidence ?? result.score ?? null, + ai_recommended_action: result.recommendedAction ?? null, + ai_analyzed_at: result.analyzedAt ?? Date.now(), + ai_error: result.error ?? null, + }) + .where(eq(messagesTable.id, messageId)); + + const rows = await this.db + .select() + .from(messagesTable) + .where(eq(messagesTable.id, messageId)); + + return (rows[0] as MessageRecord) ?? null; + } catch (error) { + this.logger.error( + { + messageId, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to update message AI analysis", + ); + throw error; + } + } + + async updateMessagesAIAnalysisBulk( + updates: Array<{ messageId: string; result: AIAnalysisUpdate }>, + ): Promise { + this.logger.debug( + { count: updates.length }, + "updateMessagesAIAnalysisBulk entry", + ); + if (updates.length === 0) return []; + try { + const now = Date.now(); + + await this.db.transaction(async (tx) => { + for (const { messageId, result } of updates) { + await tx + .update(messagesTable) + .set({ + ai_status: result.status, + ai_moderation_flags: result.flags ?? null, + ai_moderation_score: result.score ?? null, + ai_analysis: result.analysis ?? null, + ai_categories: stringifyAIList(result.categories), + ai_severity: result.severity ?? null, + ai_confidence: result.confidence ?? result.score ?? null, + ai_recommended_action: result.recommendedAction ?? null, + ai_analyzed_at: result.analyzedAt ?? now, + ai_error: result.error ?? null, + }) + .where(eq(messagesTable.id, messageId)); + } + }); + + const ids = updates.map(({ messageId }) => messageId); + const rows = await this.db + .select() + .from(messagesTable) + .where(inArray(messagesTable.id, ids)); + + return rows as MessageRecord[]; + } catch (error) { + this.logger.error( + { + error: error instanceof Error ? error.message : String(error), + }, + "Failed to bulk update messages AI analysis", + ); + throw error; + } + } + + async getPendingAIAnalysisMessages( + limit: number = 25, + ): Promise { + this.logger.debug({ limit }, "getPendingAIAnalysisMessages entry"); + try { + const rows = await this.db + .select() + .from(messagesTable) + .where( + and( + eq(messagesTable.ai_status, "pending"), + isNull(messagesTable.deleted_at), + ), + ) + .orderBy(asc(messagesTable.created_at)) + .limit(limit); + + return rows as MessageRecord[]; + } catch (error) { + this.logger.error( + { error: error instanceof Error ? error.message : String(error) }, + "Failed to get pending AI analysis messages", + ); + throw error; + } + } + + // ── Conversation Context ──────────────────────────────────────────────────── + + async getConversationContextBefore(input: { + channelId: string; + threadId: string | null; + beforeCreatedAt: number; + limit: number; + }): Promise { + this.logger.debug( + { channelId: input.channelId, threadId: input.threadId }, + "getConversationContextBefore entry", + ); + try { + const { channelId, threadId, beforeCreatedAt, limit } = input; + + const locationCondition = threadId + ? eq(messagesTable.thread_id, threadId) + : eq(messagesTable.channel_id, channelId); + + const rows = await this.db + .select() + .from(messagesTable) + .where( + and( + locationCondition, + sql`${messagesTable.created_at} < ${beforeCreatedAt}`, + isNull(messagesTable.deleted_at), + ), + ) + .orderBy(desc(messagesTable.created_at)) + .limit(limit); + + return (rows as MessageRecord[]).reverse(); + } catch (error) { + this.logger.error( + { + channelId: input.channelId, + threadId: input.threadId, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to get conversation context before", + ); + throw error; + } + } + + async getPendingMessagesByConversation( + conversationKey: string, + limit: number = 200, + ): Promise { + this.logger.debug( + { conversationKey, limit }, + "getPendingMessagesByConversation entry", + ); + try { + const rows = await this.db.transaction(async (tx) => { + const pendingIdsQuery = tx + .select({ id: messagesTable.id }) + .from(messagesTable) + .where( + and( + or( + eq(messagesTable.thread_id, conversationKey), + eq(messagesTable.channel_id, conversationKey), + ), + eq(messagesTable.ai_status, "pending"), + isNull(messagesTable.deleted_at), + ), + ) + .orderBy(asc(messagesTable.created_at)) + .limit(limit) + .for("update", { skipLocked: true }); + + const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>; + + if (pendingIds.length === 0) return []; + + return await tx + .update(messagesTable) + .set({ ai_status: "processing", ai_analyzed_at: Date.now() }) + .where( + inArray( + messagesTable.id, + pendingIds.map((r) => r.id), + ), + ) + .returning(); + }); + + return rows as MessageRecord[]; + } catch (error) { + this.logger.error( + { + conversationKey, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to get pending messages by conversation", + ); + throw error; + } + } + + async getPendingConversationKeys(limit: number = 500): Promise { + this.logger.debug({ limit }, "getPendingConversationKeys entry"); + try { + const rows = (await this.db + .selectDistinct({ + thread_id: messagesTable.thread_id, + channel_id: messagesTable.channel_id, + }) + .from(messagesTable) + .where( + and( + eq(messagesTable.ai_status, "pending"), + isNull(messagesTable.deleted_at), + ), + ) + .limit(limit)) as Array<{ + thread_id: string | null; + channel_id: string; + }>; + + const keys: string[] = []; + for (const row of rows) { + const key = row.thread_id || row.channel_id; + if (key && !keys.includes(key)) { + keys.push(key); + } + } + + return keys; + } catch (error) { + this.logger.error( + { error: error instanceof Error ? error.message : String(error) }, + "Failed to get pending conversation keys", + ); + throw error; + } + } + + async getConversationKeysWithIncompleteAnalysis( + limit: number = 200, + ): Promise { + this.logger.debug( + { limit }, + "getConversationKeysWithIncompleteAnalysis entry", + ); + try { + const rows = (await this.db + .selectDistinct({ + thread_id: messagesTable.thread_id, + channel_id: messagesTable.channel_id, + }) + .from(messagesTable) + .where( + and( + eq(messagesTable.ai_status, "error"), + sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`, + sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`, + isNull(messagesTable.deleted_at), + ), + ) + .limit(limit)) as Array<{ + thread_id: string | null; + channel_id: string; + }>; + + const keys: string[] = []; + for (const row of rows) { + const key = row.thread_id || row.channel_id; + if (key && !keys.includes(key)) { + keys.push(key); + } + } + return keys; + } catch (error) { + this.logger.error( + { error: error instanceof Error ? error.message : String(error) }, + "Failed to get conversation keys with incomplete analysis", + ); + throw error; + } + } + + async getIncompleteMessagesByConversation( + conversationKey: string, + limit: number = 500, + ): Promise { + this.logger.debug( + { conversationKey, limit }, + "getIncompleteMessagesByConversation entry", + ); + try { + const rows = await this.db.transaction(async (tx) => { + const pendingIdsQuery = tx + .select({ id: messagesTable.id }) + .from(messagesTable) + .where( + and( + or( + eq(messagesTable.thread_id, conversationKey), + eq(messagesTable.channel_id, conversationKey), + ), + eq(messagesTable.ai_status, "error"), + sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`, + sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`, + isNull(messagesTable.deleted_at), + ), + ) + .orderBy(asc(messagesTable.created_at)) + .limit(limit) + .for("update", { skipLocked: true }); + + const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>; + + if (pendingIds.length === 0) return []; + + return await tx + .update(messagesTable) + .set({ ai_status: "processing", ai_analyzed_at: Date.now() }) + .where( + inArray( + messagesTable.id, + pendingIds.map((r) => r.id), + ), + ) + .returning(); + }); + + return rows as MessageRecord[]; + } catch (error) { + this.logger.error( + { + conversationKey, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to get incomplete messages by conversation", + ); + throw error; + } + } +} diff --git a/services/discord-gateway/src/modules/message-capture/messages.cleanup.ts b/services/discord-gateway/src/modules/message-capture/messages.cleanup.ts new file mode 100644 index 0000000..a432e28 --- /dev/null +++ b/services/discord-gateway/src/modules/message-capture/messages.cleanup.ts @@ -0,0 +1,86 @@ +import { createChildLogger, type Logger } from "@bete/shared/logger"; +import { and, eq, isNull, sql } from "drizzle-orm"; +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; +import type * as schema from "../../shared/database/schema.js"; +import { messagesTable } from "../../shared/database/schema.js"; +import type { MessageRecord } from "../message-capture/types.js"; + +// ─── MessagesCleanup Class ──────────────────────────────────────────────────── + +export class MessagesCleanup { + private logger: Logger; + + constructor( + private db: NodePgDatabase, + _parentLogger?: Logger, + ) { + this.logger = createChildLogger("messages-cleanup"); + } + + async getExpiredMessages(retentionDays: number): Promise { + this.logger.debug({ retentionDays }, "getExpiredMessages entry"); + try { + const cutoffTime = Date.now() - retentionDays * 24 * 60 * 60 * 1000; + + const rows = await this.db + .select() + .from(messagesTable) + .where( + and( + sql`${messagesTable.created_at} < ${cutoffTime}`, + isNull(messagesTable.deleted_at), + ), + ) + .limit(1000); + + return rows as MessageRecord[]; + } catch (error) { + this.logger.error( + { + retentionDays, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to get expired messages", + ); + throw error; + } + } + + async revertStuckProcessingMessages( + timeoutMs: number = 300000, + ): Promise { + this.logger.debug({ timeoutMs }, "revertStuckProcessingMessages entry"); + try { + const cutoffTime = Date.now() - timeoutMs; + + const rows = await this.db + .update(messagesTable) + .set({ ai_status: "pending", ai_analyzed_at: null }) + .where( + and( + eq(messagesTable.ai_status, "processing"), + sql`${messagesTable.ai_analyzed_at} < ${cutoffTime}`, + ), + ) + .returning({ id: messagesTable.id }); + + if (Array.isArray(rows) && rows.length > 0) { + this.logger.info( + { + count: rows.length, + messageIds: rows.map((r: { id: string }) => r.id), + }, + "Reverted stuck processing messages back to pending", + ); + } + + return Array.isArray(rows) ? rows.length : 0; + } catch (error) { + this.logger.error( + { error: error instanceof Error ? error.message : String(error) }, + "Failed to revert stuck processing messages", + ); + return 0; + } + } +} diff --git a/services/discord-gateway/src/modules/message-capture/messages.crud.ts b/services/discord-gateway/src/modules/message-capture/messages.crud.ts new file mode 100644 index 0000000..4097395 --- /dev/null +++ b/services/discord-gateway/src/modules/message-capture/messages.crud.ts @@ -0,0 +1,204 @@ +import { createChildLogger, type Logger } from "@bete/shared/logger"; +import { and, desc, eq, or, type SQL } from "drizzle-orm"; +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; +import type * as schema from "../../shared/database/schema.js"; +import { messagesTable } from "../../shared/database/schema.js"; +import type { MessageRecord } from "../message-capture/types.js"; + +// ─── Shared Helpers ────────────────────────────────────────────────────────── + +export function channelOrThreadCondition(channelId: string): SQL { + return or( + eq(messagesTable.channel_id, channelId), + eq(messagesTable.thread_id, channelId), + ) as SQL; +} + +// ─── MessagesCrud Class ────────────────────────────────────────────────────── + +export class MessagesCrud { + protected logger: Logger; + + constructor( + protected db: NodePgDatabase, + _parentLogger?: Logger, + ) { + this.logger = createChildLogger("messages-crud"); + } + + // ── INSERT ────────────────────────────────────────────────────────────────── + + async insertMessage(message: MessageRecord): Promise { + this.logger.debug({ messageId: message.id }, "insertMessage entry"); + try { + await this.db + .insert(messagesTable) + .values(message as any) + .onConflictDoNothing(); + } catch (error) { + this.logger.error( + { + messageId: message.id, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to insert message", + ); + throw error; + } + } + + async upsertMessageForCapture(message: MessageRecord): Promise { + this.logger.debug( + { messageId: message.id }, + "upsertMessageForCapture entry", + ); + try { + const messageWithAIStatus = { + ...message, + ai_status: "pending" as const, + }; + + const rows = await this.db + .insert(messagesTable) + .values(messageWithAIStatus as any) + .onConflictDoNothing() + .returning({ id: messagesTable.id }); + + return rows.length > 0; + } catch (error) { + this.logger.error( + { + messageId: message.id, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to upsert message for capture", + ); + throw error; + } + } + + // ── UPDATE ────────────────────────────────────────────────────────────────── + + async updateMessageAsEdited( + messageId: string, + editedContent: string, + editedAt: number, + ): Promise { + this.logger.debug({ messageId }, "updateMessageAsEdited entry"); + try { + await this.db + .update(messagesTable) + .set({ + edited_content: editedContent, + edited_at: editedAt, + type: "edited", + ai_status: "pending", + ai_moderation_flags: null, + ai_moderation_score: null, + ai_analysis: null, + ai_categories: null, + ai_severity: null, + ai_confidence: null, + ai_recommended_action: null, + ai_analyzed_at: null, + ai_error: null, + }) + .where(eq(messagesTable.id, messageId)); + } catch (error) { + this.logger.error( + { + messageId, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to update message as edited", + ); + throw error; + } + } + + async updateMessageAsDeleted( + messageId: string, + deletedAt: number, + ): Promise { + this.logger.debug({ messageId }, "updateMessageAsDeleted entry"); + try { + await this.db + .update(messagesTable) + .set({ + deleted_at: deletedAt, + type: "deleted", + }) + .where(eq(messagesTable.id, messageId)); + } catch (error) { + this.logger.error( + { + messageId, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to update message as deleted", + ); + throw error; + } + } + + // ── GET ───────────────────────────────────────────────────────────────────── + + async getMessagesByChannel( + channelId: string, + limit: number = 50, + offset: number = 0, + guildId?: string, + ): Promise { + this.logger.debug( + { channelId, limit, offset, guildId }, + "getMessagesByChannel entry", + ); + try { + const conditions: SQL[] = [channelOrThreadCondition(channelId)]; + + if (guildId) { + conditions.push(eq(messagesTable.guild_id, guildId)); + } + + const rows = await this.db + .select() + .from(messagesTable) + .where(and(...conditions)) + .orderBy(desc(messagesTable.created_at), desc(messagesTable.id)) + .limit(limit) + .offset(offset); + + return rows as MessageRecord[]; + } catch (error) { + this.logger.error( + { + channelId, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to get messages by channel", + ); + throw error; + } + } + + async getMessageById(messageId: string): Promise { + this.logger.debug({ messageId }, "getMessageById entry"); + try { + const rows = await this.db + .select() + .from(messagesTable) + .where(eq(messagesTable.id, messageId)); + + return (rows[0] as MessageRecord) ?? null; + } catch (error) { + this.logger.error( + { + messageId, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to get message by id", + ); + throw error; + } + } +} diff --git a/services/discord-gateway/src/modules/message-capture/messages.db.ts b/services/discord-gateway/src/modules/message-capture/messages.db.ts index 3c8df70..b9cddbb 100644 --- a/services/discord-gateway/src/modules/message-capture/messages.db.ts +++ b/services/discord-gateway/src/modules/message-capture/messages.db.ts @@ -1,826 +1,161 @@ import { createChildLogger, type Logger } from "@bete/shared/logger"; -import { - and, - asc, - desc, - eq, - inArray, - isNull, - or, - type SQL, - sql, -} from "drizzle-orm"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type * as schema from "../../shared/database/schema.js"; -import { messagesTable } from "../../shared/database/schema.js"; -import { decodeCursor, encodeCursor } from "../message-capture/pagination.js"; import type { MessageQuery, MessageRecord, PageResult, } from "../message-capture/types.js"; +import type { AIAnalysisUpdate } from "./messages.analysis.js"; +import { MessagesAnalysis } from "./messages.analysis.js"; +import { MessagesCleanup } from "./messages.cleanup.js"; +import { MessagesCrud } from "./messages.crud.js"; +import { MessagesPagination } from "./messages.pagination.js"; +import { MessagesSearch } from "./messages.search.js"; -// ─── Helpers ──────────────────────────────────────────────────────────────── +// Re-export AIAnalysisUpdate for consumers (messageStore.ts imports it) +export type { AIAnalysisUpdate } from "./messages.analysis.js"; -function channelOrThreadCondition(channelId: string): SQL { - return or( - eq(messagesTable.channel_id, channelId), - eq(messagesTable.thread_id, channelId), - ) as SQL; -} - -function buildListMessageConditions(query: MessageQuery): SQL[] { - const conditions: SQL[] = []; - - if (query.guildId) { - conditions.push(eq(messagesTable.guild_id, query.guildId)); - } - - if (query.channelId) { - conditions.push(channelOrThreadCondition(query.channelId)); - } - - if (query.threadId) { - conditions.push(eq(messagesTable.thread_id, query.threadId)); - } - - if (query.userId) { - conditions.push(eq(messagesTable.user_id, query.userId)); - } - - if (query.status && query.status.length > 0) { - conditions.push(sql`${messagesTable.ai_status} in ${query.status}`); - } - - if (query.q) { - const pattern = `%${query.q.toLowerCase()}%`; - conditions.push(sql`lower(${messagesTable.content}) like ${pattern}`); - } - - const cursorData = decodeCursor(query.cursor); - if (cursorData) { - conditions.push( - sql`(${messagesTable.created_at} < ${cursorData.created_at} or (${messagesTable.created_at} = ${cursorData.created_at} and ${messagesTable.id} < ${cursorData.id}))`, - ); - } - - return conditions; -} - -function pageRows( - rows: unknown[], - limit: number, -): PageResult { - const hasMore = rows.length > limit; - const data = rows.slice(0, limit) as T[]; - const lastItem = data[data.length - 1]; - const nextCursor = - hasMore && lastItem - ? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id }) - : null; - - return { data, nextCursor }; -} - -function pageMessages( - rows: unknown[], - limit: number, -): PageResult { - return pageRows(rows, limit); -} - -function stringifyAIList( - value: string[] | string | null | undefined, -): string | null { - if (value == null) return null; - return Array.isArray(value) ? JSON.stringify(value) : value; -} - -// ─── AIAnalysisUpdate interface ──────────────────────────────────────────── - -export interface AIAnalysisUpdate { - status: "pending" | "processing" | "clean" | "warn" | "flagged" | "error"; - flags?: string | null; - score?: number | null; - analysis?: string | null; - categories?: string[] | string | null; - severity?: MessageRecord["ai_severity"] | null; - confidence?: number | null; - recommendedAction?: MessageRecord["ai_recommended_action"] | null; - analyzedAt?: number | null; - error?: string | null; -} - -// ─── MessagesDb Class ────────────────────────────────────────────────────── +// ─── MessagesDb Facade ──────────────────────────────────────────────────────── +// Thin facade that delegates to domain-specific sub-modules. export class MessagesDb { - private logger: Logger; + private crud: MessagesCrud; + private analysis: MessagesAnalysis; + private search: MessagesSearch; + private pagination: MessagesPagination; + private cleanup: MessagesCleanup; - constructor( - private db: NodePgDatabase, - _parentLogger?: Logger, - ) { - this.logger = createChildLogger("messages-db"); + constructor(db: NodePgDatabase, _parentLogger?: Logger) { + const logger = _parentLogger ?? createChildLogger("messages-db"); + this.crud = new MessagesCrud(db, logger); + this.analysis = new MessagesAnalysis(db, logger); + this.search = new MessagesSearch(db, logger); + this.pagination = new MessagesPagination(db, logger); + this.cleanup = new MessagesCleanup(db, logger); } - // ── CRUD ────────────────────────────────────────────────────────────── + // ── CRUD ──────────────────────────────────────────────────────────────── - async insertMessage(message: MessageRecord): Promise { - this.logger.debug({ messageId: message.id }, "insertMessage entry"); - try { - await this.db - .insert(messagesTable) - .values(message as any) - .onConflictDoNothing(); - } catch (error) { - this.logger.error( - { - messageId: message.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to insert message", - ); - throw error; - } + insertMessage(message: MessageRecord): Promise { + return this.crud.insertMessage(message); } - async upsertMessageForCapture(message: MessageRecord): Promise { - this.logger.debug( - { messageId: message.id }, - "upsertMessageForCapture entry", - ); - try { - const messageWithAIStatus = { - ...message, - ai_status: "pending" as const, - }; - - const rows = await this.db - .insert(messagesTable) - .values(messageWithAIStatus as any) - .onConflictDoNothing() - .returning({ id: messagesTable.id }); - - return rows.length > 0; - } catch (error) { - this.logger.error( - { - messageId: message.id, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to upsert message for capture", - ); - throw error; - } + upsertMessageForCapture(message: MessageRecord): Promise { + return this.crud.upsertMessageForCapture(message); } - async updateMessageAsEdited( + updateMessageAsEdited( messageId: string, editedContent: string, editedAt: number, ): Promise { - this.logger.debug({ messageId }, "updateMessageAsEdited entry"); - try { - await this.db - .update(messagesTable) - .set({ - edited_content: editedContent, - edited_at: editedAt, - type: "edited", - ai_status: "pending", - ai_moderation_flags: null, - ai_moderation_score: null, - ai_analysis: null, - ai_categories: null, - ai_severity: null, - ai_confidence: null, - ai_recommended_action: null, - ai_analyzed_at: null, - ai_error: null, - }) - .where(eq(messagesTable.id, messageId)); - } catch (error) { - this.logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update message as edited", - ); - throw error; - } + return this.crud.updateMessageAsEdited(messageId, editedContent, editedAt); } - async updateMessageAsDeleted( - messageId: string, - deletedAt: number, - ): Promise { - this.logger.debug({ messageId }, "updateMessageAsDeleted entry"); - try { - await this.db - .update(messagesTable) - .set({ - deleted_at: deletedAt, - type: "deleted", - }) - .where(eq(messagesTable.id, messageId)); - } catch (error) { - this.logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update message as deleted", - ); - throw error; - } + updateMessageAsDeleted(messageId: string, deletedAt: number): Promise { + return this.crud.updateMessageAsDeleted(messageId, deletedAt); } - async getMessagesByChannel( + getMessagesByChannel( channelId: string, - limit: number = 50, - offset: number = 0, + limit?: number, + offset?: number, guildId?: string, ): Promise { - this.logger.debug( - { channelId, limit, offset, guildId }, - "getMessagesByChannel entry", - ); - try { - const conditions: SQL[] = [ - or( - eq(messagesTable.channel_id, channelId), - eq(messagesTable.thread_id, channelId), - ) as SQL, - ]; - - if (guildId) { - conditions.push(eq(messagesTable.guild_id, guildId)); - } - - const rows = await this.db - .select() - .from(messagesTable) - .where(and(...conditions)) - .orderBy(desc(messagesTable.created_at), desc(messagesTable.id)) - .limit(limit) - .offset(offset); - - return rows as MessageRecord[]; - } catch (error) { - this.logger.error( - { - channelId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get messages by channel", - ); - throw error; - } + return this.crud.getMessagesByChannel(channelId, limit, offset, guildId); } - async getMessageById(messageId: string): Promise { - this.logger.debug({ messageId }, "getMessageById entry"); - try { - const rows = await this.db - .select() - .from(messagesTable) - .where(eq(messagesTable.id, messageId)); - - return (rows[0] as MessageRecord) ?? null; - } catch (error) { - this.logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get message by id", - ); - throw error; - } + getMessageById(messageId: string): Promise { + return this.crud.getMessageById(messageId); } - // ── AI Analysis ─────────────────────────────────────────────────────── + // ── AI Analysis ───────────────────────────────────────────────────────── - async updateMessageAIAnalysis( + updateMessageAIAnalysis( messageId: string, result: AIAnalysisUpdate, ): Promise { - this.logger.debug({ messageId }, "updateMessageAIAnalysis entry"); - try { - await this.db - .update(messagesTable) - .set({ - ai_status: result.status, - ai_moderation_flags: result.flags ?? null, - ai_moderation_score: result.score ?? null, - ai_analysis: result.analysis ?? null, - ai_categories: stringifyAIList(result.categories), - ai_severity: result.severity ?? null, - ai_confidence: result.confidence ?? result.score ?? null, - ai_recommended_action: result.recommendedAction ?? null, - ai_analyzed_at: result.analyzedAt ?? Date.now(), - ai_error: result.error ?? null, - }) - .where(eq(messagesTable.id, messageId)); - - const rows = await this.db - .select() - .from(messagesTable) - .where(eq(messagesTable.id, messageId)); - - return (rows[0] as MessageRecord) ?? null; - } catch (error) { - this.logger.error( - { - messageId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to update message AI analysis", - ); - throw error; - } + return this.analysis.updateMessageAIAnalysis(messageId, result); } - async updateMessagesAIAnalysisBulk( + updateMessagesAIAnalysisBulk( updates: Array<{ messageId: string; result: AIAnalysisUpdate }>, ): Promise { - this.logger.debug( - { count: updates.length }, - "updateMessagesAIAnalysisBulk entry", - ); - if (updates.length === 0) return []; - try { - const now = Date.now(); - - await this.db.transaction(async (tx) => { - for (const { messageId, result } of updates) { - await tx - .update(messagesTable) - .set({ - ai_status: result.status, - ai_moderation_flags: result.flags ?? null, - ai_moderation_score: result.score ?? null, - ai_analysis: result.analysis ?? null, - ai_categories: stringifyAIList(result.categories), - ai_severity: result.severity ?? null, - ai_confidence: result.confidence ?? result.score ?? null, - ai_recommended_action: result.recommendedAction ?? null, - ai_analyzed_at: result.analyzedAt ?? now, - ai_error: result.error ?? null, - }) - .where(eq(messagesTable.id, messageId)); - } - }); - - const ids = updates.map(({ messageId }) => messageId); - const rows = await this.db - .select() - .from(messagesTable) - .where(inArray(messagesTable.id, ids)); - - return rows as MessageRecord[]; - } catch (error) { - this.logger.error( - { - error: error instanceof Error ? error.message : String(error), - }, - "Failed to bulk update messages AI analysis", - ); - throw error; - } + return this.analysis.updateMessagesAIAnalysisBulk(updates); } - async getPendingAIAnalysisMessages( - limit: number = 25, - ): Promise { - this.logger.debug({ limit }, "getPendingAIAnalysisMessages entry"); - try { - const rows = await this.db - .select() - .from(messagesTable) - .where( - and( - eq(messagesTable.ai_status, "pending"), - isNull(messagesTable.deleted_at), - ), - ) - .orderBy(asc(messagesTable.created_at)) - .limit(limit); - - return rows as MessageRecord[]; - } catch (error) { - this.logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get pending AI analysis messages", - ); - throw error; - } + getPendingAIAnalysisMessages(limit?: number): Promise { + return this.analysis.getPendingAIAnalysisMessages(limit); } - // ── Listing / Pagination ────────────────────────────────────────────── - - async listMessages(query: MessageQuery): Promise> { - this.logger.debug({ query }, "listMessages entry"); - try { - const conditions = buildListMessageConditions(query); - const rows = await this.db - .select() - .from(messagesTable) - .where(conditions.length > 0 ? and(...conditions) : undefined) - .orderBy(desc(messagesTable.created_at), desc(messagesTable.id)) - .limit(query.limit + 1); - - return pageMessages(rows, query.limit); - } catch (error) { - this.logger.error( - { - query, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to list messages", - ); - throw error; - } - } - - async listReviewMessages( - query: Omit, - ): Promise> { - return this.listMessages({ - ...query, - status: ["warn", "flagged", "error"], - }); - } - - // ── Conversation Context ────────────────────────────────────────────── - - async getConversationContextBefore(input: { + getConversationContextBefore(input: { channelId: string; threadId: string | null; beforeCreatedAt: number; limit: number; }): Promise { - this.logger.debug( - { channelId: input.channelId, threadId: input.threadId }, - "getConversationContextBefore entry", - ); - try { - const { channelId, threadId, beforeCreatedAt, limit } = input; - - const locationCondition = threadId - ? eq(messagesTable.thread_id, threadId) - : eq(messagesTable.channel_id, channelId); - - const rows = await this.db - .select() - .from(messagesTable) - .where( - and( - locationCondition, - sql`${messagesTable.created_at} < ${beforeCreatedAt}`, - isNull(messagesTable.deleted_at), - ), - ) - .orderBy(desc(messagesTable.created_at)) - .limit(limit); - - return (rows as MessageRecord[]).reverse(); - } catch (error) { - this.logger.error( - { - channelId: input.channelId, - threadId: input.threadId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get conversation context before", - ); - throw error; - } + return this.analysis.getConversationContextBefore(input); } - async getPendingMessagesByConversation( + getPendingMessagesByConversation( conversationKey: string, - limit: number = 200, + limit?: number, ): Promise { - this.logger.debug( - { conversationKey, limit }, - "getPendingMessagesByConversation entry", + return this.analysis.getPendingMessagesByConversation( + conversationKey, + limit, ); - try { - const rows = await this.db.transaction(async (tx) => { - const pendingIdsQuery = tx - .select({ id: messagesTable.id }) - .from(messagesTable) - .where( - and( - or( - eq(messagesTable.thread_id, conversationKey), - eq(messagesTable.channel_id, conversationKey), - ), - eq(messagesTable.ai_status, "pending"), - isNull(messagesTable.deleted_at), - ), - ) - .orderBy(asc(messagesTable.created_at)) - .limit(limit) - .for("update", { skipLocked: true }); - - const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>; - - if (pendingIds.length === 0) return []; - - return await tx - .update(messagesTable) - .set({ ai_status: "processing", ai_analyzed_at: Date.now() }) - .where( - inArray( - messagesTable.id, - pendingIds.map((r) => r.id), - ), - ) - .returning(); - }); - - return rows as MessageRecord[]; - } catch (error) { - this.logger.error( - { - conversationKey, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get pending messages by conversation", - ); - throw error; - } } - // ── Conversation Keys ───────────────────────────────────────────────── - - async getPendingConversationKeys(limit: number = 500): Promise { - this.logger.debug({ limit }, "getPendingConversationKeys entry"); - try { - const rows = (await this.db - .selectDistinct({ - thread_id: messagesTable.thread_id, - channel_id: messagesTable.channel_id, - }) - .from(messagesTable) - .where( - and( - eq(messagesTable.ai_status, "pending"), - isNull(messagesTable.deleted_at), - ), - ) - .limit(limit)) as Array<{ - thread_id: string | null; - channel_id: string; - }>; - - const keys: string[] = []; - for (const row of rows) { - const key = row.thread_id || row.channel_id; - if (key && !keys.includes(key)) { - keys.push(key); - } - } - - return keys; - } catch (error) { - this.logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get pending conversation keys", - ); - throw error; - } + getPendingConversationKeys(limit?: number): Promise { + return this.analysis.getPendingConversationKeys(limit); } - async getConversationKeysWithIncompleteAnalysis( - limit: number = 200, - ): Promise { - this.logger.debug( - { limit }, - "getConversationKeysWithIncompleteAnalysis entry", - ); - try { - const rows = (await this.db - .selectDistinct({ - thread_id: messagesTable.thread_id, - channel_id: messagesTable.channel_id, - }) - .from(messagesTable) - .where( - and( - eq(messagesTable.ai_status, "error"), - sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`, - sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`, - isNull(messagesTable.deleted_at), - ), - ) - .limit(limit)) as Array<{ - thread_id: string | null; - channel_id: string; - }>; - - const keys: string[] = []; - for (const row of rows) { - const key = row.thread_id || row.channel_id; - if (key && !keys.includes(key)) { - keys.push(key); - } - } - return keys; - } catch (error) { - this.logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to get conversation keys with incomplete analysis", - ); - throw error; - } + getConversationKeysWithIncompleteAnalysis(limit?: number): Promise { + return this.analysis.getConversationKeysWithIncompleteAnalysis(limit); } - async getIncompleteMessagesByConversation( + getIncompleteMessagesByConversation( conversationKey: string, - limit: number = 500, + limit?: number, ): Promise { - this.logger.debug( - { conversationKey, limit }, - "getIncompleteMessagesByConversation entry", + return this.analysis.getIncompleteMessagesByConversation( + conversationKey, + limit, ); - try { - const rows = await this.db.transaction(async (tx) => { - const pendingIdsQuery = tx - .select({ id: messagesTable.id }) - .from(messagesTable) - .where( - and( - or( - eq(messagesTable.thread_id, conversationKey), - eq(messagesTable.channel_id, conversationKey), - ), - eq(messagesTable.ai_status, "error"), - sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`, - sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`, - isNull(messagesTable.deleted_at), - ), - ) - .orderBy(asc(messagesTable.created_at)) - .limit(limit) - .for("update", { skipLocked: true }); - - const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>; - - if (pendingIds.length === 0) return []; - - return await tx - .update(messagesTable) - .set({ ai_status: "processing", ai_analyzed_at: Date.now() }) - .where( - inArray( - messagesTable.id, - pendingIds.map((r) => r.id), - ), - ) - .returning(); - }); - - return rows as MessageRecord[]; - } catch (error) { - this.logger.error( - { - conversationKey, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get incomplete messages by conversation", - ); - throw error; - } } - // ── Search ──────────────────────────────────────────────────────────── + // ── Search ────────────────────────────────────────────────────────────── - async searchMessages(input: { + searchMessages(input: { query: string; channelId?: string; guildId?: string; limit?: number; }): Promise { - this.logger.debug({ query: input.query }, "searchMessages entry"); - try { - const { query, channelId, guildId, limit = 20 } = input; - - const searchPattern = `%${query}%`; - const conditions: (SQL | undefined)[] = [ - isNull(messagesTable.deleted_at), - ]; - - if (guildId) { - conditions.push(eq(messagesTable.guild_id, guildId)); - } - - if (channelId) { - conditions.push(channelOrThreadCondition(channelId)); - } - - conditions.push( - or( - sql`${messagesTable.content} LIKE ${searchPattern}`, - sql`${messagesTable.edited_content} LIKE ${searchPattern}`, - ), - ); - - const validConditions = conditions.filter( - (c): c is SQL => c !== undefined, - ); - - const rows = await this.db - .select() - .from(messagesTable) - .where(and(...validConditions)) - .orderBy(desc(messagesTable.created_at)) - .limit(limit); - - return rows as MessageRecord[]; - } catch (error) { - this.logger.error( - { - query: input.query, - channelId: input.channelId, - guildId: input.guildId, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to search messages", - ); - throw error; - } + return this.search.searchMessages(input); } - // ── Retention / Recovery ────────────────────────────────────────────── + // ── Pagination ────────────────────────────────────────────────────────── - async getExpiredMessages(retentionDays: number): Promise { - this.logger.debug({ retentionDays }, "getExpiredMessages entry"); - try { - const cutoffTime = Date.now() - retentionDays * 24 * 60 * 60 * 1000; - - const rows = await this.db - .select() - .from(messagesTable) - .where( - and( - sql`${messagesTable.created_at} < ${cutoffTime}`, - isNull(messagesTable.deleted_at), - ), - ) - .limit(1000); - - return rows as MessageRecord[]; - } catch (error) { - this.logger.error( - { - retentionDays, - error: error instanceof Error ? error.message : String(error), - }, - "Failed to get expired messages", - ); - throw error; - } + listMessages(query: MessageQuery): Promise> { + return this.pagination.listMessages(query); } - async revertStuckProcessingMessages( - timeoutMs: number = 300000, - ): Promise { - this.logger.debug({ timeoutMs }, "revertStuckProcessingMessages entry"); - try { - const cutoffTime = Date.now() - timeoutMs; + listReviewMessages( + query: Omit, + ): Promise> { + return this.pagination.listReviewMessages(query); + } - const rows = await this.db - .update(messagesTable) - .set({ ai_status: "pending", ai_analyzed_at: null }) - .where( - and( - eq(messagesTable.ai_status, "processing"), - sql`${messagesTable.ai_analyzed_at} < ${cutoffTime}`, - ), - ) - .returning({ id: messagesTable.id }); + // ── Cleanup ───────────────────────────────────────────────────────────── - if (Array.isArray(rows) && rows.length > 0) { - this.logger.info( - { - count: rows.length, - messageIds: rows.map((r: { id: string }) => r.id), - }, - "Reverted stuck processing messages back to pending", - ); - } + getExpiredMessages(retentionDays: number): Promise { + return this.cleanup.getExpiredMessages(retentionDays); + } - return Array.isArray(rows) ? rows.length : 0; - } catch (error) { - this.logger.error( - { error: error instanceof Error ? error.message : String(error) }, - "Failed to revert stuck processing messages", - ); - return 0; - } + revertStuckProcessingMessages(timeoutMs?: number): Promise { + return this.cleanup.revertStuckProcessingMessages(timeoutMs); } } diff --git a/services/discord-gateway/src/modules/message-capture/messages.pagination.ts b/services/discord-gateway/src/modules/message-capture/messages.pagination.ts new file mode 100644 index 0000000..70d1908 --- /dev/null +++ b/services/discord-gateway/src/modules/message-capture/messages.pagination.ts @@ -0,0 +1,100 @@ +import { decodeCursor, encodeCursor, pageResult } from "@bete/shared"; +import { createChildLogger, type Logger } from "@bete/shared/logger"; +import { and, desc, eq, type SQL, sql } from "drizzle-orm"; +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; +import type * as schema from "../../shared/database/schema.js"; +import { messagesTable } from "../../shared/database/schema.js"; +import type { + MessageQuery, + MessageRecord, + PageResult, +} from "../message-capture/types.js"; +import { channelOrThreadCondition } from "./messages.crud.js"; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +export function buildListMessageConditions(query: MessageQuery): SQL[] { + const conditions: SQL[] = []; + + if (query.guildId) { + conditions.push(eq(messagesTable.guild_id, query.guildId)); + } + + if (query.channelId) { + conditions.push(channelOrThreadCondition(query.channelId)); + } + + if (query.threadId) { + conditions.push(eq(messagesTable.thread_id, query.threadId)); + } + + if (query.userId) { + conditions.push(eq(messagesTable.user_id, query.userId)); + } + + if (query.status && query.status.length > 0) { + conditions.push(sql`${messagesTable.ai_status} in ${query.status}`); + } + + if (query.q) { + const pattern = `%${query.q.toLowerCase()}%`; + conditions.push(sql`lower(${messagesTable.content}) like ${pattern}`); + } + + const cursorData = decodeCursor(query.cursor); + if (cursorData) { + conditions.push( + sql`(${messagesTable.created_at} < ${cursorData.created_at} or (${messagesTable.created_at} = ${cursorData.created_at} and ${messagesTable.id} < ${cursorData.id}))`, + ); + } + + return conditions; +} + +const pageRows = pageResult; + +// ─── MessagesPagination Class ──────────────────────────────────────────────── + +export class MessagesPagination { + private logger: Logger; + + constructor( + private db: NodePgDatabase, + _parentLogger?: Logger, + ) { + this.logger = createChildLogger("messages-pagination"); + } + + async listMessages(query: MessageQuery): Promise> { + this.logger.debug({ query }, "listMessages entry"); + try { + const conditions = buildListMessageConditions(query); + const rows = await this.db + .select() + .from(messagesTable) + .where(conditions.length > 0 ? and(...conditions) : undefined) + .orderBy(desc(messagesTable.created_at), desc(messagesTable.id)) + .limit(query.limit + 1); + + return pageRows(rows, query.limit); + } catch (error) { + this.logger.error( + { + query, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to list messages", + ); + throw error; + } + } + + async listReviewMessages( + query: Omit, + ): Promise> { + return this.listMessages({ + ...query, + status: ["warn", "flagged", "error"], + }); + } +} diff --git a/services/discord-gateway/src/modules/message-capture/messages.search.ts b/services/discord-gateway/src/modules/message-capture/messages.search.ts new file mode 100644 index 0000000..e66eef7 --- /dev/null +++ b/services/discord-gateway/src/modules/message-capture/messages.search.ts @@ -0,0 +1,76 @@ +import { createChildLogger, type Logger } from "@bete/shared/logger"; +import { and, desc, eq, isNull, or, type SQL, sql } from "drizzle-orm"; +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; +import type * as schema from "../../shared/database/schema.js"; +import { messagesTable } from "../../shared/database/schema.js"; +import type { MessageRecord } from "../message-capture/types.js"; +import { channelOrThreadCondition } from "./messages.crud.js"; + +// ─── MessagesSearch Class ───────────────────────────────────────────────────── + +export class MessagesSearch { + private logger: Logger; + + constructor( + private db: NodePgDatabase, + _parentLogger?: Logger, + ) { + this.logger = createChildLogger("messages-search"); + } + + async searchMessages(input: { + query: string; + channelId?: string; + guildId?: string; + limit?: number; + }): Promise { + this.logger.debug({ query: input.query }, "searchMessages entry"); + try { + const { query, channelId, guildId, limit = 20 } = input; + + const searchPattern = `%${query}%`; + const conditions: (SQL | undefined)[] = [ + isNull(messagesTable.deleted_at), + ]; + + if (guildId) { + conditions.push(eq(messagesTable.guild_id, guildId)); + } + + if (channelId) { + conditions.push(channelOrThreadCondition(channelId)); + } + + conditions.push( + or( + sql`${messagesTable.content} LIKE ${searchPattern}`, + sql`${messagesTable.edited_content} LIKE ${searchPattern}`, + ), + ); + + const validConditions = conditions.filter( + (c): c is SQL => c !== undefined, + ); + + const rows = await this.db + .select() + .from(messagesTable) + .where(and(...validConditions)) + .orderBy(desc(messagesTable.created_at)) + .limit(limit); + + return rows as MessageRecord[]; + } catch (error) { + this.logger.error( + { + query: input.query, + channelId: input.channelId, + guildId: input.guildId, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to search messages", + ); + throw error; + } + } +} diff --git a/services/discord-gateway/src/modules/message-capture/moderation-actions.db.ts b/services/discord-gateway/src/modules/message-capture/moderation-actions.db.ts index 1b17cc4..878312f 100644 --- a/services/discord-gateway/src/modules/message-capture/moderation-actions.db.ts +++ b/services/discord-gateway/src/modules/message-capture/moderation-actions.db.ts @@ -1,28 +1,11 @@ +import { decodeCursor, encodeCursor, pageResult } from "@bete/shared"; import { createChildLogger, type Logger } from "@bete/shared/logger"; import { and, desc, eq, type SQL, sql } from "drizzle-orm"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type * as schema from "../../shared/database/schema.js"; import { moderationActionsTable } from "../../shared/database/schema.js"; -import { decodeCursor, encodeCursor } from "../message-capture/pagination.js"; import type { ModerationAction, PageResult } from "../message-capture/types.js"; -// ─── Helpers ──────────────────────────────────────────────────────────────── - -function pageRows( - rows: unknown[], - limit: number, -): PageResult { - const hasMore = rows.length > limit; - const data = rows.slice(0, limit) as T[]; - const lastItem = data[data.length - 1]; - const nextCursor = - hasMore && lastItem - ? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id }) - : null; - - return { data, nextCursor }; -} - // ─── ModerationActionsDb Class ────────────────────────────────────────────── export class ModerationActionsDb { @@ -126,7 +109,7 @@ export class ModerationActionsDb { ) .limit(limit + 1); - return pageRows(rows, limit); + return pageResult(rows, limit); } catch (error) { this.logger.error( { error: error instanceof Error ? error.message : String(error) }, diff --git a/services/discord-gateway/src/modules/message-capture/pagination.ts b/services/discord-gateway/src/modules/message-capture/pagination.ts index d0949aa..b5ec9df 100644 --- a/services/discord-gateway/src/modules/message-capture/pagination.ts +++ b/services/discord-gateway/src/modules/message-capture/pagination.ts @@ -1,21 +1,36 @@ +import { createChildLogger } from "@bete/shared/logger"; + +const logger = createChildLogger("pagination"); + export interface CursorData { created_at: number; id: string; } export function encodeCursor(data: CursorData): string { - return Buffer.from(JSON.stringify(data)).toString("base64"); + const encoded = Buffer.from(JSON.stringify(data)).toString("base64"); + logger.debug({ id: data.id, createdAt: data.created_at }, "Encoded cursor"); + return encoded; } export function decodeCursor(cursor?: string): CursorData | null { - if (!cursor) return null; + if (!cursor) { + logger.debug("No cursor provided to decode"); + return null; + } try { const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8")); if (typeof data.created_at === "number" && typeof data.id === "string") { + logger.debug( + { id: data.id, createdAt: data.created_at }, + "Decoded cursor", + ); return data; } + logger.warn({ cursor }, "Decoded cursor has invalid shape"); return null; - } catch { + } catch (err) { + logger.warn({ cursor, error: String(err) }, "Failed to decode cursor"); return null; } } diff --git a/services/discord-gateway/src/modules/message-capture/reviews.db.ts b/services/discord-gateway/src/modules/message-capture/reviews.db.ts index ee7f527..6bc215a 100644 --- a/services/discord-gateway/src/modules/message-capture/reviews.db.ts +++ b/services/discord-gateway/src/modules/message-capture/reviews.db.ts @@ -1,28 +1,11 @@ +import { decodeCursor, encodeCursor, pageResult } from "@bete/shared"; import { createChildLogger, type Logger } from "@bete/shared/logger"; import { and, desc, eq, type SQL, sql } from "drizzle-orm"; import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type * as schema from "../../shared/database/schema.js"; import { messageReviewsTable } from "../../shared/database/schema.js"; -import { decodeCursor, encodeCursor } from "../message-capture/pagination.js"; import type { MessageReview, PageResult } from "../message-capture/types.js"; -// ─── Helpers ──────────────────────────────────────────────────────────────── - -function pageRows( - rows: unknown[], - limit: number, -): PageResult { - const hasMore = rows.length > limit; - const data = rows.slice(0, limit) as T[]; - const lastItem = data[data.length - 1]; - const nextCursor = - hasMore && lastItem - ? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id }) - : null; - - return { data, nextCursor }; -} - // ─── ReviewsDb Class ──────────────────────────────────────────────────────── export class ReviewsDb { @@ -128,7 +111,7 @@ export class ReviewsDb { ) .limit(limit + 1); - return pageRows(rows, limit); + return pageResult(rows, limit); } catch (error) { this.logger.error( { error: error instanceof Error ? error.message : String(error) }, diff --git a/services/discord-gateway/src/modules/voice-recording/ffmpegProcess.ts b/services/discord-gateway/src/modules/voice-recording/ffmpegProcess.ts index fc5f1ed..932ab7e 100644 --- a/services/discord-gateway/src/modules/voice-recording/ffmpegProcess.ts +++ b/services/discord-gateway/src/modules/voice-recording/ffmpegProcess.ts @@ -1,4 +1,7 @@ import { spawn } from "node:child_process"; +import { createChildLogger } from "@bete/shared/logger"; + +const logger = createChildLogger("ffmpeg-process"); export interface MuxFfmpegArgsOptions { inputs: string[]; @@ -42,19 +45,24 @@ export function buildMuxFfmpegArgs(options: MuxFfmpegArgsOptions): string[] { */ export function runFfmpeg(args: string[]): Promise { return new Promise((resolve, reject) => { + logger.debug({ args }, "Starting ffmpeg"); + const proc = spawn("ffmpeg", args, { stdio: ["ignore", "inherit", "inherit"], }); proc.on("close", (code) => { if (code === 0) { + logger.debug("ffmpeg completed successfully"); resolve(); } else { + logger.warn({ exitCode: code }, "ffmpeg exited with non-zero code"); reject(new Error(`ffmpeg exited with code ${code}`)); } }); proc.on("error", (err) => { + logger.error({ error: err.message }, "ffmpeg process error"); reject(err); }); }); diff --git a/services/discord-gateway/src/modules/voice-recording/packetFilter.ts b/services/discord-gateway/src/modules/voice-recording/packetFilter.ts index 59d2902..ecd12ec 100644 --- a/services/discord-gateway/src/modules/voice-recording/packetFilter.ts +++ b/services/discord-gateway/src/modules/voice-recording/packetFilter.ts @@ -1,4 +1,7 @@ import { Transform, TransformCallback } from "node:stream"; +import { createChildLogger } from "@bete/shared/logger"; + +const logger = createChildLogger("packet-filter"); /** * Transform stream to filter out audio packets that are too small. @@ -31,6 +34,11 @@ export class PacketFilter extends Transform { callback(); } + /** Returns the number of packets filtered out and processed. */ + getStats(): { filtered: number; total: number } { + return { filtered: this.filteredCount, total: this.totalCount }; + } + _flush(callback: TransformCallback): void { callback(); } diff --git a/services/discord-gateway/src/modules/voice-recording/recorder/audioStream.ts b/services/discord-gateway/src/modules/voice-recording/recorder/audioStream.ts index 2d1e12c..43f06d3 100644 --- a/services/discord-gateway/src/modules/voice-recording/recorder/audioStream.ts +++ b/services/discord-gateway/src/modules/voice-recording/recorder/audioStream.ts @@ -1,6 +1,9 @@ +import { createChildLogger } from "@bete/shared/logger"; import { EndBehaviorType, type VoiceReceiver } from "@discordjs/voice"; import { config } from "../../../shared/config/config.js"; +const logger = createChildLogger("audio-stream"); + export interface AudioStreamHandlers { onPacket: (chunk: Buffer) => void; onEnd: () => void; @@ -12,6 +15,8 @@ export function subscribeToAudioStream( userId: string, handlers: AudioStreamHandlers, ): NodeJS.ReadableStream { + logger.debug({ userId }, "Subscribing to audio stream"); + const audioStream = receiver.subscribe(userId, { end: { behavior: EndBehaviorType.AfterSilence, @@ -20,8 +25,14 @@ export function subscribeToAudioStream( }); audioStream.on("data", handlers.onPacket); - audioStream.on("end", handlers.onEnd); - audioStream.on("error", handlers.onError); + audioStream.on("end", () => { + logger.debug({ userId }, "Audio stream ended"); + handlers.onEnd(); + }); + audioStream.on("error", (error: Error) => { + logger.warn({ userId, error: error.message }, "Audio stream error"); + handlers.onError(error); + }); return audioStream; } diff --git a/services/discord-gateway/src/modules/voice-recording/recorder/metadata.ts b/services/discord-gateway/src/modules/voice-recording/recorder/metadata.ts index fefa261..315cccd 100644 --- a/services/discord-gateway/src/modules/voice-recording/recorder/metadata.ts +++ b/services/discord-gateway/src/modules/voice-recording/recorder/metadata.ts @@ -1,6 +1,10 @@ import path from "node:path"; +import { createChildLogger } from "@bete/shared/logger"; import type { Client, VoiceChannel } from "discord.js-selfbot-v13"; import { config } from "../../../shared/config/config.js"; + +const logger = createChildLogger("voice-metadata"); + import type { SegmentMetadata, SegmentState, @@ -12,12 +16,20 @@ export async function collectUserMetadata( userId: string, channel: VoiceChannel, ): Promise { + logger.debug({ userId }, "Collecting user metadata"); + const user = client.users.cache.get(userId) || - (await client.users.fetch(userId).catch(() => null)); + (await client.users.fetch(userId).catch(() => { + logger.warn({ userId }, "Failed to fetch user"); + return null; + })); const member = channel.guild.members.cache.get(userId) || - (await channel.guild.members.fetch(userId).catch(() => null)); + (await channel.guild.members.fetch(userId).catch(() => { + logger.warn({ userId }, "Failed to fetch guild member"); + return null; + })); const username = user?.username ?? "Unknown User"; const roles = member?.roles.cache diff --git a/services/discord-gateway/src/modules/voice-recording/recorder/segment.ts b/services/discord-gateway/src/modules/voice-recording/recorder/segment.ts index 519d2a6..bcf4845 100644 --- a/services/discord-gateway/src/modules/voice-recording/recorder/segment.ts +++ b/services/discord-gateway/src/modules/voice-recording/recorder/segment.ts @@ -1,8 +1,11 @@ import fs from "node:fs"; import path from "node:path"; +import { createChildLogger } from "@bete/shared/logger"; import * as prism from "prism-media"; import type { SegmentState } from "../../message-capture/types.js"; +const logger = createChildLogger("segment"); + export function buildSegmentPaths( userDir: string, startTime: number, @@ -54,6 +57,11 @@ export class SegmentManager { oggStream, out, }; + + logger.debug( + { index, startTime, filename, userDir: this.userDir }, + "Segment opened", + ); return this.currentSegment; } @@ -64,6 +72,25 @@ export class SegmentManager { oggPacketStream.unpipe(segment.oggStream); segment.oggStream.end(); this.currentSegment = null; + + // Get file size after closing + let fileSize = 0; + try { + const stat = fs.statSync(segment.filename); + fileSize = stat.size; + } catch { + // File might not exist yet + } + + logger.debug( + { + index: segment.index, + filename: segment.filename, + fileSize, + durationMs: (segment.endTime ?? 0) - segment.startTime, + }, + "Segment closed", + ); return segment; } @@ -77,6 +104,15 @@ export class SegmentManager { ) ) return null; + + logger.debug( + { + index: this.currentSegment.index, + filename: this.currentSegment.filename, + durationMs: Date.now() - this.currentSegment.startTime, + }, + "Segment rotating", + ); this.close(oggPacketStream); return this.open(oggPacketStream); } diff --git a/services/discord-gateway/src/modules/voice-recording/recorder/sessionRecording.ts b/services/discord-gateway/src/modules/voice-recording/recorder/sessionRecording.ts index fb6c6e7..39729d3 100644 --- a/services/discord-gateway/src/modules/voice-recording/recorder/sessionRecording.ts +++ b/services/discord-gateway/src/modules/voice-recording/recorder/sessionRecording.ts @@ -1,11 +1,14 @@ import fs, { promises as fsPromises } from "node:fs"; import path from "node:path"; +import { createChildLogger } from "@bete/shared/logger"; import type { UserMetadata } from "../../message-capture/types.js"; import { buildMuxFfmpegArgs, runFfmpeg as defaultRunFfmpeg, } from "../ffmpegProcess.js"; +const logger = createChildLogger("recording-session"); + export type SessionRecordingStatus = | "pending" | "completed" @@ -86,6 +89,16 @@ export function createRecordingSession( const participants = new Map(); const segments: SessionSegmentRef[] = []; + logger.info( + { + sessionId, + guildId: options.guildId, + channelId: options.channelId, + channelName: options.channelName, + }, + "Recording session created", + ); + return { sessionId, recordingsDir: options.recordingsDir, @@ -108,6 +121,10 @@ export function createRecordingSession( durationMs: input.endTime - input.startTime, offsetMs: input.startTime - options.startTime, }); + logger.debug( + { sessionId, userId: input.user.userId, segmentCount: segments.length }, + "Segment registered in session", + ); }, snapshot(endTime: number): SessionRecordingMetadata { @@ -132,6 +149,11 @@ export function buildSessionMuxFilter( segments: Array<{ startTime: number }>, sessionStartTime: number, ): string { + if (segments.length === 0) { + logger.debug("Building mux filter with no segments"); + return ""; + } + const filters = segments.map((segment, index) => { const delayMs = Math.max(0, segment.startTime - sessionStartTime); return `[${index}:a]adelay=${delayMs}|${delayMs}[pad${index}]`; @@ -140,6 +162,11 @@ export function buildSessionMuxFilter( filters.push( `${inputs}amix=inputs=${segments.length}:dropout_transition=0[out]`, ); + + logger.debug( + { segmentCount: segments.length, filter: filters.join(";") }, + "Built mux filter", + ); return filters.join(";"); } @@ -166,26 +193,66 @@ export async function finalizeRecordingSession( await mkdir(sessionDir); const metadata = session.snapshot(endTime); + logger.info( + { + sessionId: session.sessionId, + segmentCount: metadata.segments.length, + outputFile, + }, + "Finalizing recording session", + ); + if (metadata.segments.length === 0) { await writeJson(metadataFile, { ...metadata, status: "empty" }); + logger.info( + { sessionId: session.sessionId }, + "Recording session finalized with no segments", + ); return; } try { - await runFfmpeg( - buildMuxFfmpegArgs({ - inputs: metadata.segments.map((segment) => segment.oggPath), - filter: buildSessionMuxFilter(metadata.segments, metadata.startTime), - output: outputFile, - codec: "libopus", - }), + const ffmpegArgs = buildMuxFfmpegArgs({ + inputs: metadata.segments.map((segment) => segment.oggPath), + filter: buildSessionMuxFilter(metadata.segments, metadata.startTime), + output: outputFile, + codec: "libopus", + }); + + logger.debug( + { sessionId: session.sessionId, ffmpegArgs }, + "Running FFmpeg mux for session", ); + + await runFfmpeg(ffmpegArgs); + + // Get output file size + let outputSize = 0; + try { + const outStat = await fsPromises.stat(outputFile); + outputSize = outStat.size; + } catch { + // File might not exist yet, ignore + } + await writeJson(metadataFile, { ...metadata, status: "completed", outputFile, }); + + logger.info( + { sessionId: session.sessionId, outputFile, outputSize }, + "Recording session finalized successfully", + ); } catch (error) { + logger.error( + { + sessionId: session.sessionId, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to finalize recording session via FFmpeg", + ); await writeJson(metadataFile, { ...metadata, status: "failed", diff --git a/services/discord-gateway/src/modules/voice-recording/teleUpload.ts b/services/discord-gateway/src/modules/voice-recording/teleUpload.ts index 633ad50..1ba1dca 100644 --- a/services/discord-gateway/src/modules/voice-recording/teleUpload.ts +++ b/services/discord-gateway/src/modules/voice-recording/teleUpload.ts @@ -1,5 +1,8 @@ +import { createChildLogger } from "@bete/shared/logger"; import { retryWithBackoff } from "@bete/shared/utils"; +const logger = createChildLogger("tele-upload"); + export interface TeleUploadResponse { download_url: string; public_id?: string; @@ -40,6 +43,8 @@ export async function uploadToTele(input: { const { buffer, filename, contentType, uploadUrl, timeoutMs, retries } = input; + logger.debug({ filename, uploadUrl }, "Starting tele upload"); + const response = await retryWithBackoff( async () => { const fileBlob = new Blob([new Uint8Array(buffer)], { diff --git a/services/discord-gateway/src/shared/database/schema.ts b/services/discord-gateway/src/shared/database/schema.ts index 8ce7a81..4ad7984 100644 --- a/services/discord-gateway/src/shared/database/schema.ts +++ b/services/discord-gateway/src/shared/database/schema.ts @@ -1,3 +1,4 @@ +import { pgAttachmentsTable, pgMessagesTable } from "@bete/shared"; import { bigint as pgBigint, boolean as pgBoolean, @@ -5,7 +6,6 @@ import { index as pgIndex, integer as pgInteger, jsonb as pgJsonb, - real as pgReal, pgTable, text as pgText, timestamp as pgTimestamp, @@ -41,137 +41,7 @@ export const pgMuxerJobsTable = pgTable( }), ); -/** - * Messages Table (PostgreSQL) - * Stores text messages with AI moderation analysis - */ -export const pgMessagesTable = pgTable( - "messages", - { - id: pgText("id").primaryKey(), - guild_id: pgText("guild_id").notNull(), - channel_id: pgText("channel_id").notNull(), - thread_id: pgText("thread_id"), - user_id: pgText("user_id").notNull(), - username: pgText("username").notNull(), - avatar_url: pgText("avatar_url"), - content: pgText("content").notNull(), - edited_content: pgText("edited_content"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - edited_at: pgBigint("edited_at", { mode: "number" }), - deleted_at: pgBigint("deleted_at", { mode: "number" }), - type: pgText("type", { enum: ["text", "edited", "deleted"] }) - .notNull() - .default("text"), - metadata: pgText("metadata"), - ai_status: pgText("ai_status", { - enum: ["pending", "processing", "clean", "warn", "flagged", "error"], - }) - .notNull() - .default("pending"), - ai_moderation_flags: pgText("ai_moderation_flags"), - ai_moderation_score: pgReal("ai_moderation_score"), - ai_analysis: pgText("ai_analysis"), - ai_categories: pgText("ai_categories"), - ai_severity: pgText("ai_severity", { - enum: ["none", "low", "medium", "high", "critical"], - }), - ai_confidence: pgReal("ai_confidence"), - ai_recommended_action: pgText("ai_recommended_action", { - enum: ["none", "monitor", "warn", "review", "delete", "escalate"], - }), - ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }), - ai_error: pgText("ai_error"), - }, - (table) => ({ - channelIdx: pgIndex("idx_messages_channel").on(table.channel_id), - userIdx: pgIndex("idx_messages_user").on(table.user_id), - createdIdx: pgIndex("idx_messages_created").on(table.created_at), - threadIdx: pgIndex("idx_messages_thread").on(table.thread_id), - channelCreatedIdx: pgIndex("idx_messages_channel_created").on( - table.channel_id, - table.created_at, - table.id, - ), - threadCreatedIdx: pgIndex("idx_messages_thread_created").on( - table.thread_id, - table.created_at, - table.id, - ), - aiStatusCreatedIdx: pgIndex("idx_messages_ai_status_created").on( - table.ai_status, - table.created_at, - table.id, - ), - guildAiStatusCreatedIdx: pgIndex("idx_messages_guild_ai_status_created").on( - table.guild_id, - table.ai_status, - table.created_at, - table.id, - ), - guildCreatedDeletedIdx: pgIndex("idx_messages_guild_created_deleted").on( - table.guild_id, - table.created_at, - table.deleted_at, - table.id, - ), - channelAiStatusCreatedIdx: pgIndex( - "idx_messages_channel_ai_status_created", - ).on(table.channel_id, table.ai_status, table.created_at, table.id), - threadAiStatusCreatedIdx: pgIndex( - "idx_messages_thread_ai_status_created", - ).on(table.thread_id, table.ai_status, table.created_at, table.id), - }), -); - -/** - * Attachments Table (PostgreSQL) - * Stores attachment metadata with upload status tracking - */ -export const pgAttachmentsTable = pgTable( - "attachments", - { - id: pgText("id").primaryKey(), - message_id: pgText("message_id").notNull(), - guild_id: pgText("guild_id").notNull(), - channel_id: pgText("channel_id").notNull(), - thread_id: pgText("thread_id"), - user_id: pgText("user_id").notNull(), - filename: pgText("filename").notNull(), - size: pgInteger("size").notNull(), - type: pgText("type").notNull(), - discord_url: pgText("discord_url").notNull(), - uploaded_url: pgText("uploaded_url"), - upload_status: pgText("upload_status", { - enum: ["pending", "uploaded", "failed"], - }) - .notNull() - .default("pending"), - upload_error: pgText("upload_error"), - created_at: pgBigint("created_at", { mode: "number" }).notNull(), - uploaded_at: pgBigint("uploaded_at", { mode: "number" }), - }, - (table) => ({ - channelIdx: pgIndex("idx_attachments_channel").on(table.channel_id), - messageIdx: pgIndex("idx_attachments_message").on(table.message_id), - statusIdx: pgIndex("idx_attachments_status").on(table.upload_status), - channelCreatedIdx: pgIndex("idx_attachments_channel_created").on( - table.channel_id, - table.created_at, - table.id, - ), - threadCreatedIdx: pgIndex("idx_attachments_thread_created").on( - table.thread_id, - table.created_at, - table.id, - ), - messageFk: pgForeignKey({ - columns: [table.message_id], - foreignColumns: [pgMessagesTable.id], - name: "fk_attachments_message_id", - }).onDelete("cascade"), - }), -); +// (pgMessagesTable and pgAttachmentsTable are imported from @bete/shared) /** * UI State Table (PostgreSQL) diff --git a/services/frontend/src/features/live/hooks/useMediaControl.ts b/services/frontend/src/features/live/hooks/useMediaControl.ts index e96560c..2ce7008 100644 --- a/services/frontend/src/features/live/hooks/useMediaControl.ts +++ b/services/frontend/src/features/live/hooks/useMediaControl.ts @@ -7,6 +7,10 @@ import { skipMedia, stopMedia, } from "../../../shared/api/client"; +import { useAsyncAction } from "../../../shared/hooks/useAsyncAction.js"; +import { createLogger } from "../../../shared/lib/logger.js"; + +const logger = createLogger("use-media-control"); const emptyMediaState: MediaState = { playing: false, @@ -17,8 +21,7 @@ const emptyMediaState: MediaState = { export function useMediaControl() { const [mediaState, setMediaState] = useState(emptyMediaState); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); + const { loading, error, execute, clearError } = useAsyncAction(); const refreshMedia = useCallback(async () => { const state = await getMediaStatus(); @@ -28,71 +31,62 @@ export function useMediaControl() { const enqueue = useCallback( async (source: string, mode: "music" | "screen") => { - setLoading(true); - setError(null); - try { - const state = await queueMedia(source, mode); - setMediaState(state); - return state; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - setError(message); - throw err; - } finally { - setLoading(false); + const result = await execute(() => queueMedia(source, mode)); + if (result) { + setMediaState(result); + logger.info("Media queued", { source, mode }); + } else { + logger.error("Failed to queue media", { source, mode }); } + return result; }, - [], + [execute], ); const skip = useCallback(async () => { - setLoading(true); - setError(null); - try { - const state = await skipMedia(); - setMediaState(state); - return state; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - setError(message); - throw err; - } finally { - setLoading(false); + const result = await execute(() => skipMedia()); + if (result) { + setMediaState(result); + logger.info("Media skipped"); + } else { + logger.error("Failed to skip media"); } - }, []); + return result; + }, [execute]); const stop = useCallback(async () => { - setLoading(true); - setError(null); - try { - const state = await stopMedia(); - setMediaState(state); - return state; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - setError(message); - throw err; - } finally { - setLoading(false); + const result = await execute(() => stopMedia()); + if (result) { + setMediaState(result); + logger.info("Media stopped"); + } else { + logger.error("Failed to stop media"); } - }, []); + return result; + }, [execute]); - const setVolume = useCallback(async (volume: number) => { - setError(null); - try { - const state = await setMediaVolume(volume); - setMediaState(state); - return state; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - setError(message); - throw err; - } - }, []); + const setVolume = useCallback( + async (volume: number) => { + clearError(); + try { + const state = await setMediaVolume(volume); + setMediaState(state); + logger.info("Volume set", { volume }); + return state; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + logger.error("Failed to set volume", { volume, error: message }); + throw err; + } + }, + [clearError], + ); useEffect(() => { refreshMedia().catch((err) => - setError(err instanceof Error ? err.message : String(err)), + logger.error("Failed to refresh media state on mount", { + error: String(err), + }), ); }, [refreshMedia]); diff --git a/services/frontend/src/features/live/hooks/useVoiceControl.ts b/services/frontend/src/features/live/hooks/useVoiceControl.ts index 5a76c63..f3728d5 100644 --- a/services/frontend/src/features/live/hooks/useVoiceControl.ts +++ b/services/frontend/src/features/live/hooks/useVoiceControl.ts @@ -8,6 +8,10 @@ import { getVoiceChannels, getVoiceStatus, } from "../../../shared/api/client"; +import { useAsyncAction } from "../../../shared/hooks/useAsyncAction.js"; +import { createLogger } from "../../../shared/lib/logger.js"; + +const logger = createLogger("use-voice-control"); export function useVoiceControl() { const [guilds, setGuilds] = useState([]); @@ -19,15 +23,14 @@ export function useVoiceControl() { activeChannelId: null, activeChannelName: null, }); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); + const { loading, error, execute, clearError } = useAsyncAction(); const refreshGuilds = useCallback(async () => { - setError(null); + clearError(); const nextGuilds = await getGuilds(); setGuilds(nextGuilds); return nextGuilds; - }, []); + }, [clearError]); const refreshVoiceStatus = useCallback(async () => { const status = await getVoiceStatus(); @@ -55,44 +58,39 @@ export function useVoiceControl() { return channels; }, []); - const joinVoice = useCallback(async (guildId: string, channelId: string) => { - setLoading(true); - setError(null); - try { - const status = await connectVoice(guildId, channelId); - setVoiceStatus(status); - return status; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - setError(message); - throw err; - } finally { - setLoading(false); - } - }, []); + const joinVoice = useCallback( + async (guildId: string, channelId: string) => { + const result = await execute(() => connectVoice(guildId, channelId)); + if (result) { + setVoiceStatus(result); + logger.info("Connected to voice", { guildId, channelId }); + } else { + logger.error("Failed to connect to voice", { guildId, channelId }); + } + return result; + }, + [execute], + ); const leaveVoice = useCallback(async () => { - setLoading(true); - setError(null); - try { - const status = await disconnectVoice(); - setVoiceStatus(status); - return status; - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - setError(message); - throw err; - } finally { - setLoading(false); + const result = await execute(() => disconnectVoice()); + if (result) { + setVoiceStatus(result); + logger.info("Disconnected from voice"); + } else { + logger.error("Failed to disconnect from voice"); } - }, []); + return result; + }, [execute]); useEffect(() => { refreshGuilds().catch((err) => - setError(err instanceof Error ? err.message : String(err)), + logger.error("Failed to refresh guilds on mount", { error: String(err) }), ); refreshVoiceStatus().catch((err) => - setError(err instanceof Error ? err.message : String(err)), + logger.error("Failed to refresh voice status on mount", { + error: String(err), + }), ); }, [refreshGuilds, refreshVoiceStatus]); diff --git a/services/frontend/src/features/messages/hooks/useMessages.ts b/services/frontend/src/features/messages/hooks/useMessages.ts index 442ee9b..82de34c 100644 --- a/services/frontend/src/features/messages/hooks/useMessages.ts +++ b/services/frontend/src/features/messages/hooks/useMessages.ts @@ -5,6 +5,9 @@ import { reanalyzeErrorBatch, reanalyzeMessage, } from "../../../shared/api/client"; +import { createLogger } from "../../../shared/lib/logger.js"; + +const logger = createLogger("use-messages"); const PAGE_SIZE = 100; @@ -54,6 +57,7 @@ export function useMessages() { } catch (err) { const message = err instanceof Error ? err.message : String(err); setError(message); + logger.error("Failed to fetch messages", { guildId, error: message }); throw err; } finally { setLoading(false); @@ -72,6 +76,9 @@ export function useMessages() { setMessages((prev) => [...prev, ...result.data]); setCursor(result.nextCursor); setHasMore(!!result.nextCursor); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + logger.error("Failed to load more messages", { error: message }); } finally { setLoadingMore(false); } @@ -106,6 +113,8 @@ export function useMessages() { prev.map((message) => (message.id === id ? snapshot : message)), ); } + const message = err instanceof Error ? err.message : String(err); + logger.error("Failed to reanalyze message", { id, error: message }); throw err; } }, []); @@ -124,10 +133,17 @@ export function useMessages() { : message, ), ); - const { count } = await reanalyzeErrorBatch({ - guildId: currentGuild.current ?? undefined, - }); - return count; + try { + const { count } = await reanalyzeErrorBatch({ + guildId: currentGuild.current ?? undefined, + }); + logger.info("Reanalyze all errors complete", { count }); + return count; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + logger.error("Failed to reanalyze error batch", { error: message }); + throw err; + } }, []); return { diff --git a/services/frontend/src/shared/api/client.ts b/services/frontend/src/shared/api/client.ts index 0e19d27..ba3e70c 100644 --- a/services/frontend/src/shared/api/client.ts +++ b/services/frontend/src/shared/api/client.ts @@ -1,6 +1,9 @@ // ─── Shared HTTP client — all API endpoints in one file ────────────────────── import type { MessageRecord, PageResult } from "@bete/shared"; +import { createLogger } from "../lib/logger.js"; + +const logger = createLogger("api"); const BE_API_URL = import.meta.env.VITE_BE_API_URL || "http://localhost:3001"; const BE_WS_URL = import.meta.env.VITE_BE_WS_URL || "ws://localhost:3001"; @@ -20,6 +23,8 @@ class ApiError extends Error { export async function request(path: string, init?: RequestInit): Promise { const password = localStorage.getItem("admin-password"); const url = path.startsWith("http") ? path : `${BE_API_URL}${path}`; + logger.debug("Request", { method: init?.method ?? "GET", url }); + const res = await fetch(url, { headers: { "Content-Type": "application/json", @@ -38,10 +43,13 @@ export async function request(path: string, init?: RequestInit): Promise { } catch { // ignore parse errors } + logger.error("Request failed", { url, status: res.status, code, message }); throw new ApiError(code, message, res.status); } - return res.json() as Promise; + const result = (await res.json()) as T; + logger.debug("Response", { url, status: res.status }); + return result; } export function getWebSocketURL(): string { diff --git a/services/frontend/src/shared/hooks/useAsyncAction.ts b/services/frontend/src/shared/hooks/useAsyncAction.ts new file mode 100644 index 0000000..a71f887 --- /dev/null +++ b/services/frontend/src/shared/hooks/useAsyncAction.ts @@ -0,0 +1,37 @@ +// ─── Generic async action state hook ────────────────────────────────────── + +import { useCallback, useState } from "react"; + +interface AsyncActionState { + loading: boolean; + error: string | null; +} + +export function useAsyncAction() { + const [state, setState] = useState({ + loading: false, + error: null, + }); + + const execute = useCallback( + async (fn: () => Promise): Promise => { + setState({ loading: true, error: null }); + try { + const result = await fn(); + setState({ loading: false, error: null }); + return result; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + setState({ loading: false, error: message }); + return null; + } + }, + [], + ); + + const clearError = useCallback(() => { + setState((prev) => ({ ...prev, error: null })); + }, []); + + return { ...state, execute, clearError }; +} diff --git a/services/frontend/src/shared/hooks/useAudioPlayback.ts b/services/frontend/src/shared/hooks/useAudioPlayback.ts index 76eafc6..a576324 100644 --- a/services/frontend/src/shared/hooks/useAudioPlayback.ts +++ b/services/frontend/src/shared/hooks/useAudioPlayback.ts @@ -1,5 +1,8 @@ // ─── Audio playback hook — receives PCM from WebSocket and plays through Web Audio API ── import { useCallback, useRef, useState } from "react"; +import { createLogger } from "../lib/logger.js"; + +const logger = createLogger("use-audio-playback"); const SAMPLE_RATE = 24000; const CHANNELS = 1; @@ -15,56 +18,64 @@ export function useAudioPlayback() { const handleIncomingPcm = useCallback( (data: { userId: string; pcm: string }) => { // Decode base64 PCM data - const binaryString = atob(data.pcm); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - const int16Array = new Int16Array(bytes.buffer); + try { + const binaryString = atob(data.pcm); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + const int16Array = new Int16Array(bytes.buffer); - // Calculate audio levels for visualization - let sum = 0; - for (const sample of int16Array) sum += Math.abs(sample / 32768); - const average = int16Array.length ? sum / int16Array.length : 0; - setLevels((prev) => - prev.map((_, index) => - Math.max( - 0.04, - average * - (0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) * - 5, + // Calculate audio levels for visualization + let sum = 0; + for (const sample of int16Array) sum += Math.abs(sample / 32768); + const average = int16Array.length ? sum / int16Array.length : 0; + setLevels((prev) => + prev.map((_, index) => + Math.max( + 0.04, + average * + (0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) * + 5, + ), ), - ), - ); + ); - const audioContext = audioContextRef.current; - if (!isListening || !audioContext) return; + const audioContext = audioContextRef.current; + if (!isListening || !audioContext) return; - // Convert to float32 for Web Audio API - const float32Array = new Float32Array(int16Array.length); - for (let i = 0; i < int16Array.length; i++) - float32Array[i] = int16Array[i] / 32768; + // Convert to float32 for Web Audio API + const float32Array = new Float32Array(int16Array.length); + for (let i = 0; i < int16Array.length; i++) + float32Array[i] = int16Array[i] / 32768; - const audioBuffer = audioContext.createBuffer( - CHANNELS, - float32Array.length, - SAMPLE_RATE, - ); - audioBuffer.getChannelData(0).set(float32Array); + const audioBuffer = audioContext.createBuffer( + CHANNELS, + float32Array.length, + SAMPLE_RATE, + ); + audioBuffer.getChannelData(0).set(float32Array); - const source = audioContext.createBufferSource(); - source.buffer = audioBuffer; - source.connect(audioContext.destination); + const source = audioContext.createBufferSource(); + source.buffer = audioBuffer; + source.connect(audioContext.destination); - // Schedule playback per user to avoid overlaps - const currentTime = audioContext.currentTime; - let nextStart = userTimelinesRef.current.get(data.userId) || 0; - if (nextStart < currentTime) nextStart = currentTime + 0.05; - source.start(nextStart); - userTimelinesRef.current.set( - data.userId, - nextStart + audioBuffer.duration, - ); + // Schedule playback per user to avoid overlaps + const currentTime = audioContext.currentTime; + let nextStart = userTimelinesRef.current.get(data.userId) || 0; + if (nextStart < currentTime) nextStart = currentTime + 0.05; + source.start(nextStart); + userTimelinesRef.current.set( + data.userId, + nextStart + audioBuffer.duration, + ); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + logger.error("Failed to decode PCM audio", { + userId: data.userId, + error: message, + }); + } }, [isListening], ); @@ -74,6 +85,7 @@ export function useAudioPlayback() { await audioContextRef.current?.suspend(); userTimelinesRef.current.clear(); setIsListening(false); + logger.info("Audio playback paused"); return; } const AudioContextCtor = @@ -85,6 +97,7 @@ export function useAudioPlayback() { }); await audioContextRef.current.resume(); setIsListening(true); + logger.info("Audio playback started"); }, [isListening]); return { diff --git a/services/frontend/src/shared/lib/logger.ts b/services/frontend/src/shared/lib/logger.ts new file mode 100644 index 0000000..b031cd1 --- /dev/null +++ b/services/frontend/src/shared/lib/logger.ts @@ -0,0 +1,22 @@ +// ─── Client-side structured logger ──────────────────────────────────────── + +const LOG_PREFIX = "[Bete]"; + +export function createLogger(context: string) { + const prefix = `${LOG_PREFIX} [${context}]`; + + return { + debug: (msg: string, data?: Record) => { + if (import.meta.env.DEV) console.debug(prefix, msg, data ?? ""); + }, + info: (msg: string, data?: Record) => { + console.info(prefix, msg, data ?? ""); + }, + warn: (msg: string, data?: Record) => { + console.warn(prefix, msg, data ?? ""); + }, + error: (msg: string, data?: Record) => { + console.error(prefix, msg, data ?? ""); + }, + }; +} diff --git a/services/frontend/src/shared/ws/socket.ts b/services/frontend/src/shared/ws/socket.ts index 502d7e4..a386683 100644 --- a/services/frontend/src/shared/ws/socket.ts +++ b/services/frontend/src/shared/ws/socket.ts @@ -1,5 +1,8 @@ // ─── WebSocket singleton with reconnect, typed events, and observable status ─ import { useCallback, useEffect, useRef, useState } from "react"; +import { createLogger } from "../lib/logger.js"; + +const logger = createLogger("socket"); export type WsStatus = "connecting" | "connected" | "disconnected" | "error"; @@ -26,6 +29,7 @@ export interface WsHandlers { let _wsInstance: WebSocket | null = null; let _reconnectTimer: ReturnType | null = null; let _closed = false; +let _reconnectAttempts = 0; const _listeners = new Set(); const _statusCallbacks = new Set<(s: WsStatus) => void>(); @@ -41,12 +45,23 @@ function doConnect(): WebSocket { const ws = new WebSocket(url); ws.binaryType = "arraybuffer"; dispatchStatus("connecting"); + logger.info("Connecting", { url }); - ws.addEventListener("open", () => dispatchStatus("connected")); - ws.addEventListener("error", () => dispatchStatus("error")); - ws.addEventListener("close", () => { + ws.addEventListener("open", () => { + _reconnectAttempts = 0; + dispatchStatus("connected"); + logger.info("Connected"); + }); + ws.addEventListener("error", () => { + dispatchStatus("error"); + logger.error("WebSocket error"); + }); + ws.addEventListener("close", (event) => { dispatchStatus("disconnected"); + logger.info("Disconnected", { code: event.code, reason: event.reason }); if (!_closed && _listeners.size > 0) { + _reconnectAttempts++; + logger.warn("Reconnecting", { attempt: _reconnectAttempts }); _reconnectTimer = setTimeout(() => doReconnect(), 2500); } }); @@ -108,7 +123,9 @@ function doConnect(): WebSocket { } } } catch { - // ignore malformed messages + logger.error("Failed to parse message", { + raw: event.data.slice(0, 200), + }); } });