refactor: atomic, DRY, and logging improvements across codebase

- Split llmModerationClient.ts (2170 lines) into 5 focused sub-modules
- Split aiAnalyzer.ts (1282 lines) into 4 modular pipelines
- Split messages.db.ts (826 lines) into 5 domain-specific modules
- Moved shared schema to @bete/shared, eliminated backend duplication
- Added createChildLogger to all voice-recording and AI moderation modules
- Extracted tryCommandThenFallback, normalizeMediaState, DEFAULT_VOICE_STATUS
- Created shared pagination.ts utility, eliminated 5+ cursor-pagination duplications
- Created shared messageMapper.ts for row mapping
- Standardized backend error handling with asyncHandler
- Added frontend createLogger utility and useAsyncAction hook
- Added structured logging to frontend hooks, socket, and API client

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-06-09 19:46:08 +07:00
co-authored by Claude Opus 4.8
parent b68789fffc
commit 07032ab521
61 changed files with 3808 additions and 3043 deletions
+2
View File
@@ -8,6 +8,7 @@
"exports": { "exports": {
".": "./dist/index.js", ".": "./dist/index.js",
"./config": "./dist/config/index.js", "./config": "./dist/config/index.js",
"./database/schema": "./dist/database/schema.js",
"./errors": "./dist/errors/index.js", "./errors": "./dist/errors/index.js",
"./logger": "./dist/logger/index.js", "./logger": "./dist/logger/index.js",
"./utils": "./dist/utils/index.js" "./utils": "./dist/utils/index.js"
@@ -17,6 +18,7 @@
"typecheck": "tsc --noEmit" "typecheck": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"drizzle-orm": "^0.45.2",
"pino": "^9.0.0", "pino": "^9.0.0",
"zod": "^4.4.3" "zod": "^4.4.3"
}, },
+141
View File
@@ -0,0 +1,141 @@
import {
bigint as pgBigint,
foreignKey as pgForeignKey,
index as pgIndex,
integer as pgInteger,
real as pgReal,
pgTable,
text as pgText,
} from "drizzle-orm/pg-core";
/**
* Messages Table (PostgreSQL)
* Stores text messages with AI moderation analysis
*/
export const pgMessagesTable = pgTable(
"messages",
{
id: pgText("id").primaryKey(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
username: pgText("username").notNull(),
avatar_url: pgText("avatar_url"),
content: pgText("content").notNull(),
edited_content: pgText("edited_content"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
edited_at: pgBigint("edited_at", { mode: "number" }),
deleted_at: pgBigint("deleted_at", { mode: "number" }),
type: pgText("type", { enum: ["text", "edited", "deleted"] })
.notNull()
.default("text"),
metadata: pgText("metadata"),
ai_status: pgText("ai_status", {
enum: ["pending", "processing", "clean", "warn", "flagged", "error"],
})
.notNull()
.default("pending"),
ai_moderation_flags: pgText("ai_moderation_flags"),
ai_moderation_score: pgReal("ai_moderation_score"),
ai_analysis: pgText("ai_analysis"),
ai_categories: pgText("ai_categories"),
ai_severity: pgText("ai_severity", {
enum: ["none", "low", "medium", "high", "critical"],
}),
ai_confidence: pgReal("ai_confidence"),
ai_recommended_action: pgText("ai_recommended_action", {
enum: ["none", "monitor", "warn", "review", "delete", "escalate"],
}),
ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }),
ai_error: pgText("ai_error"),
},
(table) => ({
channelIdx: pgIndex("idx_messages_channel").on(table.channel_id),
userIdx: pgIndex("idx_messages_user").on(table.user_id),
createdIdx: pgIndex("idx_messages_created").on(table.created_at),
threadIdx: pgIndex("idx_messages_thread").on(table.thread_id),
channelCreatedIdx: pgIndex("idx_messages_channel_created").on(
table.channel_id,
table.created_at,
table.id,
),
threadCreatedIdx: pgIndex("idx_messages_thread_created").on(
table.thread_id,
table.created_at,
table.id,
),
aiStatusCreatedIdx: pgIndex("idx_messages_ai_status_created").on(
table.ai_status,
table.created_at,
table.id,
),
guildAiStatusCreatedIdx: pgIndex("idx_messages_guild_ai_status_created").on(
table.guild_id,
table.ai_status,
table.created_at,
table.id,
),
guildCreatedDeletedIdx: pgIndex("idx_messages_guild_created_deleted").on(
table.guild_id,
table.created_at,
table.deleted_at,
table.id,
),
channelAiStatusCreatedIdx: pgIndex(
"idx_messages_channel_ai_status_created",
).on(table.channel_id, table.ai_status, table.created_at, table.id),
threadAiStatusCreatedIdx: pgIndex(
"idx_messages_thread_ai_status_created",
).on(table.thread_id, table.ai_status, table.created_at, table.id),
}),
);
/**
* Attachments Table (PostgreSQL)
* Stores attachment metadata with upload status tracking
*/
export const pgAttachmentsTable = pgTable(
"attachments",
{
id: pgText("id").primaryKey(),
message_id: pgText("message_id").notNull(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
filename: pgText("filename").notNull(),
size: pgInteger("size").notNull(),
type: pgText("type").notNull(),
discord_url: pgText("discord_url").notNull(),
uploaded_url: pgText("uploaded_url"),
upload_status: pgText("upload_status", {
enum: ["pending", "uploaded", "failed"],
})
.notNull()
.default("pending"),
upload_error: pgText("upload_error"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
uploaded_at: pgBigint("uploaded_at", { mode: "number" }),
},
(table) => ({
channelIdx: pgIndex("idx_attachments_channel").on(table.channel_id),
messageIdx: pgIndex("idx_attachments_message").on(table.message_id),
statusIdx: pgIndex("idx_attachments_status").on(table.upload_status),
channelCreatedIdx: pgIndex("idx_attachments_channel_created").on(
table.channel_id,
table.created_at,
table.id,
),
threadCreatedIdx: pgIndex("idx_attachments_thread_created").on(
table.thread_id,
table.created_at,
table.id,
),
messageFk: pgForeignKey({
columns: [table.message_id],
foreignColumns: [pgMessagesTable.id],
name: "fk_attachments_message_id",
}).onDelete("cascade"),
}),
);
+1
View File
@@ -1,4 +1,5 @@
export * from "./config/index.js"; export * from "./config/index.js";
export * from "./database/schema.js";
export * from "./errors/index.js"; export * from "./errors/index.js";
export * from "./logger/index.js"; export * from "./logger/index.js";
export * from "./moderation-types.js"; export * from "./moderation-types.js";
+2
View File
@@ -4,6 +4,8 @@ export function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
export * from "./pagination.js";
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Retry with exponential backoff // Retry with exponential backoff
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
+65
View File
@@ -0,0 +1,65 @@
// Shared cursor-based pagination utilities
export interface CursorData {
created_at: number;
id: string;
}
/**
* Encode a cursor to a base64 string.
*/
export function encodeCursor(data: CursorData): string {
return Buffer.from(JSON.stringify(data)).toString("base64");
}
/**
* Decode a cursor from a base64 string. Returns null on invalid input.
*/
export function decodeCursor(cursor?: string): CursorData | null {
if (!cursor) return null;
try {
const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8"));
if (typeof data.created_at === "number" && typeof data.id === "string") {
return data;
}
return null;
} catch {
return null;
}
}
/**
* Build a `PageResult` from a slice of rows (limit + 1) using cursor-based pagination.
*/
export function pageResult<T extends { created_at: number; id: string }>(
rows: unknown[],
limit: number,
): { data: T[]; nextCursor: string | null } {
const hasMore = rows.length > limit;
const data = rows.slice(0, limit) as T[];
const lastItem = data[data.length - 1];
const nextCursor =
hasMore && lastItem
? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id })
: null;
return { data, nextCursor };
}
/**
* Build a Drizzle cursor condition expression.
* Used in WHERE clauses: `(created_at < cursor.created_at OR (created_at = cursor.created_at AND id < cursor.id))`
*
* Returns the SQL expression or undefined when cursor is absent.
*/
import { type SQL, sql } from "drizzle-orm";
export function buildCursorCondition(
created_at_col: SQL | unknown,
id_col: SQL | unknown,
cursor?: string,
): SQL | undefined {
const data = decodeCursor(cursor);
if (!data) return undefined;
return sql`(${created_at_col} < ${data.created_at} or (${created_at_col} = ${data.created_at} and ${id_col} < ${data.id}))`;
}
+3
View File
@@ -33,6 +33,9 @@ importers:
packages/shared: packages/shared:
dependencies: dependencies:
drizzle-orm:
specifier: ^0.45.2
version: 0.45.2(@opentelemetry/api@1.9.1)(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@11.10.0)(pg@8.21.0)
pino: pino:
specifier: ^9.0.0 specifier: ^9.0.0
version: 9.14.0 version: 9.14.0
@@ -1,5 +1,9 @@
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { getPool } from "../../shared/database/index.js"; import { getPool } from "../../shared/database/index.js";
import {
type MappedMessage,
mapMessageRow,
} from "../../shared/utils/messageMapper.js";
const logger = createChildLogger("analysis.repository"); const logger = createChildLogger("analysis.repository");
@@ -10,61 +14,8 @@ export interface AnalysisSearchQuery {
limit?: number; limit?: number;
} }
export interface AnalysisSearchResult { // AnalysisSearchResult is identical to MappedMessage — reuse the shared mapper
id: string; export type AnalysisSearchResult = MappedMessage;
guild_id: string;
channel_id: string;
thread_id: string | null;
user_id: string;
username: string;
avatar_url: string | null;
content: string;
edited_content: string | null;
created_at: number;
edited_at: number | null;
deleted_at: number | null;
type: string;
metadata: string | null;
ai_status: string | null;
ai_moderation_flags: string | null;
ai_moderation_score: number | null;
ai_analysis: string | null;
ai_categories: string | null;
ai_severity: string | null;
ai_confidence: number | null;
ai_recommended_action: string | null;
ai_analyzed_at: number | null;
ai_error: string | null;
}
function mapSearchResult(row: Record<string, unknown>): AnalysisSearchResult {
return {
id: String(row.id ?? ""),
guild_id: String(row.guild_id ?? ""),
channel_id: String(row.channel_id ?? ""),
thread_id: (row.thread_id as string | null) ?? null,
user_id: String(row.user_id ?? ""),
username: String(row.username ?? ""),
avatar_url: (row.avatar_url as string | null) ?? null,
content: String(row.content ?? ""),
edited_content: (row.edited_content as string | null) ?? null,
created_at: Number(row.created_at ?? 0),
edited_at: (row.edited_at as number | null) ?? null,
deleted_at: (row.deleted_at as number | null) ?? null,
type: String(row.type ?? "text"),
metadata: (row.metadata as string | null) ?? null,
ai_status: (row.ai_status as string | null) ?? null,
ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null,
ai_moderation_score: (row.ai_moderation_score as number | null) ?? null,
ai_analysis: (row.ai_analysis as string | null) ?? null,
ai_categories: (row.ai_categories as string | null) ?? null,
ai_severity: (row.ai_severity as string | null) ?? null,
ai_confidence: (row.ai_confidence as number | null) ?? null,
ai_recommended_action: (row.ai_recommended_action as string | null) ?? null,
ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null,
ai_error: (row.ai_error as string | null) ?? null,
};
}
export class AnalysisRepository { export class AnalysisRepository {
async search(query: AnalysisSearchQuery): Promise<AnalysisSearchResult[]> { async search(query: AnalysisSearchQuery): Promise<AnalysisSearchResult[]> {
@@ -107,7 +58,7 @@ export class AnalysisRepository {
[...params, limit], [...params, limit],
); );
return rows.map((r) => mapSearchResult(r as Record<string, unknown>)); return rows.map((r) => mapMessageRow(r as Record<string, unknown>));
} }
} }
@@ -1,16 +1,12 @@
import type { NextFunction, Request, Response } from "express"; import type { Request, Response } from "express";
import { asyncHandler } from "../../shared/middlewares/index.js"; import { asyncHandler } from "../../shared/middlewares/index.js";
import { healthService } from "./health.service.js"; import { healthService } from "./health.service.js";
export function handleHealthCheck( export const handleHealthCheck = asyncHandler(
req: Request, async (req: Request, res: Response) => {
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const verbose = req.query.verbose === "true"; const verbose = req.query.verbose === "true";
const result = await healthService.getHealth(verbose); const result = await healthService.getHealth(verbose);
const status = result.status === "healthy" ? 200 : 503; const status = result.status === "healthy" ? 200 : 503;
res.status(status).json(result); res.status(status).json(result);
})(req, res, next); },
} );
@@ -1,5 +1,6 @@
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response } from "express"; import type { Request, Response } from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import { mascotChatService } from "./mascot-chat.service.js"; import { mascotChatService } from "./mascot-chat.service.js";
const logger = createChildLogger("mascot-chat.controller"); const logger = createChildLogger("mascot-chat.controller");
@@ -8,8 +9,8 @@ interface AuthenticatedRequest extends Request {
userId?: string; userId?: string;
} }
export async function handleMascotChat(req: Request, res: Response) { export const handleMascotChat = asyncHandler(
try { async (req: Request, res: Response) => {
const { message, context } = req.body; const { message, context } = req.body;
if (!message || typeof message !== "string") { if (!message || typeof message !== "string") {
@@ -49,17 +50,11 @@ export async function handleMascotChat(req: Request, res: Response) {
response, response,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
}); });
} catch (error) { },
logger.error({ error }, "Error processing mascot chat"); );
res.status(500).json({
error: "INTERNAL_SERVER_ERROR",
message: "Failed to process mascot chat",
});
}
}
export async function getMascotChatHistory(req: Request, res: Response) { export const getMascotChatHistory = asyncHandler(
try { async (req: Request, res: Response) => {
const userId = (req as AuthenticatedRequest).userId || "anonymous"; const userId = (req as AuthenticatedRequest).userId || "anonymous";
const limit = Math.min(parseInt(req.query.limit as string) || 50, 100); const limit = Math.min(parseInt(req.query.limit as string) || 50, 100);
@@ -69,17 +64,11 @@ export async function getMascotChatHistory(req: Request, res: Response) {
history, history,
total: history.length, total: history.length,
}); });
} catch (error) { },
logger.error({ error }, "Error fetching chat history"); );
res.status(500).json({
error: "INTERNAL_SERVER_ERROR",
message: "Failed to fetch chat history",
});
}
}
export async function clearMascotChatHistory(req: Request, res: Response) { export const clearMascotChatHistory = asyncHandler(
try { async (req: Request, res: Response) => {
const userId = (req as AuthenticatedRequest).userId || "anonymous"; const userId = (req as AuthenticatedRequest).userId || "anonymous";
await mascotChatService.clearChatHistory(userId); await mascotChatService.clearChatHistory(userId);
@@ -87,11 +76,5 @@ export async function clearMascotChatHistory(req: Request, res: Response) {
res.status(200).json({ res.status(200).json({
message: "Chat history cleared successfully", message: "Chat history cleared successfully",
}); });
} catch (error) { },
logger.error({ error }, "Error clearing chat history"); );
res.status(500).json({
error: "INTERNAL_SERVER_ERROR",
message: "Failed to clear chat history",
});
}
}
@@ -1,5 +1,4 @@
import express, { type Router } from "express"; import express, { type Router } from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import { import {
clearMascotChatHistory, clearMascotChatHistory,
getMascotChatHistory, getMascotChatHistory,
@@ -9,9 +8,9 @@ import {
export function createMascotChatRouter(): Router { export function createMascotChatRouter(): Router {
const router = express.Router(); const router = express.Router();
router.post("/mascot/chat", asyncHandler(handleMascotChat)); router.post("/mascot/chat", handleMascotChat);
router.get("/mascot/chat/history", asyncHandler(getMascotChatHistory)); router.get("/mascot/chat/history", getMascotChatHistory);
router.delete("/mascot/chat/history", asyncHandler(clearMascotChatHistory)); router.delete("/mascot/chat/history", clearMascotChatHistory);
return router; return router;
} }
@@ -5,7 +5,10 @@ import {
COMMAND_MEDIA_VOLUME, COMMAND_MEDIA_VOLUME,
MEDIA_STATUS_KEY, MEDIA_STATUS_KEY,
} from "@bete/shared"; } from "@bete/shared";
import { createChildLogger } from "@bete/shared/logger"; import {
createChildLogger,
tryCommandThenFallback,
} from "../../shared/commandHelper.js";
import { publishCommand, readRedisStatus } from "../../shared/redis/index.js"; import { publishCommand, readRedisStatus } from "../../shared/redis/index.js";
const logger = createChildLogger("media.service"); const logger = createChildLogger("media.service");
@@ -43,6 +46,35 @@ const DEFAULT_STATE: MediaState = {
queue: [], queue: [],
}; };
// ---------------------------------------------------------------------------
// Normalisation — handle both boolean (new) and string (legacy) playing values
// ---------------------------------------------------------------------------
function normalizeMediaState(raw: Record<string, unknown>): MediaState {
const rawPlaying = raw.playing;
const playing =
rawPlaying === true ||
rawPlaying === "playing" ||
rawPlaying === "buffering";
return {
playing,
musicVolume: Number(raw.musicVolume ?? 1.0),
current: (raw.current as MediaItem | null) ?? null,
queue: (raw.queue as MediaItem[]) ?? [],
};
}
type MediaReplyData = Record<string, unknown> | MediaState;
function fromReply(data: MediaReplyData): MediaState {
return normalizeMediaState(data as Record<string, unknown>);
}
async function readStatusFallback(): Promise<MediaState> {
const cached = await readRedisStatus(MEDIA_STATUS_KEY);
return cached ? normalizeMediaState(cached) : DEFAULT_STATE;
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Service methods // Service methods
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -52,24 +84,7 @@ const DEFAULT_STATE: MediaState = {
*/ */
export async function getStatus(): Promise<MediaState> { export async function getStatus(): Promise<MediaState> {
logger.debug("getStatus called"); logger.debug("getStatus called");
const cached = await readRedisStatus(MEDIA_STATUS_KEY); return readStatusFallback();
if (cached) {
const rawPlaying = cached.playing;
// Handle both boolean (new) and string (legacy from String(discordPlayer.getStatus()))
const playing =
rawPlaying === true ||
rawPlaying === "playing" ||
rawPlaying === "buffering";
return {
playing,
musicVolume: Number(cached.musicVolume ?? 1.0),
current: (cached.current as MediaItem | null) ?? null,
queue: (cached.queue as MediaItem[]) ?? [],
};
}
return DEFAULT_STATE;
} }
/** /**
@@ -80,26 +95,16 @@ export async function queue(
mode: "music" | "screen" = "music", mode: "music" | "screen" = "music",
): Promise<MediaState> { ): Promise<MediaState> {
logger.info({ source, mode }, "queue called"); logger.info({ source, mode }, "queue called");
const reply = await publishCommand<MediaState>( return tryCommandThenFallback(
COMMAND_MEDIA_QUEUE, () =>
{ source, mode }, publishCommand<MediaState>(
DEFAULT_COMMAND_TIMEOUT_MS, COMMAND_MEDIA_QUEUE,
{ source, mode },
DEFAULT_COMMAND_TIMEOUT_MS,
),
() => readStatusFallback(),
"queue",
); );
if (reply?.success && reply.data) {
return {
playing: reply.data.playing,
musicVolume: reply.data.musicVolume,
current: reply.data.current ?? null,
queue: reply.data.queue ?? [],
};
}
logger.warn(
{ source, mode },
"discord-gateway unreachable, returning current media status",
);
return getStatus();
} }
/** /**
@@ -107,23 +112,16 @@ export async function queue(
*/ */
export async function skip(): Promise<MediaState> { export async function skip(): Promise<MediaState> {
logger.info("skip called"); logger.info("skip called");
const reply = await publishCommand<MediaState>( return tryCommandThenFallback(
COMMAND_MEDIA_SKIP, () =>
{}, publishCommand<MediaState>(
DEFAULT_COMMAND_TIMEOUT_MS, COMMAND_MEDIA_SKIP,
{},
DEFAULT_COMMAND_TIMEOUT_MS,
),
() => readStatusFallback(),
"skip",
); );
if (reply?.success && reply.data) {
return {
playing: reply.data.playing,
musicVolume: reply.data.musicVolume,
current: reply.data.current ?? null,
queue: reply.data.queue ?? [],
};
}
logger.warn("discord-gateway unreachable, returning current media status");
return getStatus();
} }
/** /**
@@ -131,23 +129,16 @@ export async function skip(): Promise<MediaState> {
*/ */
export async function stop(): Promise<MediaState> { export async function stop(): Promise<MediaState> {
logger.info("stop called"); logger.info("stop called");
const reply = await publishCommand<MediaState>( return tryCommandThenFallback(
COMMAND_MEDIA_STOP, () =>
{}, publishCommand<MediaState>(
DEFAULT_COMMAND_TIMEOUT_MS, COMMAND_MEDIA_STOP,
{},
DEFAULT_COMMAND_TIMEOUT_MS,
),
() => readStatusFallback(),
"stop",
); );
if (reply?.success && reply.data) {
return {
playing: reply.data.playing,
musicVolume: reply.data.musicVolume,
current: reply.data.current ?? null,
queue: reply.data.queue ?? [],
};
}
logger.warn("discord-gateway unreachable, returning current media status");
return getStatus();
} }
/** /**
@@ -155,24 +146,14 @@ export async function stop(): Promise<MediaState> {
*/ */
export async function setVolume(volume: number): Promise<MediaState> { export async function setVolume(volume: number): Promise<MediaState> {
logger.info({ volume }, "setVolume called"); logger.info({ volume }, "setVolume called");
const reply = await publishCommand<MediaState>( return tryCommandThenFallback(
COMMAND_MEDIA_VOLUME, () =>
{ volume }, publishCommand<MediaState>(
DEFAULT_COMMAND_TIMEOUT_MS, COMMAND_MEDIA_VOLUME,
{ volume },
DEFAULT_COMMAND_TIMEOUT_MS,
),
() => readStatusFallback(),
"setVolume",
); );
if (reply?.success && reply.data) {
return {
playing: reply.data.playing,
musicVolume: reply.data.musicVolume,
current: reply.data.current ?? null,
queue: reply.data.queue ?? [],
};
}
logger.warn(
{ volume },
"discord-gateway unreachable, returning current media status",
);
return getStatus();
} }
@@ -1,14 +1,9 @@
import type { PageResult } from "@bete/shared"; import type { PageResult } from "@bete/shared";
import { pgAttachmentsTable, pgMessagesTable } from "@bete/shared";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { and, desc, eq, inArray, lt, ne, type SQL } from "drizzle-orm"; import { and, desc, eq, inArray, lt, ne, type SQL } from "drizzle-orm";
import {
bigint as pgBigint,
integer as pgInteger,
real as pgReal,
pgTable,
text as pgText,
} from "drizzle-orm/pg-core";
import { getDatabase } from "../../shared/database/index.js"; import { getDatabase } from "../../shared/database/index.js";
import { mapMessageRow } from "../../shared/utils/messageMapper.js";
import type { import type {
MessageCreate, MessageCreate,
MessageQuery, MessageQuery,
@@ -17,71 +12,6 @@ import type {
const logger = createChildLogger("messages.repository"); const logger = createChildLogger("messages.repository");
/**
* Local table definitions mirroring services/discord-gateway/src/shared/database/schema.ts.
* These are query-building references only — schema source of truth remains in discord-gateway.
*/
const messages = pgTable("messages", {
id: pgText("id").primaryKey(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
username: pgText("username").notNull(),
avatar_url: pgText("avatar_url"),
content: pgText("content").notNull(),
edited_content: pgText("edited_content"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
edited_at: pgBigint("edited_at", { mode: "number" }),
deleted_at: pgBigint("deleted_at", { mode: "number" }),
type: pgText("type", {
enum: ["text", "edited", "deleted"],
})
.notNull()
.default("text"),
metadata: pgText("metadata"),
ai_status: pgText("ai_status", {
enum: ["pending", "processing", "clean", "warn", "flagged", "error"],
})
.notNull()
.default("pending"),
ai_moderation_flags: pgText("ai_moderation_flags"),
ai_moderation_score: pgReal("ai_moderation_score"),
ai_analysis: pgText("ai_analysis"),
ai_categories: pgText("ai_categories"),
ai_severity: pgText("ai_severity", {
enum: ["none", "low", "medium", "high", "critical"],
}),
ai_confidence: pgReal("ai_confidence"),
ai_recommended_action: pgText("ai_recommended_action", {
enum: ["none", "monitor", "warn", "review", "delete", "escalate"],
}),
ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }),
ai_error: pgText("ai_error"),
});
const attachments = pgTable("attachments", {
id: pgText("id").primaryKey(),
message_id: pgText("message_id").notNull(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
filename: pgText("filename").notNull(),
size: pgInteger("size").notNull(),
type: pgText("type").notNull(),
discord_url: pgText("discord_url").notNull(),
uploaded_url: pgText("uploaded_url"),
upload_status: pgText("upload_status", {
enum: ["pending", "uploaded", "failed"],
})
.notNull()
.default("pending"),
upload_error: pgText("upload_error"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
uploaded_at: pgBigint("uploaded_at", { mode: "number" }),
});
export interface AttachmentResult { export interface AttachmentResult {
id: string; id: string;
message_id: string; message_id: string;
@@ -100,35 +30,6 @@ export interface AttachmentResult {
uploaded_at: number | null; uploaded_at: number | null;
} }
function mapMessageRow(row: Record<string, unknown>) {
return {
id: String(row.id ?? ""),
guild_id: String(row.guild_id ?? ""),
channel_id: String(row.channel_id ?? ""),
thread_id: (row.thread_id as string | null) ?? null,
user_id: String(row.user_id ?? ""),
username: String(row.username ?? ""),
avatar_url: (row.avatar_url as string | null) ?? null,
content: String(row.content ?? ""),
edited_content: (row.edited_content as string | null) ?? null,
created_at: Number(row.created_at ?? 0),
edited_at: (row.edited_at as number | null) ?? null,
deleted_at: (row.deleted_at as number | null) ?? null,
type: String(row.type ?? "text"),
metadata: (row.metadata as string | null) ?? null,
ai_status: (row.ai_status as string | null) ?? null,
ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null,
ai_moderation_score: (row.ai_moderation_score as number | null) ?? null,
ai_analysis: (row.ai_analysis as string | null) ?? null,
ai_categories: (row.ai_categories as string | null) ?? null,
ai_severity: (row.ai_severity as string | null) ?? null,
ai_confidence: (row.ai_confidence as number | null) ?? null,
ai_recommended_action: (row.ai_recommended_action as string | null) ?? null,
ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null,
ai_error: (row.ai_error as string | null) ?? null,
};
}
export class MessagesRepository { export class MessagesRepository {
async findMany( async findMany(
query: MessageQuery, query: MessageQuery,
@@ -138,27 +39,27 @@ export class MessagesRepository {
const conditions: SQL[] = []; const conditions: SQL[] = [];
if (query.guildId) { if (query.guildId) {
conditions.push(eq(messages.guild_id, query.guildId)); conditions.push(eq(pgMessagesTable.guild_id, query.guildId));
} }
if (query.channelId) { if (query.channelId) {
conditions.push(eq(messages.channel_id, query.channelId)); conditions.push(eq(pgMessagesTable.channel_id, query.channelId));
} }
if (query.userId) { if (query.userId) {
conditions.push(eq(messages.user_id, query.userId)); conditions.push(eq(pgMessagesTable.user_id, query.userId));
} }
if (query.status) { if (query.status) {
conditions.push(eq(messages.ai_status, query.status)); conditions.push(eq(pgMessagesTable.ai_status, query.status));
} }
if (query.cursor) { if (query.cursor) {
conditions.push(lt(messages.created_at, Number(query.cursor))); conditions.push(lt(pgMessagesTable.created_at, Number(query.cursor)));
} }
const where = conditions.length > 0 ? and(...conditions) : undefined; const where = conditions.length > 0 ? and(...conditions) : undefined;
const rows = await db const rows = await db
.select() .select()
.from(messages) .from(pgMessagesTable)
.where(where) .where(where)
.orderBy(desc(messages.created_at)) .orderBy(desc(pgMessagesTable.created_at))
.limit(limit + 1); .limit(limit + 1);
const data = rows const data = rows
@@ -175,8 +76,8 @@ export class MessagesRepository {
const db = getDatabase(); const db = getDatabase();
const [row] = await db const [row] = await db
.select() .select()
.from(messages) .from(pgMessagesTable)
.where(eq(messages.id, id)) .where(eq(pgMessagesTable.id, id))
.limit(1); .limit(1);
if (!row) return null; if (!row) return null;
@@ -189,17 +90,17 @@ export class MessagesRepository {
): Promise<PageResult<ReturnType<typeof mapMessageRow>>> { ): Promise<PageResult<ReturnType<typeof mapMessageRow>>> {
const db = getDatabase(); const db = getDatabase();
const limit = query.limit ?? 50; const limit = query.limit ?? 50;
const conditions: SQL[] = [eq(messages.channel_id, channelId)]; const conditions: SQL[] = [eq(pgMessagesTable.channel_id, channelId)];
if (query.cursor) { if (query.cursor) {
conditions.push(lt(messages.created_at, Number(query.cursor))); conditions.push(lt(pgMessagesTable.created_at, Number(query.cursor)));
} }
const rows = await db const rows = await db
.select() .select()
.from(messages) .from(pgMessagesTable)
.where(and(...conditions)) .where(and(...conditions))
.orderBy(desc(messages.created_at)) .orderBy(desc(pgMessagesTable.created_at))
.limit(limit + 1); .limit(limit + 1);
const data = rows const data = rows
@@ -216,7 +117,7 @@ export class MessagesRepository {
const id = crypto.randomUUID(); const id = crypto.randomUUID();
const [row] = await db const [row] = await db
.insert(messages) .insert(pgMessagesTable)
.values({ .values({
id, id,
guild_id: data.guildId, guild_id: data.guildId,
@@ -242,7 +143,7 @@ export class MessagesRepository {
async update(id: string, data: MessageUpdate) { async update(id: string, data: MessageUpdate) {
const db = getDatabase(); const db = getDatabase();
const setData: Partial<typeof messages.$inferInsert> = {}; const setData: Partial<typeof pgMessagesTable.$inferInsert> = {};
if (data.editedContent !== undefined) { if (data.editedContent !== undefined) {
setData.edited_content = data.editedContent; setData.edited_content = data.editedContent;
@@ -266,9 +167,9 @@ export class MessagesRepository {
if (Object.keys(setData).length === 0) return this.findById(id); if (Object.keys(setData).length === 0) return this.findById(id);
const [row] = await db const [row] = await db
.update(messages) .update(pgMessagesTable)
.set(setData) .set(setData)
.where(eq(messages.id, id)) .where(eq(pgMessagesTable.id, id))
.returning(); .returning();
if (!row) return null; if (!row) return null;
@@ -288,20 +189,20 @@ export class MessagesRepository {
messageIds?: string[]; messageIds?: string[];
}): Promise<number> { }): Promise<number> {
const db = getDatabase(); const db = getDatabase();
const conditions: SQL[] = [eq(messages.ai_status, "error")]; const conditions: SQL[] = [eq(pgMessagesTable.ai_status, "error")];
if (opts.messageIds && opts.messageIds.length > 0) { if (opts.messageIds && opts.messageIds.length > 0) {
conditions.push(inArray(messages.id, opts.messageIds)); conditions.push(inArray(pgMessagesTable.id, opts.messageIds));
} }
if (opts.guildId) { if (opts.guildId) {
conditions.push(eq(messages.guild_id, opts.guildId)); conditions.push(eq(pgMessagesTable.guild_id, opts.guildId));
} }
if (opts.channelId) { if (opts.channelId) {
conditions.push(eq(messages.channel_id, opts.channelId)); conditions.push(eq(pgMessagesTable.channel_id, opts.channelId));
} }
const result = await db const result = await db
.update(messages) .update(pgMessagesTable)
.set({ ai_status: "pending" }) .set({ ai_status: "pending" })
.where(and(...conditions)); .where(and(...conditions));
@@ -317,9 +218,14 @@ export class MessagesRepository {
async markForReanalysis(id: string): Promise<void> { async markForReanalysis(id: string): Promise<void> {
const db = getDatabase(); const db = getDatabase();
await db await db
.update(messages) .update(pgMessagesTable)
.set({ ai_status: "pending" }) .set({ ai_status: "pending" })
.where(and(eq(messages.id, id), ne(messages.ai_status, "pending"))); .where(
and(
eq(pgMessagesTable.id, id),
ne(pgMessagesTable.ai_status, "pending"),
),
);
} }
/** /**
@@ -332,32 +238,32 @@ export class MessagesRepository {
): Promise<Record<string, unknown>[]> { ): Promise<Record<string, unknown>[]> {
const db = getDatabase(); const db = getDatabase();
const conditions: SQL[] = [ const conditions: SQL[] = [
inArray(messages.ai_status, ["warn", "flagged"]), inArray(pgMessagesTable.ai_status, ["warn", "flagged"]),
]; ];
if (channelId) { if (channelId) {
conditions.push(eq(messages.channel_id, channelId)); conditions.push(eq(pgMessagesTable.channel_id, channelId));
} }
const rows = await db const rows = await db
.select({ .select({
id: messages.id, id: pgMessagesTable.id,
guild_id: messages.guild_id, guild_id: pgMessagesTable.guild_id,
channel_id: messages.channel_id, channel_id: pgMessagesTable.channel_id,
user_id: messages.user_id, user_id: pgMessagesTable.user_id,
username: messages.username, username: pgMessagesTable.username,
avatar_url: messages.avatar_url, avatar_url: pgMessagesTable.avatar_url,
content: messages.content, content: pgMessagesTable.content,
type: messages.type, type: pgMessagesTable.type,
created_at: messages.created_at, created_at: pgMessagesTable.created_at,
ai_status: messages.ai_status, ai_status: pgMessagesTable.ai_status,
ai_severity: messages.ai_severity, ai_severity: pgMessagesTable.ai_severity,
ai_confidence: messages.ai_confidence, ai_confidence: pgMessagesTable.ai_confidence,
ai_analysis: messages.ai_analysis, ai_analysis: pgMessagesTable.ai_analysis,
}) })
.from(messages) .from(pgMessagesTable)
.where(and(...conditions)) .where(and(...conditions))
.orderBy(desc(messages.created_at)) .orderBy(desc(pgMessagesTable.created_at))
.limit(limit); .limit(limit);
return rows as unknown as Record<string, unknown>[]; return rows as unknown as Record<string, unknown>[];
@@ -365,7 +271,9 @@ export class MessagesRepository {
async delete(id: string): Promise<boolean> { async delete(id: string): Promise<boolean> {
const db = getDatabase(); const db = getDatabase();
const result = await db.delete(messages).where(eq(messages.id, id)); const result = await db
.delete(pgMessagesTable)
.where(eq(pgMessagesTable.id, id));
return (result.rowCount ?? 0) > 0; return (result.rowCount ?? 0) > 0;
} }
@@ -376,17 +284,17 @@ export class MessagesRepository {
): Promise<PageResult<AttachmentResult>> { ): Promise<PageResult<AttachmentResult>> {
const db = getDatabase(); const db = getDatabase();
const limit = query.limit ?? 50; const limit = query.limit ?? 50;
const conditions: SQL[] = [eq(attachments.channel_id, channelId)]; const conditions: SQL[] = [eq(pgAttachmentsTable.channel_id, channelId)];
if (query.cursor) { if (query.cursor) {
conditions.push(lt(attachments.created_at, Number(query.cursor))); conditions.push(lt(pgAttachmentsTable.created_at, Number(query.cursor)));
} }
const rows = await db const rows = await db
.select() .select()
.from(attachments) .from(pgAttachmentsTable)
.where(and(...conditions)) .where(and(...conditions))
.orderBy(desc(attachments.created_at)) .orderBy(desc(pgAttachmentsTable.created_at))
.limit(limit + 1); .limit(limit + 1);
const data = rows.map((r) => ({ const data = rows.map((r) => ({
@@ -1,21 +1,36 @@
import type { Router } from "express"; import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response, Router } from "express";
import express from "express"; import express from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import { getGuilds, getTextChannels } from "./voice.service.js"; import { getGuilds, getTextChannels } from "./voice.service.js";
const logger = createChildLogger("guilds.routes");
export function createGuildsRouter(): Router { export function createGuildsRouter(): Router {
const router = express.Router(); const router = express.Router();
// GET /api/guilds // GET /api/guilds
router.get("/", async (_req, res) => { router.get(
const guilds = await getGuilds(); "/",
res.json(guilds); asyncHandler(async (_req: Request, res: Response) => {
}); logger.debug("Fetching guilds");
const guilds = await getGuilds();
res.json(guilds);
}),
);
// GET /api/guilds/:guildId/channels // GET /api/guilds/:guildId/channels
router.get("/:guildId/channels", async (req, res) => { router.get(
const channels = await getTextChannels(req.params.guildId); "/:guildId/channels",
res.json(channels); asyncHandler(async (req: Request, res: Response) => {
}); const guildId = Array.isArray(req.params.guildId)
? req.params.guildId[0]
: req.params.guildId;
logger.debug({ guildId }, "Fetching text channels");
const channels = await getTextChannels(guildId);
res.json(channels);
}),
);
return router; return router;
} }
@@ -1,4 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response } from "express"; import type { Request, Response } from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import { publishCommandNoReply } from "../../shared/redis/index.js"; import { publishCommandNoReply } from "../../shared/redis/index.js";
import { import {
connectVoice, connectVoice,
@@ -7,10 +9,14 @@ import {
getVoiceStatus, getVoiceStatus,
} from "./voice.service.js"; } from "./voice.service.js";
export async function handleGetVoiceStatus(_req: Request, res: Response) { const logger = createChildLogger("voice.controller");
const status = await getVoiceStatus();
res.json(status); export const handleGetVoiceStatus = asyncHandler(
} async (_req: Request, res: Response) => {
const status = await getVoiceStatus();
res.json(status);
},
);
/** Safely extract a string value that may be a single string or string array. */ /** Safely extract a string value that may be a single string or string array. */
function asString(val: unknown): string { function asString(val: unknown): string {
@@ -18,47 +24,52 @@ function asString(val: unknown): string {
return String(val ?? ""); return String(val ?? "");
} }
export async function handleConnectVoice(req: Request, res: Response) { export const handleConnectVoice = asyncHandler(
const guildId = asString(req.body.guildId); async (req: Request, res: Response) => {
const channelId = asString(req.body.channelId); const guildId = asString(req.body.guildId);
if (!guildId || !channelId) { const channelId = asString(req.body.channelId);
return res.status(400).json({ if (!guildId || !channelId) {
error: "VALIDATION_ERROR", return res.status(400).json({
message: "guildId and channelId are required", error: "VALIDATION_ERROR",
}); message: "guildId and channelId are required",
} });
const status = await connectVoice(guildId, channelId); }
res.json(status); logger.debug({ guildId, channelId }, "Connecting to voice channel");
} const status = await connectVoice(guildId, channelId);
res.json(status);
},
);
export async function handleDisconnectVoice(_req: Request, res: Response) { export const handleDisconnectVoice = asyncHandler(
const status = await disconnectVoice(); async (_req: Request, res: Response) => {
res.json(status); logger.debug("Disconnecting from voice");
} const status = await disconnectVoice();
res.json(status);
},
);
export async function handleGetVoiceChannels(req: Request, res: Response) { export const handleGetVoiceChannels = asyncHandler(
const guildId = asString(req.params.guildId); async (req: Request, res: Response) => {
const channels = await getVoiceChannels(guildId); const guildId = asString(req.params.guildId);
res.json(channels); logger.debug({ guildId }, "Fetching voice channels");
} const channels = await getVoiceChannels(guildId);
res.json(channels);
},
);
export async function handleVoiceCommand(req: Request, res: Response) { export const handleVoiceCommand = asyncHandler(
const command = asString(req.body.command); async (req: Request, res: Response) => {
const command = asString(req.body.command);
if (!command) { if (!command) {
return res.status(400).json({ return res.status(400).json({
error: "VALIDATION_ERROR", error: "VALIDATION_ERROR",
message: "command is required", message: "command is required",
}); });
} }
try { logger.debug({ command }, "Publishing voice command");
await publishCommandNoReply(command); await publishCommandNoReply(command);
res.json({ success: true, command }); res.json({ success: true, command });
} catch (err) { },
res.status(500).json({ );
error: "COMMAND_FAILED",
message: err instanceof Error ? err.message : "Unknown error",
});
}
}
@@ -6,7 +6,10 @@ import {
COMMAND_VOICE_DISCONNECT, COMMAND_VOICE_DISCONNECT,
VOICE_STATUS_KEY, VOICE_STATUS_KEY,
} from "@bete/shared"; } from "@bete/shared";
import { createChildLogger } from "@bete/shared/logger"; import {
createChildLogger,
tryCommandThenFallback,
} from "../../shared/commandHelper.js";
import { getPool } from "../../shared/database/index.js"; import { getPool } from "../../shared/database/index.js";
import { publishCommand, readRedisStatus } from "../../shared/redis/index.js"; import { publishCommand, readRedisStatus } from "../../shared/redis/index.js";
@@ -31,29 +34,40 @@ export interface VoiceStatus {
activeChannelName: string | null; activeChannelName: string | null;
} }
export const DEFAULT_VOICE_STATUS: VoiceStatus = {
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
};
function readVoiceStatusFallback(): Promise<VoiceStatus> {
return readRedisStatus(VOICE_STATUS_KEY).then(
(cached) => (cached as unknown as VoiceStatus) ?? DEFAULT_VOICE_STATUS,
);
}
/** /**
* Get guilds — query from discord-gateway via Redis command for real names. * Get guilds — query from discord-gateway via Redis command for real names.
* Falls back to database (distinct guild_id from messages) if gateway unreachable. * Falls back to database (distinct guild_id from messages) if gateway unreachable.
*/ */
export async function getGuilds(): Promise<Guild[]> { export async function getGuilds(): Promise<Guild[]> {
logger.info("getGuilds called"); logger.info("getGuilds called");
const reply = await publishCommand<Guild[]>(COMMAND_GUILDS_LIST, {}); return tryCommandThenFallback(
if (reply?.success && reply.data && reply.data.length > 0) return reply.data; () => publishCommand<Guild[]>(COMMAND_GUILDS_LIST, {}),
async () => {
// Fallback: Postgres with synthetic names const pool = getPool();
logger.warn( const { rows } = await pool.query(
"discord-gateway unreachable, falling back to Postgres for guilds", `SELECT DISTINCT guild_id FROM messages ORDER BY guild_id`,
);
return rows.map((row: Record<string, unknown>) => ({
id: String(row.guild_id ?? ""),
name: `Guild ${String(row.guild_id).slice(0, 8)}`,
icon: null,
}));
},
"getGuilds",
); );
const pool = getPool();
const { rows } = await pool.query(
`SELECT DISTINCT guild_id FROM messages ORDER BY guild_id`,
);
return rows.map((row: Record<string, unknown>) => ({
id: String(row.guild_id ?? ""),
name: `Guild ${String(row.guild_id).slice(0, 8)}`,
icon: null,
}));
} }
/** /**
@@ -62,27 +76,22 @@ export async function getGuilds(): Promise<Guild[]> {
*/ */
export async function getTextChannels(guildId: string): Promise<Channel[]> { export async function getTextChannels(guildId: string): Promise<Channel[]> {
logger.info({ guildId }, "getTextChannels called"); logger.info({ guildId }, "getTextChannels called");
const reply = await publishCommand<Channel[]>(COMMAND_GUILDS_TEXT_CHANNELS, { return tryCommandThenFallback(
guildId, () => publishCommand<Channel[]>(COMMAND_GUILDS_TEXT_CHANNELS, { guildId }),
}); async () => {
if (reply?.success && reply.data && reply.data.length > 0) return reply.data; const pool = getPool();
const { rows } = await pool.query(
// Fallback: Postgres with synthetic names `SELECT DISTINCT channel_id FROM messages WHERE guild_id = $1 ORDER BY channel_id`,
logger.warn( [guildId],
{ guildId }, );
"discord-gateway unreachable, falling back to Postgres for text channels", return rows.map((row: Record<string, unknown>) => ({
id: String(row.channel_id ?? ""),
name: `Channel ${String(row.channel_id).slice(0, 8)}`,
type: "text" as const,
}));
},
"getTextChannels",
); );
const pool = getPool();
const { rows } = await pool.query(
`SELECT DISTINCT channel_id FROM messages WHERE guild_id = $1 ORDER BY channel_id`,
[guildId],
);
return rows.map((row: Record<string, unknown>) => ({
id: String(row.channel_id ?? ""),
name: `Channel ${String(row.channel_id).slice(0, 8)}`,
type: "text" as const,
}));
} }
/** /**
@@ -102,13 +111,7 @@ export async function getVoiceChannels(guildId: string): Promise<Channel[]> {
export async function getVoiceStatus(): Promise<VoiceStatus> { export async function getVoiceStatus(): Promise<VoiceStatus> {
logger.debug("getVoiceStatus called"); logger.debug("getVoiceStatus called");
const cached = await readRedisStatus(VOICE_STATUS_KEY); const cached = await readRedisStatus(VOICE_STATUS_KEY);
if (cached) return cached as unknown as VoiceStatus; return (cached as unknown as VoiceStatus) ?? DEFAULT_VOICE_STATUS;
return {
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
};
} }
/** /**
@@ -119,21 +122,14 @@ export async function connectVoice(
channelId: string, channelId: string,
): Promise<VoiceStatus> { ): Promise<VoiceStatus> {
logger.info({ guildId, channelId }, "connectVoice called"); logger.info({ guildId, channelId }, "connectVoice called");
const reply = await publishCommand<VoiceStatus>(COMMAND_VOICE_CONNECT, { return tryCommandThenFallback(
guildId, () =>
channelId, publishCommand<VoiceStatus>(COMMAND_VOICE_CONNECT, {
}); guildId,
if (reply?.success && reply.data) return reply.data; channelId,
}),
// Fallback: read from Redis status key () => readVoiceStatusFallback(),
const cached = await readRedisStatus(VOICE_STATUS_KEY); "connectVoice",
return (
(cached as unknown as VoiceStatus) ?? {
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
}
); );
} }
@@ -142,16 +138,9 @@ export async function connectVoice(
*/ */
export async function disconnectVoice(): Promise<VoiceStatus> { export async function disconnectVoice(): Promise<VoiceStatus> {
logger.info("disconnectVoice called"); logger.info("disconnectVoice called");
const reply = await publishCommand<VoiceStatus>(COMMAND_VOICE_DISCONNECT, {}); return tryCommandThenFallback(
if (reply?.success && reply.data) return reply.data; () => publishCommand<VoiceStatus>(COMMAND_VOICE_DISCONNECT, {}),
() => readVoiceStatusFallback(),
const cached = await readRedisStatus(VOICE_STATUS_KEY); "disconnectVoice",
return (
(cached as unknown as VoiceStatus) ?? {
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
}
); );
} }
@@ -0,0 +1,26 @@
import type { CommandReply } from "@bete/shared";
import { createChildLogger } from "@bete/shared/logger";
import { publishCommand, readRedisStatus } from "./redis/index.js";
export { createChildLogger };
/**
* Attempt a Redis command first; if it fails or times out, fall back.
*
* @param commandFn - Function that issues the publishCommand and returns the reply.
* @param fallbackFn - Async fallback, typically reads from Redis status key.
* @param commandLabel - Label used for logging (e.g. "voice:connect").
*/
export async function tryCommandThenFallback<T>(
commandFn: () => Promise<CommandReply<T> | null>,
fallbackFn: () => Promise<T>,
commandLabel: string,
): Promise<T> {
const logger = createChildLogger(`command-helper:${commandLabel}`);
const reply = await commandFn();
if (reply?.success && reply.data !== undefined && reply.data !== null) {
return reply.data;
}
logger.warn("discord-gateway unreachable, falling back");
return fallbackFn();
}
@@ -0,0 +1,57 @@
// Shared message row mapper for backend repository modules
export interface MappedMessage {
id: string;
guild_id: string;
channel_id: string;
thread_id: string | null;
user_id: string;
username: string;
avatar_url: string | null;
content: string;
edited_content: string | null;
created_at: number;
edited_at: number | null;
deleted_at: number | null;
type: string;
metadata: string | null;
ai_status: string | null;
ai_moderation_flags: string | null;
ai_moderation_score: number | null;
ai_analysis: string | null;
ai_categories: string | null;
ai_severity: string | null;
ai_confidence: number | null;
ai_recommended_action: string | null;
ai_analyzed_at: number | null;
ai_error: string | null;
}
export function mapMessageRow(row: Record<string, unknown>): MappedMessage {
return {
id: String(row.id ?? ""),
guild_id: String(row.guild_id ?? ""),
channel_id: String(row.channel_id ?? ""),
thread_id: (row.thread_id as string | null) ?? null,
user_id: String(row.user_id ?? ""),
username: String(row.username ?? ""),
avatar_url: (row.avatar_url as string | null) ?? null,
content: String(row.content ?? ""),
edited_content: (row.edited_content as string | null) ?? null,
created_at: Number(row.created_at ?? 0),
edited_at: (row.edited_at as number | null) ?? null,
deleted_at: (row.deleted_at as number | null) ?? null,
type: String(row.type ?? "text"),
metadata: (row.metadata as string | null) ?? null,
ai_status: (row.ai_status as string | null) ?? null,
ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null,
ai_moderation_score: (row.ai_moderation_score as number | null) ?? null,
ai_analysis: (row.ai_analysis as string | null) ?? null,
ai_categories: (row.ai_categories as string | null) ?? null,
ai_severity: (row.ai_severity as string | null) ?? null,
ai_confidence: (row.ai_confidence as number | null) ?? null,
ai_recommended_action: (row.ai_recommended_action as string | null) ?? null,
ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null,
ai_error: (row.ai_error as string | null) ?? null,
};
}
+76 -32
View File
@@ -9,6 +9,10 @@
* broadcastMessageCreated(messageData); * broadcastMessageCreated(messageData);
*/ */
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("broadcast");
type BroadcastFn = (data: unknown) => void; type BroadcastFn = (data: unknown) => void;
type BroadcastRawFn = (type: string, data: unknown) => void; type BroadcastRawFn = (type: string, data: unknown) => void;
type BroadcastBinaryFn = (data: Buffer) => void; type BroadcastBinaryFn = (data: Buffer) => void;
@@ -30,63 +34,103 @@ export interface BroadcastFunctions {
binary: BroadcastBinaryFn; binary: BroadcastBinaryFn;
} }
const noop: BroadcastFn = () => {};
const noopRaw: BroadcastRawFn = () => {};
const noopBinary: BroadcastBinaryFn = () => {};
let _fns: BroadcastFunctions | null = null; let _fns: BroadcastFunctions | null = null;
let _enabled = true;
/** Enable or disable broadcast logging (disabled by default to reduce noise). */
export function setBroadcastLogging(enabled: boolean): void {
_enabled = enabled;
}
/** /**
* Inject broadcast functions from the WebSocket server initializer. * Inject broadcast functions from the WebSocket server initializer.
* Must be called once during server startup before any broadcast is used. * Must be called once during server startup before any broadcast is used.
*/ */
export function setBroadcastFunctions(fns: BroadcastFunctions): void { export function setBroadcastFunctions(fns: BroadcastFunctions): void {
_fns = fns; _fns = fns;
logger.info("Broadcast functions initialized");
} }
/** Clear injected functions (used during cleanup). */ /** Clear injected functions (used during cleanup). */
export function clearBroadcastFunctions(): void { export function clearBroadcastFunctions(): void {
_fns = null; _fns = null;
logger.info("Broadcast functions cleared");
} }
export const broadcastMessageCreated: BroadcastFn = (data) => function logBroadcast(name: string, data: unknown): void {
(_fns?.messageCreated ?? noop)(data); if (!_enabled) return;
// Avoid logging binary or PCM data due to volume
if (name === "voice_pcm_data" || name === "binary") return;
logger.debug({ event: name }, "Broadcasting event");
}
export const broadcastMessageUpdated: BroadcastFn = (data) => export const broadcastMessageCreated: BroadcastFn = (data) => {
(_fns?.messageUpdated ?? noop)(data); logBroadcast("message_created", data);
_fns?.messageCreated?.(data);
};
export const broadcastMessageDeleted: BroadcastFn = (data) => export const broadcastMessageUpdated: BroadcastFn = (data) => {
(_fns?.messageDeleted ?? noop)(data); logBroadcast("message_updated", data);
_fns?.messageUpdated?.(data);
};
export const broadcastAttachmentCreated: BroadcastFn = (data) => export const broadcastMessageDeleted: BroadcastFn = (data) => {
(_fns?.attachmentCreated ?? noop)(data); logBroadcast("message_deleted", data);
_fns?.messageDeleted?.(data);
};
export const broadcastAttachmentUploaded: BroadcastFn = (data) => export const broadcastAttachmentCreated: BroadcastFn = (data) => {
(_fns?.attachmentUploaded ?? noop)(data); logBroadcast("attachment_created", data);
_fns?.attachmentCreated?.(data);
};
export const broadcastMessageAnalyzed: BroadcastFn = (data) => export const broadcastAttachmentUploaded: BroadcastFn = (data) => {
(_fns?.messageAnalyzed ?? noop)(data); logBroadcast("attachment_uploaded", data);
_fns?.attachmentUploaded?.(data);
};
export const broadcastVoiceRecordingStarted: BroadcastFn = (data) => export const broadcastMessageAnalyzed: BroadcastFn = (data) => {
(_fns?.voiceRecordingStarted ?? noop)(data); logBroadcast("message_analyzed", data);
_fns?.messageAnalyzed?.(data);
};
export const broadcastVoiceRecordingStopped: BroadcastFn = (data) => export const broadcastVoiceRecordingStarted: BroadcastFn = (data) => {
(_fns?.voiceRecordingStopped ?? noop)(data); logBroadcast("voice_recording_started", data);
_fns?.voiceRecordingStarted?.(data);
};
export const broadcastVoiceRecordingUploaded: BroadcastFn = (data) => export const broadcastVoiceRecordingStopped: BroadcastFn = (data) => {
(_fns?.voiceRecordingUploaded ?? noop)(data); logBroadcast("voice_recording_stopped", data);
_fns?.voiceRecordingStopped?.(data);
};
export const broadcastVoicePcmData: BroadcastFn = (data) => export const broadcastVoiceRecordingUploaded: BroadcastFn = (data) => {
(_fns?.voicePcmData ?? noop)(data); logBroadcast("voice_recording_uploaded", data);
_fns?.voiceRecordingUploaded?.(data);
};
export const broadcastVoiceActiveUser: BroadcastFn = (data) => export const broadcastVoicePcmData: BroadcastFn = (data) => {
(_fns?.voiceActiveUser ?? noop)(data); // PCM data is high-volume; logging is skipped unconditionally
_fns?.voicePcmData?.(data);
};
export const broadcastAnalysisQueueStatus: BroadcastFn = (data) => export const broadcastVoiceActiveUser: BroadcastFn = (data) => {
(_fns?.analysisQueueStatus ?? noop)(data); logBroadcast("voice_active_user", data);
_fns?.voiceActiveUser?.(data);
};
export const broadcastRaw: BroadcastRawFn = (type, data) => export const broadcastAnalysisQueueStatus: BroadcastFn = (data) => {
(_fns?.raw ?? noopRaw)(type, data); logBroadcast("analysis_queue_status", data);
_fns?.analysisQueueStatus?.(data);
};
export const broadcastBinary: BroadcastBinaryFn = (data) => export const broadcastRaw: BroadcastRawFn = (type, data) => {
(_fns?.binary ?? noopBinary)(data); logBroadcast(type, data);
_fns?.raw?.(type, data);
};
export const broadcastBinary: BroadcastBinaryFn = (data) => {
// Binary data is high-volume; logging is skipped unconditionally
_fns?.binary?.(data);
};
@@ -1,3 +1,4 @@
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/config.js"; import { config } from "../../shared/config/config.js";
import { initializeDatabase } from "../../shared/database/drizzle.js"; import { initializeDatabase } from "../../shared/database/drizzle.js";
import { import {
@@ -15,6 +16,8 @@ import {
runSimpleTextFallback, runSimpleTextFallback,
} from "./llmModerationClient.js"; } from "./llmModerationClient.js";
const logger = createChildLogger("aiAnalysisWorker");
let dbInitialized = false; let dbInitialized = false;
let dbInitPromise: Promise<any> | null = null; let dbInitPromise: Promise<any> | null = null;
@@ -70,13 +73,9 @@ export default async function workerRouter(
if (!config.AI_LLM_API_KEY) { if (!config.AI_LLM_API_KEY) {
const errorMsg = const errorMsg =
"AI_LLM_API_KEY is missing from environment. Worker cannot process moderation requests without credentials."; "AI_LLM_API_KEY is missing from environment. Worker cannot process moderation requests without credentials.";
console.error( logger.error(
JSON.stringify({ { error: errorMsg },
level: "ERROR", "AI_LLM_API_KEY is missing from environment",
context: "aiAnalysisWorker",
error: errorMsg,
timestamp: new Date().toISOString(),
}),
); );
if (job.type === "batch") { if (job.type === "batch") {
@@ -113,15 +112,9 @@ export default async function workerRouter(
} catch (error) { } catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error); const errorMessage = error instanceof Error ? error.message : String(error);
const errorStack = error instanceof Error ? error.stack : undefined; const errorStack = error instanceof Error ? error.stack : undefined;
console.error( logger.error(
JSON.stringify({ { type: job.type, error: errorMessage, stack: errorStack },
level: "ERROR", "Worker job failed",
context: "aiAnalysisWorker",
type: job.type,
error: errorMessage,
stack: errorStack,
timestamp: new Date().toISOString(),
}),
); );
if (job.type === "batch") { if (job.type === "batch") {
return { return {
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,399 @@
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/config.js";
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
import { updateMessagesAIAnalysisBulk } from "../message-capture/messageStore.js";
import type { MessageRecord } from "../message-capture/types.js";
import {
broadcastAnalysisCompleted,
conversationErrorCooldown,
conversationProcessing,
LAST_ERROR,
recordConversationBatchFailure,
resetConversationBatchFailures,
scheduleAutoDelete,
workerPool,
} from "./circuitBreaker.js";
import { estimateTokens } from "./conversationContext.js";
import { enqueueIndividualFallbacks } from "./individualFallbackProcessor.js";
const logger = createChildLogger("batch-processor");
export interface AnalysisWorkerResponse {
ok: boolean;
conversationKey: string;
rows: MessageRecord[];
error?: string;
}
// ---------------------------------------------------------------------------
// Observability
// ---------------------------------------------------------------------------
export let activeRequests = 0;
// ---------------------------------------------------------------------------
// Exported helpers
// ---------------------------------------------------------------------------
/**
* Picks a batch of messages within a token budget.
* `tokensPerMessage` accounts for JSON structure overhead around each entry.
* Uses a rough character-based token estimate (avoids async formatMessageForPrompt
* since this function runs in a synchronous promise chain).
*/
export function pickBatchWithinBudget(
messages: MessageRecord[],
maxTokens: number,
tokensPerMessage: number,
): MessageRecord[] {
const batch: MessageRecord[] = [];
let usedTokens = 0;
for (const msg of messages) {
const content = msg.edited_content ?? msg.content;
// Accurate token count via tiktoken (+ overhead for JSON structure)
const msgTokens = estimateTokens(content) + tokensPerMessage;
if (usedTokens + msgTokens <= maxTokens) {
batch.push(msg);
usedTokens += msgTokens;
}
}
return batch;
}
// ---------------------------------------------------------------------------
// Age-restricted message helpers
// ---------------------------------------------------------------------------
export function isAgeRestrictedMessage(message: MessageRecord): boolean {
return isAgeRestrictedMetadata(message.metadata);
}
export function buildAgeRestrictedSkipResult(): {
status: "clean";
flags: string | null;
score: number;
analysis: string;
categories: string[];
severity: "none";
confidence: number;
recommendedAction: "none";
analyzedAt: number;
error: null;
} {
return {
status: "clean",
flags: JSON.stringify(["age_restricted"]),
score: 0,
analysis: "Skipped moderation for age-restricted content.",
categories: ["age_restricted"],
severity: "none",
confidence: 1,
recommendedAction: "none",
analyzedAt: Date.now(),
error: null,
};
}
export async function skipAgeRestrictedMessages(
messages: MessageRecord[],
): Promise<MessageRecord[]> {
const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage);
if (ageRestrictedMessages.length === 0) {
return messages;
}
const skippedRows = await updateMessagesAIAnalysisBulk(
ageRestrictedMessages.map((message) => ({
messageId: message.id,
result: buildAgeRestrictedSkipResult(),
})),
);
for (const row of skippedRows) {
broadcastAnalysisCompleted(row);
}
const skippedIds = new Set(
ageRestrictedMessages.map((message) => message.id),
);
return messages.filter((message) => !skippedIds.has(message.id));
}
// ---------------------------------------------------------------------------
// Batch pipeline
// ---------------------------------------------------------------------------
async function postBatchReputationUpdate(rows: MessageRecord[]): Promise<void> {
for (const row of rows) {
if (row.ai_status === "clean") {
import("./userReputationStore.js")
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
.catch((e) =>
logger.error({ error: e }, "Failed to record clean message streak"),
);
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
import("./userReputationStore.js")
.then((store) =>
store.recordInfraction(
row.user_id,
row.guild_id,
row.ai_severity as "low" | "medium" | "high" | "critical",
),
)
.catch((e) =>
logger.error({ error: e }, "Failed to record infraction penalty"),
);
}
}
}
export async function processBatch(
conversationKey: string,
messages: MessageRecord[],
processingStartedAt: number,
): Promise<void> {
if (messages.length === 0) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
return;
}
const cooldownUntil = conversationErrorCooldown.get(conversationKey) ?? 0;
if (Date.now() < cooldownUntil) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
return;
}
activeRequests++;
let shouldScheduleNext = false;
try {
const result = (await workerPool.run({
type: "batch",
conversationKey,
messages,
})) as AnalysisWorkerResponse;
// Do not broadcast or auto-delete if it's an API failure that will be reverted.
for (const row of result.rows) {
let isApiFailure = false;
if (row.ai_status === "error") {
try {
const flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
isApiFailure = flags.includes("analysis_api_failed");
} catch {}
}
if (!isApiFailure) {
broadcastAnalysisCompleted(row);
scheduleAutoDelete(row);
}
}
// Post-batch reputation updates (fire-and-forget)
postBatchReputationUpdate(
result.rows.filter((r) => {
if (r.ai_status === "error") {
try {
const flags = JSON.parse(r.ai_moderation_flags ?? "[]") as string[];
return !flags.includes("analysis_api_failed");
} catch {
return false;
}
}
return true;
}),
);
if (!result.ok) {
recordConversationBatchFailure(conversationKey);
// Batch failed entirely -- fall back all messages to individual queue
logger.warn(
{
conversationKey,
messageCount: messages.length,
error: result.error,
},
"Batch failed entirely -- routing all messages to individual fallback queue",
);
enqueueIndividualFallbacks(messages);
LAST_ERROR.value = result.error ?? "Analysis worker failed";
conversationErrorCooldown.set(
conversationKey,
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
);
logger.error(
{
conversationKey,
error: LAST_ERROR.value,
messageCount: messages.length,
messageIds: messages.map((m) => m.id),
cooldownUntil: new Date(
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
).toISOString(),
timestamp: new Date().toISOString(),
},
"Batch analysis failed, will retry after cooldown",
);
return;
}
// Batch succeeded -- check for messages the LLM silently dropped or failed
const incompleteMessages: MessageRecord[] = [];
const parseFailedMessages: MessageRecord[] = [];
const apiFailedMessages: MessageRecord[] = [];
for (const msg of messages) {
const row = result.rows.find((r) => r.id === msg.id);
if (!row) {
incompleteMessages.push(msg);
continue;
}
if (row.ai_status === "error") {
let flags: string[] = [];
try {
flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
} catch {}
if (flags.includes("analysis_incomplete")) {
incompleteMessages.push(msg);
} else if (flags.includes("analysis_parse_failed")) {
parseFailedMessages.push(msg);
} else if (flags.includes("analysis_api_failed")) {
apiFailedMessages.push(msg);
}
}
}
const messagesForIndividualQueue = [
...incompleteMessages,
...parseFailedMessages,
];
if (messagesForIndividualQueue.length > 0) {
logger.warn(
{
conversationKey,
count: messagesForIndividualQueue.length,
ids: messagesForIndividualQueue.map((m) => m.id),
totalBatchSize: messages.length,
},
"Batch returned incomplete or unparseable results -- fanning out to individual fallback queue",
);
enqueueIndividualFallbacks(messagesForIndividualQueue);
}
if (apiFailedMessages.length > 0) {
logger.warn(
{
conversationKey,
count: apiFailedMessages.length,
ids: apiFailedMessages.map((m) => m.id),
},
"Batch returned API failures -- reverting to pending to put back in queue",
);
// Revert to pending so they are picked up again
const revertedRows = await updateMessagesAIAnalysisBulk(
apiFailedMessages.map((msg) => ({
messageId: msg.id,
result: {
status: "pending",
flags: null,
score: null,
analysis: null,
categories: null,
severity: null,
confidence: null,
recommendedAction: null,
analyzedAt: null,
error: null,
},
})),
).catch((err) => {
logger.error(
{ error: String(err) },
"Failed to revert API failures to pending",
);
return [];
});
for (const row of revertedRows) {
broadcastAnalysisCompleted(row);
}
// Trigger conversation cooldown
recordConversationBatchFailure(conversationKey);
const existingCooldown =
conversationErrorCooldown.get(conversationKey) ?? 0;
const newCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS;
if (newCooldown > existingCooldown) {
conversationErrorCooldown.set(conversationKey, newCooldown);
}
// Release the processing lock immediately so the cooldown timer controls retry
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
// Do NOT schedule next -- let the cooldown gate it
shouldScheduleNext = false;
}
if (apiFailedMessages.length === 0) {
resetConversationBatchFailures(conversationKey);
conversationErrorCooldown.delete(conversationKey);
}
shouldScheduleNext = true;
} catch (error) {
recordConversationBatchFailure(conversationKey);
logger.warn(
{ conversationKey, messageCount: messages.length },
"Batch threw exception -- routing all messages to individual fallback queue",
);
enqueueIndividualFallbacks(messages);
LAST_ERROR.value = error instanceof Error ? error.message : String(error);
const errorStack = error instanceof Error ? error.stack : undefined;
const existingCatchCooldown =
conversationErrorCooldown.get(conversationKey) ?? 0;
const newCatchCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS;
if (newCatchCooldown > existingCatchCooldown) {
conversationErrorCooldown.set(conversationKey, newCatchCooldown);
}
logger.error(
{
conversationKey,
error: LAST_ERROR.value,
stack: errorStack,
messageCount: messages.length,
messageIds: messages.map((m) => m.id),
cooldownUntil: new Date(
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
).toISOString(),
timestamp: new Date().toISOString(),
},
"Analysis worker failed, will retry after cooldown",
);
} finally {
activeRequests--;
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
if (shouldScheduleNext) {
setImmediate(() => {
// Dynamic import to avoid circular dependency at module scope
import("./batchScheduler.js").then((m) =>
m.scheduleConversationAnalysis(conversationKey),
);
});
}
}
}
@@ -0,0 +1,140 @@
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/config.js";
import { getPendingMessagesByConversation } from "../message-capture/messageStore.js";
import type { MessageRecord } from "../message-capture/types.js";
import {
pickBatchWithinBudget,
processBatch,
skipAgeRestrictedMessages,
} from "./batchProcessor.js";
import {
conversationConsecutiveErrors,
conversationDebounceTimers,
conversationErrorCooldown,
conversationProcessing,
isConversationProcessingLocked,
MAX_CONSECUTIVE_ERRORS,
} from "./circuitBreaker.js";
const logger = createChildLogger("batch-scheduler");
// ---------------------------------------------------------------------------
// Scheduling
// ---------------------------------------------------------------------------
/**
* Schedules a debounced analysis run for a conversation.
*
* FIX #3: The async work inside setTimeout is now wrapped in an explicit
* .catch() so DB errors don't produce unhandled promise rejections.
* FIX #6: Calls pickBatchWithinBudget after fetching messages so token budget
* is respected before handing the batch to the LLM.
* FIX #7: Unified single-timer path -- always clear-and-reset one timer per
* conversation key regardless of whether a cooldown is active. The delay is
* simply max(cooldownRemainder+500, debounce) so the same timer serves both
* the "throttled by error cooldown" and "normal debounce" cases, eliminating
* the previous two-path logic that could leave both timers live simultaneously.
*/
export function scheduleConversationAnalysis(conversationKey: string): void {
if (isConversationProcessingLocked(conversationKey)) {
return;
}
const convoCooldown = conversationErrorCooldown.get(conversationKey) ?? 0;
const convoErrors = conversationConsecutiveErrors.get(conversationKey) ?? 0;
// Hard-block: circuit breaker threshold reached AND cooldown still active.
if (convoErrors >= MAX_CONSECUTIVE_ERRORS && Date.now() < convoCooldown) {
return;
}
// Unified delay: honour the cooldown window if active, otherwise use the
// normal debounce interval. Always clear-and-reset so only ONE timer is
// ever pending per conversation key regardless of call source.
const now = Date.now();
const delayMs =
convoCooldown > now
? convoCooldown - now + 500
: config.AI_ANALYSIS_DEBOUNCE_MS;
const existingTimer = conversationDebounceTimers.get(conversationKey);
if (existingTimer) {
clearTimeout(existingTimer);
}
const timer = setTimeout(() => {
conversationDebounceTimers.delete(conversationKey);
// FIX TOCTOU: Set lock synchronously BEFORE the async DB fetch starts
if (isConversationProcessingLocked(conversationKey)) {
return;
}
const processingStartedAt = Date.now();
conversationProcessing.set(conversationKey, processingStartedAt);
// FIX #3: explicit .catch() -- no async arrow function to avoid unhandled rejection.
getPendingMessagesByConversation(
conversationKey,
config.AI_ANALYSIS_MAX_BATCH_SIZE,
)
.then(async (messages: MessageRecord[]) => {
if (messages.length === 0) {
if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey);
}
return;
}
const processableMessages = await skipAgeRestrictedMessages(messages);
if (processableMessages.length === 0) {
if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey);
}
return;
}
// FIX #6: trim to token budget before sending to LLM.
let trimmed = pickBatchWithinBudget(
processableMessages,
config.AI_ANALYSIS_MAX_TARGET_TOKENS,
50,
);
// FIX #10: if every message individually exceeds the token budget,
// fall back to the first message alone.
if (trimmed.length === 0 && processableMessages.length > 0) {
trimmed = processableMessages.slice(0, 1);
logger.warn(
{
conversationKey,
messageId: processableMessages[0]?.id,
tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS,
},
"All messages exceed token budget -- processing first message alone to avoid stuck-pending deadlock",
);
}
return processBatch(conversationKey, trimmed, processingStartedAt);
})
.catch((err: unknown) => {
if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey);
}
logger.error(
{
conversationKey,
error: err instanceof Error ? err.message : String(err),
},
"Failed to fetch or dispatch pending messages for scheduled analysis",
);
});
}, delayMs);
conversationDebounceTimers.set(conversationKey, timer);
}
@@ -1,3 +1,4 @@
import { createChildLogger } from "@bete/shared/logger";
import { eq } from "drizzle-orm"; import { eq } from "drizzle-orm";
import { getDatabase } from "../../shared/database/drizzle.js"; import { getDatabase } from "../../shared/database/drizzle.js";
import { import {
@@ -5,6 +6,8 @@ import {
channelCulturesTable, channelCulturesTable,
} from "../../shared/database/schema.js"; } from "../../shared/database/schema.js";
const logger = createChildLogger("channelCultureStore");
/** /**
* Fetch the AI-generated culture summary for a channel. * Fetch the AI-generated culture summary for a channel.
*/ */
@@ -18,6 +21,11 @@ export async function getChannelCulture(
.where(eq(channelCulturesTable.channel_id, channelId)) .where(eq(channelCulturesTable.channel_id, channelId))
.limit(1); .limit(1);
if (existing[0]) {
logger.debug({ channelId }, "Channel culture lookup: found");
} else {
logger.debug({ channelId }, "Channel culture lookup: not found");
}
return existing[0] || null; return existing[0] || null;
} }
@@ -46,4 +54,9 @@ export async function updateChannelCulture(
last_analyzed_at: Date.now(), last_analyzed_at: Date.now(),
}, },
}); });
logger.debug(
{ channelId, guildId, cultureSummary },
"Channel culture updated",
);
} }
@@ -0,0 +1,235 @@
import { existsSync } from "node:fs";
import { availableParallelism } from "node:os";
import { fileURLToPath } from "node:url";
import { createChildLogger } from "@bete/shared/logger";
import type { Client } from "discord.js-selfbot-v13";
import { LRUCache } from "lru-cache";
import { Piscina } from "piscina";
import { config } from "../../shared/config/config.js";
import type { EventBroadcaster } from "../event-broadcaster/index.js";
import type { MessageRecord } from "../message-capture/types.js";
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
const logger = createChildLogger("circuit-breaker");
// ---------------------------------------------------------------------------
// Piscina worker pool (shared by batch + individual pipelines)
// ---------------------------------------------------------------------------
function getAnalysisWorkerUrl(): URL {
const candidates = [
new URL("./aiAnalysisWorker.js", import.meta.url),
new URL("../aiAnalysisWorker.js", import.meta.url),
new URL("./aiAnalysisWorker.ts", import.meta.url),
];
for (const candidate of candidates) {
if (existsSync(fileURLToPath(candidate))) {
return candidate;
}
}
return candidates[2];
}
export const workerPool = new Piscina({
filename: fileURLToPath(getAnalysisWorkerUrl()),
execArgv: process.execArgv,
maxThreads: config.PISCINA_MAX_THREADS ?? availableParallelism(),
});
/**
* Gets the conversation key for a message (thread_id or channel_id).
*/
export function getConversationKey(message: MessageRecord): string {
return message.thread_id || message.channel_id;
}
// ---------------------------------------------------------------------------
// Shared observable state
// ---------------------------------------------------------------------------
/** Redis EventBroadcaster -- set externally so sub-modules can publish events. */
export let _redisEventBroadcaster: EventBroadcaster | undefined;
/** Discord client reference -- needed for auto-delete actions. */
export let moderationClient: Client | undefined;
export function setSharedEventBroadcaster(
eb: EventBroadcaster | undefined,
): void {
_redisEventBroadcaster = eb;
}
export function setModerationClient(mc: Client | undefined): void {
moderationClient = mc;
}
/**
* Per-message in-flight guard for the auto-delete side-effect.
* (LRU-backed to prevent unbounded growth)
*/
export const autoDeleteInFlight = new LRUCache<string, true>({ max: 10000 });
/** Last recorded error across all pipelines. */
export const LAST_ERROR: { value: string | null } = { value: null };
// ---------------------------------------------------------------------------
// Batch circuit breaker state
// ---------------------------------------------------------------------------
export const conversationConsecutiveErrors = new LRUCache<string, number>({
max: 10000,
});
export const MAX_CONSECUTIVE_ERRORS = 5;
export const CONVERSATION_CB_COOLDOWN_MS = 60000;
export const conversationErrorCooldown = new LRUCache<string, number>({
max: 10000,
});
// ---------------------------------------------------------------------------
// Scheduling / timing state (shared so sub-modules can access without cycles)
// ---------------------------------------------------------------------------
/** Debounce timer handle per conversation key. */
export const conversationDebounceTimers = new LRUCache<string, NodeJS.Timeout>({
max: 10000,
dispose: (value) => {
clearTimeout(value);
},
});
/** Timestamp of when processing started per conversation key. */
export const conversationProcessing = new LRUCache<string, number>({
max: 10000,
});
// ---------------------------------------------------------------------------
// Conversation lock helper
// ---------------------------------------------------------------------------
export function isConversationProcessingLocked(
conversationKey: string,
): boolean {
const startedAt = conversationProcessing.get(conversationKey);
return Boolean(
startedAt &&
Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS,
);
}
// ---------------------------------------------------------------------------
// Alert system
// ---------------------------------------------------------------------------
export type CircuitBreakerAlert = {
type: "conversation_cb" | "individual_cb" | "sustained_error";
conversationKey?: string;
consecutiveErrors: number;
message: string;
lastError?: string | null;
};
const alertHandlers: Array<(alert: CircuitBreakerAlert) => void> = [];
/**
* Register an alert handler (e.g., for webhook integration).
*/
export function onCircuitBreakerAlert(
handler: (alert: CircuitBreakerAlert) => void,
): void {
alertHandlers.push(handler);
}
export function fireAlert(alert: CircuitBreakerAlert): void {
logger.warn(alert, `CB Alert: ${alert.type} -- ${alert.message}`);
for (const handler of alertHandlers) {
try {
handler(alert);
} catch {
// handler errors are non-critical
}
}
}
// ---------------------------------------------------------------------------
// Circuit breaker helpers
// ---------------------------------------------------------------------------
export function recordConversationBatchFailure(conversationKey: string): void {
const nextCount =
(conversationConsecutiveErrors.get(conversationKey) ?? 0) + 1;
conversationConsecutiveErrors.set(conversationKey, nextCount);
if (nextCount >= MAX_CONSECUTIVE_ERRORS) {
conversationErrorCooldown.set(
conversationKey,
Date.now() + CONVERSATION_CB_COOLDOWN_MS,
);
fireAlert({
type: "conversation_cb",
conversationKey,
consecutiveErrors: nextCount,
message: `Conversation ${conversationKey} circuit breaker triggered after ${nextCount} consecutive errors`,
lastError: LAST_ERROR.value,
});
conversationConsecutiveErrors.set(conversationKey, 0);
}
}
export function resetConversationBatchFailures(conversationKey: string): void {
conversationConsecutiveErrors.delete(conversationKey);
}
// ---------------------------------------------------------------------------
// Broadcast & auto-delete helpers
// ---------------------------------------------------------------------------
export function broadcastAnalysisCompleted(row: MessageRecord): void {
if (_redisEventBroadcaster) {
_redisEventBroadcaster.messageAnalyzed(row).catch((err: unknown) =>
logger.warn(
{
messageId: row.id,
error: err instanceof Error ? err.message : String(err),
},
"Failed to publish message_analyzed via Redis EventBroadcaster",
),
);
}
}
export function scheduleAutoDelete(row: MessageRecord): void {
if (row.ai_status !== "flagged" && row.ai_status !== "warn") return;
if (autoDeleteInFlight.has(row.id)) {
logger.debug(
{ messageId: row.id },
"Auto-delete skipped: already in-flight for this message",
);
return;
}
autoDeleteInFlight.set(row.id, true);
const run = () => {
attemptAutoDeleteFlaggedMessage(moderationClient, row)
.catch((error: unknown) => {
logger.error(
{
messageId: row.id,
error: error instanceof Error ? error.message : String(error),
},
"Unexpected auto-delete error",
);
})
.finally(() => {
autoDeleteInFlight.delete(row.id);
});
};
if (config.AUTO_DELETE_FLAGGED_DELAY_MS > 0) {
setTimeout(run, config.AUTO_DELETE_FLAGGED_DELAY_MS);
return;
}
setImmediate(run);
}
@@ -1,6 +1,9 @@
import { createChildLogger } from "@bete/shared/logger";
import pLimit from "p-limit"; import pLimit from "p-limit";
import { config } from "../../shared/config/config.js"; import { config } from "../../shared/config/config.js";
const logger = createChildLogger("concurrencyLimiter");
/** /**
* Concurrency limiter for LLM API calls. * Concurrency limiter for LLM API calls.
* *
@@ -9,6 +12,42 @@ import { config } from "../../shared/config/config.js";
*/ */
const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5); const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5);
let activeCount = 0;
let pendingCount = 0;
// Track queue state changes for logging
function updateCounts(): void {
// p-limit exposes queueSize and activeCount via constructor internals,
// but we track via our wrapper to avoid depending on internals.
}
export async function withLlmConcurrency<T>(fn: () => Promise<T>): Promise<T> { export async function withLlmConcurrency<T>(fn: () => Promise<T>): Promise<T> {
return llmSemaphore(fn); const queuedAt = activeCount + pendingCount;
pendingCount++;
logger.debug(
{ activeCount, pendingCount, maxConcurrent: config.AI_LLM_MAX_CONCURRENT },
"Queuing LLM request",
);
return llmSemaphore(async () => {
pendingCount--;
activeCount++;
if (activeCount >= (config.AI_LLM_MAX_CONCURRENT ?? 5)) {
logger.warn(
{ activeCount, maxConcurrent: config.AI_LLM_MAX_CONCURRENT },
"LLM concurrency limit reached",
);
}
try {
return await fn();
} finally {
activeCount--;
logger.debug(
{ activeCount, pendingCount },
"LLM request completed, concurrency slot released",
);
}
});
} }
@@ -1,7 +1,10 @@
import { createChildLogger } from "@bete/shared/logger";
import { encoding_for_model as encodingForModel } from "tiktoken"; import { encoding_for_model as encodingForModel } from "tiktoken";
import { formatMediaEvidenceForPrompt } from "../message-capture/messageMetadata.js"; import { formatMediaEvidenceForPrompt } from "../message-capture/messageMetadata.js";
import type { MessageRecord } from "../message-capture/types.js"; import type { MessageRecord } from "../message-capture/types.js";
const logger = createChildLogger("conversationContext");
export interface ConversationContextInput { export interface ConversationContextInput {
contextBefore: MessageRecord[]; contextBefore: MessageRecord[];
targets: MessageRecord[]; targets: MessageRecord[];
@@ -29,7 +32,12 @@ function formatTimestamp(ms: number): string {
*/ */
export function estimateTokens(text: string): number { export function estimateTokens(text: string): number {
// Use tiktoken for accurate token counting (+15 overhead for JSON structure) // Use tiktoken for accurate token counting (+15 overhead for JSON structure)
return getEncoder().encode(text).length + 15; const tokens = getEncoder().encode(text).length + 15;
logger.debug(
{ tokenEstimate: tokens, textLength: text.length },
"Estimated tokens for text",
);
return tokens;
} }
/** /**
@@ -81,5 +89,14 @@ export function buildConversationContext(
} }
} }
logger.debug(
{
targetCount: targets.length,
contextCount: selectedContextLines.length,
usedTokens,
maxTokens,
},
"Conversation context built",
);
return selectedContextLines; return selectedContextLines;
} }
@@ -0,0 +1,76 @@
import { createChildLogger } from "@bete/shared/logger";
const log = createChildLogger("imageMimeSniffer");
/**
* Sniff the first bytes of a buffer to determine if it is a supported image
* format. Returns the canonical MIME type string on success, or null if the
* bytes are not a recognizable image.
*/
export function sniffImageMimeType(buf: Buffer): string | null {
if (buf.length < 12) return null;
if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {
return "image/jpeg";
}
if (
buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47 &&
buf[4] === 0x0d &&
buf[5] === 0x0a &&
buf[6] === 0x1a &&
buf[7] === 0x0a
) {
return "image/png";
}
if (
buf[0] === 0x47 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x38
) {
return "image/gif";
}
if (
buf[0] === 0x52 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x46 &&
buf[8] === 0x57 &&
buf[9] === 0x45 &&
buf[10] === 0x42 &&
buf[11] === 0x50
) {
return "image/webp";
}
if (
buf.length >= 12 &&
buf[4] === 0x66 &&
buf[5] === 0x74 &&
buf[6] === 0x79 &&
buf[7] === 0x70
) {
const brand = buf.subarray(8, 12).toString("ascii");
if (brand.startsWith("avif") || brand.startsWith("avis")) {
return "image/avif";
}
if (
brand.startsWith("mif1") ||
brand.startsWith("heic") ||
brand.startsWith("heis")
) {
return "image/heic";
}
}
return null;
}
// Keep log referenced so TS does not tree-shake the logger init
log.debug("imageMimeSniffer loaded");
@@ -0,0 +1,346 @@
import { createChildLogger } from "@bete/shared/logger";
import { LRUCache } from "lru-cache";
import { config } from "../../shared/config/config.js";
import { updateMessagesAIAnalysisBulk } from "../message-capture/messageStore.js";
import type {
AnalysisResult,
MessageRecord,
} from "../message-capture/types.js";
import {
broadcastAnalysisCompleted,
fireAlert,
getConversationKey,
LAST_ERROR,
scheduleAutoDelete,
workerPool,
} from "./circuitBreaker.js";
import { logModerationError } from "./responseLogger.js";
const logger = createChildLogger("individual-fallback");
// ---------------------------------------------------------------------------
// Individual fallback queue state
// ---------------------------------------------------------------------------
/** IDs currently being processed one-by-one (LRU-backed, max 10k entries). */
export const individualInFlight = new LRUCache<string, true>({ max: 10000 });
/**
* Per-conversation count of in-flight individual messages.
* (LRU-backed to prevent unbounded growth)
*/
export const individualInFlightByConversation = new LRUCache<string, number>({
max: 10000,
});
/** Last-touched timestamp for pruning stale entries (LRU-backed). */
export const individualInFlightLastTouched = new LRUCache<string, number>({
max: 10000,
});
/** Counter for observability. */
export let activeIndividualRequests = 0;
// ---------------------------------------------------------------------------
// Individual fallback circuit breaker (independent of batch CB)
// ---------------------------------------------------------------------------
let individualConsecutiveErrors = 0;
export let individualCooldownUntil = 0;
const INDIVIDUAL_COOLDOWN_MS = 60000;
// ---------------------------------------------------------------------------
// Individual fallback pipeline
// ---------------------------------------------------------------------------
/**
* Processes a single message via the Piscina worker pool (offloaded from
* main thread to avoid blocking the event loop).
*/
async function processIndividualFallback(
message: MessageRecord,
): Promise<void> {
const { id: messageId } = message;
const conversationKey = getConversationKey(message);
activeIndividualRequests++;
individualInFlightByConversation.set(
conversationKey,
(individualInFlightByConversation.get(conversationKey) ?? 0) + 1,
);
individualInFlightLastTouched.set(conversationKey, Date.now());
let exhaustedOnIncomplete = false;
try {
// Run the LLM-heavy work in the worker thread
const workerResult = (await workerPool.run({
type: "individual",
message,
skipNormalAnalysis: false,
} as unknown)) as
| { ok: true; results: AnalysisResult[] }
| { ok: false; results: AnalysisResult[]; error: string };
let analysisResult: { results: AnalysisResult[] } | null = null;
let usedSimpleFallback = false;
if (workerResult.ok) {
const stillIncomplete = workerResult.results.some((r) =>
r.flags.includes("analysis_incomplete"),
);
if (stillIncomplete) {
exhaustedOnIncomplete = true;
analysisResult = null;
} else {
analysisResult = workerResult;
}
}
// Step 2: If normal analysis failed, try SIMPLE fallback via worker
if (!analysisResult) {
logger.info(
{ messageId },
"Normal analysis failed -- trying simple text fallback via worker",
);
const simpleResult = (await workerPool.run({
type: "individual",
message,
skipNormalAnalysis: true,
} as unknown)) as
| { ok: true; results: AnalysisResult[] }
| { ok: false; results: AnalysisResult[]; error: string };
if (simpleResult.ok) {
analysisResult = simpleResult;
usedSimpleFallback = true;
exhaustedOnIncomplete = false;
}
}
if (!analysisResult) {
throw new Error(
`Both normal and simple analysis failed for message ${messageId}`,
);
}
if (usedSimpleFallback) {
logger.info(
{ messageId, status: analysisResult.results[0]?.status },
"Used simple text fallback for individual message (via worker)",
);
}
// Main thread: DB writes + broadcast
const updates = analysisResult.results.map((r) => ({
messageId: r.messageId,
result: {
status: r.status,
flags: JSON.stringify(r.flags),
score: r.score,
analysis: r.analysis,
categories: r.categories,
severity: r.severity,
confidence: r.confidence,
recommendedAction: r.recommendedAction,
analyzedAt: Date.now(),
error: null,
},
}));
const rows = await updateMessagesAIAnalysisBulk(updates);
for (const row of rows) {
broadcastAnalysisCompleted(row);
scheduleAutoDelete(row);
// Update reputation autonomously
if (row.ai_status === "clean") {
import("./userReputationStore.js")
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
.catch((e) =>
logger.error(
{ error: e },
"Failed to record clean message streak in fallback",
),
);
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
import("./userReputationStore.js")
.then((store) =>
store.recordInfraction(
row.user_id,
row.guild_id,
row.ai_severity as "low" | "medium" | "high" | "critical",
),
)
.catch((e) =>
logger.error(
{ error: e },
"Failed to record infraction penalty in fallback",
),
);
}
}
const resultSummary = analysisResult.results[0];
logModerationError([messageId], config.AI_LLM_MODEL, new Error("Success"), {
phase: "individual_fallback",
status: resultSummary?.status,
flags: resultSummary?.flags,
severity: resultSummary?.severity,
confidence: resultSummary?.confidence,
});
individualConsecutiveErrors = 0;
logger.debug(
{ messageId, status: analysisResult.results[0]?.status },
"Individual fallback analysis complete (via worker)",
);
} catch (error) {
individualConsecutiveErrors++;
if (
individualConsecutiveErrors >= config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD
) {
individualCooldownUntil = Date.now() + INDIVIDUAL_COOLDOWN_MS;
fireAlert({
type: "individual_cb",
consecutiveErrors: individualConsecutiveErrors,
message: `Individual fallback circuit breaker triggered after ${individualConsecutiveErrors} consecutive errors`,
lastError: LAST_ERROR.value,
});
}
LAST_ERROR.value = error instanceof Error ? error.message : String(error);
logModerationError(
[messageId],
config.AI_LLM_MODEL,
error as Error | string,
{
phase: "individual_fallback",
conversationKey,
exhaustedOnIncomplete,
},
);
if (exhaustedOnIncomplete) {
await updateMessagesAIAnalysisBulk([
{
messageId,
result: {
status: "error",
flags: JSON.stringify(["individual_analysis_exhausted"]),
score: 0,
analysis:
"Individual fallback exhausted all retries: LLM consistently dropped this message even in single-target mode",
categories: ["individual_analysis_exhausted"],
severity: "none",
confidence: 0,
recommendedAction: "review",
analyzedAt: Date.now(),
error: LAST_ERROR.value,
},
},
]).catch((dbErr: unknown) => {
logger.error(
{ messageId, error: String(dbErr) },
"Failed to write terminal exhausted status",
);
});
logger.warn(
{ messageId },
"Individual fallback exhausted -- marked as individual_analysis_exhausted",
);
} else {
logger.error(
{
messageId,
error: LAST_ERROR.value,
stack: error instanceof Error ? error.stack : undefined,
},
"Individual fallback analysis failed (transient) -- will be retried",
);
}
} finally {
activeIndividualRequests--;
individualInFlight.delete(messageId);
const prev = individualInFlightByConversation.get(conversationKey) ?? 1;
if (prev <= 1) {
individualInFlightByConversation.delete(conversationKey);
individualInFlightLastTouched.delete(conversationKey);
} else {
individualInFlightByConversation.set(conversationKey, prev - 1);
individualInFlightLastTouched.set(conversationKey, Date.now());
}
}
}
// ---------------------------------------------------------------------------
// Enqueue individual fallbacks
// ---------------------------------------------------------------------------
/**
* Fans out message records to the individual fallback queue.
*
* FIX #1: Checks concurrency cap before admitting new work.
* FIX #5: Checks individual circuit breaker before admitting new work.
*/
export function enqueueIndividualFallbacks(messages: MessageRecord[]): void {
// FIX #5: Honour the individual circuit breaker.
if (Date.now() < individualCooldownUntil) {
logger.warn(
{
until: new Date(individualCooldownUntil).toISOString(),
skipped: messages.length,
},
"Individual fallback circuit breaker active -- messages will be recovered later",
);
return;
}
// FIX #5: Enforce concurrency cap
const maxConcurrent = config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT ?? 50;
const availableSlots = Math.max(0, maxConcurrent - activeIndividualRequests);
if (availableSlots <= 0) {
logger.debug(
{ maxConcurrent, active: activeIndividualRequests },
"Individual fallback concurrency cap reached -- messages will be recovered later",
);
return;
}
const newMessages = messages
.filter((m) => !individualInFlight.has(m.id))
.slice(0, availableSlots);
if (newMessages.length === 0) return;
logger.debug(
{
count: newMessages.length,
messageIds: newMessages.map((m) => m.id),
},
"Enqueueing individual fallback analysis for batch-incomplete messages",
);
for (const msg of newMessages) {
individualInFlight.set(msg.id, true);
processIndividualFallback(msg).catch((err: unknown) => {
logger.error(
{ messageId: msg.id, error: String(err) },
"Unexpected uncaught error escaping processIndividualFallback",
);
individualInFlight.delete(msg.id);
const ck = getConversationKey(msg);
const prev = individualInFlightByConversation.get(ck) ?? 1;
if (prev <= 1) {
individualInFlightByConversation.delete(ck);
individualInFlightLastTouched.delete(ck);
} else {
individualInFlightByConversation.set(ck, prev - 1);
individualInFlightLastTouched.set(ck, Date.now());
}
});
}
}
@@ -0,0 +1,81 @@
import { createChildLogger } from "@bete/shared/logger";
const log = createChildLogger("jsonExtractor");
/**
* Helper to extract JSON from a potentially conversational or markdown-wrapped string.
*/
export function extractJson(content: string): unknown {
const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g;
const matches = content.matchAll(codeBlockRegex);
for (const match of matches) {
const codeContent = match[1].trim();
try {
const parsed = JSON.parse(codeContent);
if (parsed && typeof parsed === "object") {
return parsed;
}
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to parse JSON from code block — trying next block",
);
}
}
for (let start = 0; start < content.length; start++) {
const firstChar = content[start];
if (firstChar !== "{" && firstChar !== "[") continue;
const stack = [firstChar];
let inString = false;
let escaped = false;
for (let i = start + 1; i < content.length; i++) {
const char = content[i];
if (inString) {
if (escaped) {
escaped = false;
} else if (char === "\\") {
escaped = true;
} else if (char === '"') {
inString = false;
}
continue;
}
if (char === '"') {
inString = true;
continue;
}
if (char === "{" || char === "[") {
stack.push(char);
continue;
}
const last = stack[stack.length - 1];
if ((char === "}" && last === "{") || (char === "]" && last === "[")) {
stack.pop();
if (stack.length === 0) {
const candidate = content.slice(start, i + 1);
try {
const parsed = JSON.parse(candidate);
if (parsed && typeof parsed === "object") {
return parsed;
}
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to parse JSON candidate — trying next position",
);
}
break;
}
}
}
}
throw new Error("No JSON object found in response");
}
@@ -2,7 +2,6 @@ import { createChildLogger } from "@bete/shared/logger";
import { delay, retryWithBackoff } from "@bete/shared/utils"; import { delay, retryWithBackoff } from "@bete/shared/utils";
import { LRUCache } from "lru-cache"; import { LRUCache } from "lru-cache";
import type { ChatCompletion } from "openai/resources/chat/completions"; import type { ChatCompletion } from "openai/resources/chat/completions";
import { z } from "zod";
import { config } from "../../shared/config/config.js"; import { config } from "../../shared/config/config.js";
import { resizeImageForVision } from "../attachment-upload/imageResizer.js"; import { resizeImageForVision } from "../attachment-upload/imageResizer.js";
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js"; import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
@@ -46,33 +45,31 @@ import {
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js"; import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
import { initializeUserReputation } from "./userReputationStore.js"; import { initializeUserReputation } from "./userReputationStore.js";
const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]); export { sniffImageMimeType } from "./imageMimeSniffer.js";
const RecommendedActionSchema = z.enum([ export { extractJson } from "./jsonExtractor.js";
"none", export {
"monitor", parseModerationResponse,
"warn", sanitizeErrorMessage,
"review", } from "./moderationResponseParser.js";
"delete", // Re-export all symbols from sub-modules to preserve public API
"escalate", export {
]); ModerationResponseSchema,
RecommendedActionSchema,
ResultItemSchema,
SeveritySchema,
} from "./moderationSchemas.js";
export {
clampScore,
DEFERRAL_ANALYSIS_PATTERN,
DEFERRAL_EXCEPTION_PATTERN,
deriveRecommendedAction,
deriveSeverity,
hasDeferralAnalysis,
} from "./severityDeriver.js";
const ResultItemSchema = z.object({ import { sniffImageMimeType } from "./imageMimeSniffer.js";
message_id: z.union([z.string(), z.number()]).transform(String), // Internal imports for functions used locally in the facade
status: z.enum(["clean", "warn", "flagged"]), import { parseModerationResponse } from "./moderationResponseParser.js";
flags: z.array(z.string()).optional(),
score: z.number(),
analysis: z.string().nullable().optional(),
categories: z.array(z.string()).optional(),
severity: SeveritySchema.optional(),
confidence: z.number().optional(),
recommended_action: RecommendedActionSchema.optional(),
policy_version: z.string().optional(),
evidence: z.array(z.string()).optional(),
});
const ModerationResponseSchema = z.object({
results: z.array(ResultItemSchema),
});
const log = createChildLogger("llmModerationClient"); const log = createChildLogger("llmModerationClient");
@@ -112,371 +109,6 @@ async function buildCorrectedFewShotExamples(): Promise<string> {
} }
} }
/**
* Enhanced deferral detection pattern (R9).
*
* Only matches patterns where the model explicitly states it cannot make
* a decision and needs human review. Removed overly broad patterns that
* caused false positives:
* - "admin (perlu|harus|sebaiknya)" → common in regular sentences
* - "bisa (berpotensi|mengandung)" → decisive statements, not deferral
* - "maaf|sorry" → opinions/apologies, not deferral
* - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral
*/
const DEFERRAL_ANALYSIS_PATTERN =
/(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i;
/**
* Exceptions: patterns that look like deferral but are actually decisive.
* Expanded to catch more variations where the model gives a clear verdict.
*/
const DEFERRAL_EXCEPTION_PATTERN =
/tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i;
function hasDeferralAnalysis(analysis: string): boolean {
if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false;
return DEFERRAL_ANALYSIS_PATTERN.test(analysis);
}
function clampScore(value: number | undefined, fallback = 0): number {
return Math.max(
0,
Math.min(1, Number.isFinite(value) ? (value as number) : fallback),
);
}
function deriveSeverity(
status: "clean" | "warn" | "flagged",
score: number,
): z.infer<typeof SeveritySchema> {
if (status === "clean") return "none";
if (status === "warn") return score >= 0.65 ? "medium" : "low";
if (score >= 0.9) return "critical";
return score >= 0.75 ? "high" : "medium";
}
function deriveRecommendedAction(
status: "clean" | "warn" | "flagged",
severity: z.infer<typeof SeveritySchema>,
): z.infer<typeof RecommendedActionSchema> {
if (status === "clean") return "none";
if (status === "warn") return severity === "medium" ? "review" : "warn";
if (severity === "critical") return "escalate";
if (severity === "high") return "delete";
return "review";
}
/**
* Helper to extract JSON from a potentially conversational or markdown-wrapped string.
*/
export function extractJson(content: string): unknown {
const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g;
const matches = content.matchAll(codeBlockRegex);
for (const match of matches) {
const codeContent = match[1].trim();
try {
const parsed = JSON.parse(codeContent);
if (parsed && typeof parsed === "object") {
return parsed;
}
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to parse JSON from code block — trying next block",
);
}
}
for (let start = 0; start < content.length; start++) {
const firstChar = content[start];
if (firstChar !== "{" && firstChar !== "[") continue;
const stack = [firstChar];
let inString = false;
let escaped = false;
for (let i = start + 1; i < content.length; i++) {
const char = content[i];
if (inString) {
if (escaped) {
escaped = false;
} else if (char === "\\") {
escaped = true;
} else if (char === '"') {
inString = false;
}
continue;
}
if (char === '"') {
inString = true;
continue;
}
if (char === "{" || char === "[") {
stack.push(char);
continue;
}
const last = stack[stack.length - 1];
if ((char === "}" && last === "{") || (char === "]" && last === "[")) {
stack.pop();
if (stack.length === 0) {
const candidate = content.slice(start, i + 1);
try {
const parsed = JSON.parse(candidate);
if (parsed && typeof parsed === "object") {
return parsed;
}
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to parse JSON candidate — trying next position",
);
}
break;
}
}
}
}
throw new Error("No JSON object found in response");
}
/**
* Sanitize error messages for client-facing output (R10).
* Internal details are logged but the caller gets a generic message.
*/
function sanitizeErrorMessage(internalMsg: string, messageId: string): string {
// Log the full error for debugging
log.warn(
{ messageId, internalError: internalMsg },
"Internal moderation error (sanitized for client)",
);
// Return generic message without internal details
return `Analisis gagal dan memerlukan pemeriksaan manual. Error code: MOD_${Date.now().toString(36).slice(0, 6)}`;
}
export function parseModerationResponse(
content: string,
targetIds: string[],
): AnalysisResult[] {
let parsed: any;
try {
parsed = JSON.parse(content);
} catch (e) {
parsed = extractJson(content);
}
if (Array.isArray(parsed)) {
parsed = { results: parsed };
} else if (parsed && typeof parsed === "object" && !("results" in parsed)) {
if ("message_id" in parsed) {
parsed = { results: [parsed] };
} else {
const arrayKey = Object.keys(parsed).find((key) => {
const val = parsed[key];
return (
Array.isArray(val) &&
val.length > 0 &&
val.every(
(item: unknown) =>
typeof item === "object" &&
item !== null &&
"message_id" in (item as Record<string, unknown>),
)
);
});
if (arrayKey) {
parsed.results = parsed[arrayKey];
} else {
parsed = { results: [parsed] };
}
}
}
const parseResult = ModerationResponseSchema.safeParse(parsed);
if (!parseResult.success) {
throw new Error(`Zod validation failed: ${parseResult.error.message}`);
}
const response = parseResult.data;
const foundIds = new Set<string>();
const targetIdSet = new Set(targetIds);
const results: (AnalysisResult | null)[] = response.results.map((result) => {
const {
message_id,
status,
flags,
score,
analysis,
categories,
severity,
confidence,
recommended_action,
policy_version,
evidence,
} = result;
const finalId = message_id.trim();
if (!targetIdSet.has(finalId)) {
return null;
}
if (foundIds.has(finalId)) {
throw new Error(
`Duplicate message_id in moderation response: ${finalId}`,
);
}
foundIds.add(finalId);
const coalescedAnalysis = analysis ?? "";
if (hasDeferralAnalysis(coalescedAnalysis)) {
throw new Error(
`Deferral analysis is not allowed for message ${finalId}; return a direct moderation decision`,
);
}
const normalizedScore = clampScore(score);
const normalizedConfidence = clampScore(confidence, normalizedScore);
const normalizedSeverity =
severity ?? deriveSeverity(status, normalizedScore);
return {
messageId: finalId,
status: status as "clean" | "warn" | "flagged",
flags: flags ?? [],
score: normalizedScore,
analysis: coalescedAnalysis,
categories: categories ?? flags ?? [],
severity: normalizedSeverity,
confidence: normalizedConfidence,
recommendedAction:
recommended_action ??
deriveRecommendedAction(status, normalizedSeverity),
policyVersion: policy_version ?? "default-2026-05-30",
evidence: evidence ?? [],
};
});
const filteredResults = results.filter(
(r): r is AnalysisResult => r !== null,
);
const missingIds = targetIds.filter((id) => !foundIds.has(id));
if (missingIds.length > 0) {
log.warn(
{ missingIds, foundCount: foundIds.size, totalCount: targetIds.length },
"Some target IDs missing in response - marking as incomplete",
);
for (const missingId of missingIds) {
filteredResults.push({
messageId: missingId,
status: "error",
flags: ["analysis_incomplete"],
score: 0,
analysis: sanitizeErrorMessage(
"Analysis incomplete - LLM did not process this message",
missingId,
),
categories: ["analysis_incomplete"],
severity: "none",
confidence: 0,
recommendedAction: "review",
policyVersion: "default-2026-05-30",
evidence: [],
});
}
}
return filteredResults;
}
interface ModerationInput {
targets: MessageRecord[];
contextText: string;
attachments?: AttachmentRecord[];
}
interface ModerationOutput {
results: AnalysisResult[];
raw: unknown;
}
/**
* Sniff the first bytes of a buffer to determine if it is a supported image
* format. Returns the canonical MIME type string on success, or null if the
* bytes are not a recognizable image.
*/
function sniffImageMimeType(buf: Buffer): string | null {
if (buf.length < 12) return null;
if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {
return "image/jpeg";
}
if (
buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47 &&
buf[4] === 0x0d &&
buf[5] === 0x0a &&
buf[6] === 0x1a &&
buf[7] === 0x0a
) {
return "image/png";
}
if (
buf[0] === 0x47 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x38
) {
return "image/gif";
}
if (
buf[0] === 0x52 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x46 &&
buf[8] === 0x57 &&
buf[9] === 0x45 &&
buf[10] === 0x42 &&
buf[11] === 0x50
) {
return "image/webp";
}
if (
buf.length >= 12 &&
buf[4] === 0x66 &&
buf[5] === 0x74 &&
buf[6] === 0x79 &&
buf[7] === 0x70
) {
const brand = buf.subarray(8, 12).toString("ascii");
if (brand.startsWith("avif") || brand.startsWith("avis")) {
return "image/avif";
}
if (
brand.startsWith("mif1") ||
brand.startsWith("heic") ||
brand.startsWith("heis")
) {
return "image/heic";
}
}
return null;
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Shared types for image resolution // Shared types for image resolution
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -1495,6 +1127,17 @@ async function runMediaBatch(
// Main entry point — splits text-only vs media, runs both paths in parallel // Main entry point — splits text-only vs media, runs both paths in parallel
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
interface ModerationInput {
targets: MessageRecord[];
contextText: string;
attachments?: AttachmentRecord[];
}
interface ModerationOutput {
results: AnalysisResult[];
raw: unknown;
}
/** /**
* Runs LLM-based moderation analysis on messages. * Runs LLM-based moderation analysis on messages.
* *
@@ -0,0 +1,178 @@
import { createChildLogger } from "@bete/shared/logger";
import type { AnalysisResult } from "../message-capture/types.js";
import { extractJson } from "./jsonExtractor.js";
import { ModerationResponseSchema } from "./moderationSchemas.js";
import {
clampScore,
DEFERRAL_ANALYSIS_PATTERN,
DEFERRAL_EXCEPTION_PATTERN,
deriveRecommendedAction,
deriveSeverity,
hasDeferralAnalysis,
} from "./severityDeriver.js";
const log = createChildLogger("moderationResponseParser");
/**
* Re-export deferral patterns for backward compatibility.
* See severityDeriver.ts for the full regex definitions.
*/
export {
DEFERRAL_ANALYSIS_PATTERN,
DEFERRAL_EXCEPTION_PATTERN,
} from "./severityDeriver.js";
/**
* Sanitize error messages for client-facing output (R10).
* Internal details are logged but the caller gets a generic message.
*/
export function sanitizeErrorMessage(
internalMsg: string,
messageId: string,
): string {
// Log the full error for debugging
log.warn(
{ messageId, internalError: internalMsg },
"Internal moderation error (sanitized for client)",
);
// Return generic message without internal details
return `Analisis gagal dan memerlukan pemeriksaan manual. Error code: MOD_${Date.now().toString(36).slice(0, 6)}`;
}
export function parseModerationResponse(
content: string,
targetIds: string[],
): AnalysisResult[] {
let parsed: any;
try {
parsed = JSON.parse(content);
} catch (e) {
parsed = extractJson(content);
}
if (Array.isArray(parsed)) {
parsed = { results: parsed };
} else if (parsed && typeof parsed === "object" && !("results" in parsed)) {
if ("message_id" in parsed) {
parsed = { results: [parsed] };
} else {
const arrayKey = Object.keys(parsed).find((key) => {
const val = parsed[key];
return (
Array.isArray(val) &&
val.length > 0 &&
val.every(
(item: unknown) =>
typeof item === "object" &&
item !== null &&
"message_id" in (item as Record<string, unknown>),
)
);
});
if (arrayKey) {
parsed.results = parsed[arrayKey];
} else {
parsed = { results: [parsed] };
}
}
}
const parseResult = ModerationResponseSchema.safeParse(parsed);
if (!parseResult.success) {
throw new Error(`Zod validation failed: ${parseResult.error.message}`);
}
const response = parseResult.data;
const foundIds = new Set<string>();
const targetIdSet = new Set(targetIds);
const results: (AnalysisResult | null)[] = response.results.map((result) => {
const {
message_id,
status,
flags,
score,
analysis,
categories,
severity,
confidence,
recommended_action,
policy_version,
evidence,
} = result;
const finalId = message_id.trim();
if (!targetIdSet.has(finalId)) {
return null;
}
if (foundIds.has(finalId)) {
throw new Error(
`Duplicate message_id in moderation response: ${finalId}`,
);
}
foundIds.add(finalId);
const coalescedAnalysis = analysis ?? "";
if (hasDeferralAnalysis(coalescedAnalysis)) {
throw new Error(
`Deferral analysis is not allowed for message ${finalId}; return a direct moderation decision`,
);
}
const normalizedScore = clampScore(score);
const normalizedConfidence = clampScore(confidence, normalizedScore);
const normalizedSeverity =
severity ?? deriveSeverity(status, normalizedScore);
return {
messageId: finalId,
status: status as "clean" | "warn" | "flagged",
flags: flags ?? [],
score: normalizedScore,
analysis: coalescedAnalysis,
categories: categories ?? flags ?? [],
severity: normalizedSeverity,
confidence: normalizedConfidence,
recommendedAction:
recommended_action ??
deriveRecommendedAction(status, normalizedSeverity),
policyVersion: policy_version ?? "default-2026-05-30",
evidence: evidence ?? [],
};
});
const filteredResults = results.filter(
(r): r is AnalysisResult => r !== null,
);
const missingIds = targetIds.filter((id) => !foundIds.has(id));
if (missingIds.length > 0) {
log.warn(
{ missingIds, foundCount: foundIds.size, totalCount: targetIds.length },
"Some target IDs missing in response - marking as incomplete",
);
for (const missingId of missingIds) {
filteredResults.push({
messageId: missingId,
status: "error",
flags: ["analysis_incomplete"],
score: 0,
analysis: sanitizeErrorMessage(
"Analysis incomplete - LLM did not process this message",
missingId,
),
categories: ["analysis_incomplete"],
severity: "none",
confidence: 0,
recommendedAction: "review",
policyVersion: "default-2026-05-30",
evidence: [],
});
}
}
return filteredResults;
}
@@ -0,0 +1,41 @@
import { createChildLogger } from "@bete/shared/logger";
import { z } from "zod";
const log = createChildLogger("moderationSchemas");
export const SeveritySchema = z.enum([
"none",
"low",
"medium",
"high",
"critical",
]);
export const RecommendedActionSchema = z.enum([
"none",
"monitor",
"warn",
"review",
"delete",
"escalate",
]);
export const ResultItemSchema = z.object({
message_id: z.union([z.string(), z.number()]).transform(String),
status: z.enum(["clean", "warn", "flagged"]),
flags: z.array(z.string()).optional(),
score: z.number(),
analysis: z.string().nullable().optional(),
categories: z.array(z.string()).optional(),
severity: SeveritySchema.optional(),
confidence: z.number().optional(),
recommended_action: RecommendedActionSchema.optional(),
policy_version: z.string().optional(),
evidence: z.array(z.string()).optional(),
});
export const ModerationResponseSchema = z.object({
results: z.array(ResultItemSchema),
});
// Keep log referenced so TS does not tree-shake the logger init
log.debug("moderationSchemas loaded");
@@ -0,0 +1,64 @@
import { createChildLogger } from "@bete/shared/logger";
import type { z } from "zod";
import {
RecommendedActionSchema,
SeveritySchema,
} from "./moderationSchemas.js";
const log = createChildLogger("severityDeriver");
/**
* Enhanced deferral detection pattern (R9).
*
* Only matches patterns where the model explicitly states it cannot make
* a decision and needs human review. Removed overly broad patterns that
* caused false positives:
* - "admin (perlu|harus|sebaiknya)" → common in regular sentences
* - "bisa (berpotensi|mengandung)" → decisive statements, not deferral
* - "maaf|sorry" → opinions/apologies, not deferral
* - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral
*/
export const DEFERRAL_ANALYSIS_PATTERN =
/(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i;
/**
* Exceptions: patterns that look like deferral but are actually decisive.
* Expanded to catch more variations where the model gives a clear verdict.
*/
export const DEFERRAL_EXCEPTION_PATTERN =
/tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i;
export function hasDeferralAnalysis(analysis: string): boolean {
if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false;
return DEFERRAL_ANALYSIS_PATTERN.test(analysis);
}
export function clampScore(value: number | undefined, fallback = 0): number {
return Math.max(
0,
Math.min(1, Number.isFinite(value) ? (value as number) : fallback),
);
}
export function deriveSeverity(
status: "clean" | "warn" | "flagged",
score: number,
): z.infer<typeof SeveritySchema> {
if (status === "clean") return "none";
if (status === "warn") return score >= 0.65 ? "medium" : "low";
if (score >= 0.9) return "critical";
return score >= 0.75 ? "high" : "medium";
}
export function deriveRecommendedAction(
status: "clean" | "warn" | "flagged",
severity: z.infer<typeof SeveritySchema>,
): z.infer<typeof RecommendedActionSchema> {
if (status === "clean") return "none";
if (status === "warn") return severity === "medium" ? "review" : "warn";
if (severity === "critical") return "escalate";
if (severity === "high") return "delete";
return "review";
}
log.debug("severityDeriver loaded");
@@ -1,3 +1,7 @@
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("stickerPrompt");
/** /**
* Sticker-specific prompt templates for AI moderation. * Sticker-specific prompt templates for AI moderation.
* *
@@ -17,6 +21,7 @@ export function buildStickerVisionPrompt(
stickerName: string, stickerName: string,
messageId: string, messageId: string,
): string { ): string {
logger.debug({ stickerName, messageId }, "Building sticker vision prompt");
return [ return [
`Analisis sticker Discord berikut sebagai evidence moderasi.`, `Analisis sticker Discord berikut sebagai evidence moderasi.`,
`Sticker "${stickerName}" berasal dari pesan id=${messageId}.`, `Sticker "${stickerName}" berasal dari pesan id=${messageId}.`,
@@ -49,6 +54,10 @@ export function buildStickerTextOnlyWarning(
stickerName: string, stickerName: string,
stickerUrl: string, stickerUrl: string,
): string { ): string {
logger.debug(
{ stickerName, stickerUrl },
"Building sticker text-only warning",
);
return ( return (
`[sticker: "${stickerName}" (${stickerUrl}) — GAMBAR GAGAL DIUNDUH. ` + `[sticker: "${stickerName}" (${stickerUrl}) — GAMBAR GAGAL DIUNDUH. ` +
`"${stickerName}" adalah sticker kartun/meme Discord. ` + `"${stickerName}" adalah sticker kartun/meme Discord. ` +
@@ -68,6 +77,7 @@ export function buildCustomEmojiVisionPrompt(
emojiName: string, emojiName: string,
messageId: string, messageId: string,
): string { ): string {
logger.debug({ emojiName, messageId }, "Building custom emoji vision prompt");
return [ return [
`Analisis custom emoji Discord berikut sebagai evidence moderasi.`, `Analisis custom emoji Discord berikut sebagai evidence moderasi.`,
`Emoji "${emojiName}" berasal dari pesan id=${messageId}.`, `Emoji "${emojiName}" berasal dari pesan id=${messageId}.`,
@@ -87,6 +97,7 @@ export function buildCustomEmojiVisionPrompt(
* Fallback text for when a custom emoji image failed to download. * Fallback text for when a custom emoji image failed to download.
*/ */
export function buildCustomEmojiTextOnlyFallback(emojiName: string): string { export function buildCustomEmojiTextOnlyFallback(emojiName: string): string {
logger.debug({ emojiName }, "Building custom emoji text-only fallback");
return ( return (
`[custom_emoji: "${emojiName}" — GAMBAR GAGAL DIUNDUH. ` + `[custom_emoji: "${emojiName}" — GAMBAR GAGAL DIUNDUH. ` +
`"${emojiName}" adalah custom emoji Discord (ikon kecil). ` + `"${emojiName}" adalah custom emoji Discord (ikon kecil). ` +
@@ -105,6 +116,7 @@ export function buildGeneralImageVisionPrompt(
sourceLabel: string, sourceLabel: string,
_messageId: string, _messageId: string,
): string { ): string {
logger.debug({ sourceLabel }, "Building general image vision prompt");
return [ return [
`Deskripsikan gambar ini secara objektif dan spesifik.`, `Deskripsikan gambar ini secara objektif dan spesifik.`,
`${sourceLabel}`, `${sourceLabel}`,
@@ -1,3 +1,4 @@
import { createChildLogger } from "@bete/shared/logger";
import { and, desc, eq } from "drizzle-orm"; import { and, desc, eq } from "drizzle-orm";
import { getDatabase } from "../../shared/database/drizzle.js"; import { getDatabase } from "../../shared/database/drizzle.js";
import { import {
@@ -6,6 +7,8 @@ import {
userReputationsTable, userReputationsTable,
} from "../../shared/database/schema.js"; } from "../../shared/database/schema.js";
const logger = createChildLogger("userReputationStore");
/** /**
* Ensures a user reputation record exists. * Ensures a user reputation record exists.
*/ */
@@ -21,6 +24,7 @@ export async function initializeUserReputation(
.limit(1); .limit(1);
if (existing.length > 0) { if (existing.length > 0) {
logger.debug({ userId }, "Reputation record already exists");
return existing[0]; return existing[0];
} }
@@ -40,6 +44,7 @@ export async function initializeUserReputation(
if (!inserted) { if (!inserted) {
// If concurrent insert happened // If concurrent insert happened
logger.debug({ userId }, "Concurrent reputation insert detected, retrying");
const retry = await db const retry = await db
.select() .select()
.from(userReputationsTable) .from(userReputationsTable)
@@ -48,6 +53,10 @@ export async function initializeUserReputation(
return retry[0]; return retry[0];
} }
logger.debug(
{ userId, trustScore: inserted.trust_score },
"Initialized user reputation",
);
return inserted; return inserted;
} }
@@ -64,6 +73,14 @@ export async function getUserReputation(
.where(eq(userReputationsTable.user_id, userId)) .where(eq(userReputationsTable.user_id, userId))
.limit(1); .limit(1);
if (existing[0]) {
logger.debug(
{ userId, trustScore: existing[0].trust_score },
"Fetched user reputation",
);
} else {
logger.debug({ userId }, "No reputation record found, returning null");
}
return existing[0] || null; return existing[0] || null;
} }
@@ -93,6 +110,11 @@ export async function recordCleanMessage(
updated_at: Date.now(), updated_at: Date.now(),
}) })
.where(eq(userReputationsTable.user_id, userId)); .where(eq(userReputationsTable.user_id, userId));
logger.debug(
{ userId, previousScore: rep.trust_score, newScore, newStreak },
"Clean message recorded, reputation updated",
);
} }
/** /**
@@ -133,6 +155,17 @@ export async function recordInfraction(
updated_at: Date.now(), updated_at: Date.now(),
}) })
.where(eq(userReputationsTable.user_id, userId)); .where(eq(userReputationsTable.user_id, userId));
logger.info(
{
userId,
severity,
penalty,
newScore,
totalInfractions: rep.total_infractions + 1,
},
"Infraction recorded",
);
} }
/** /**
@@ -1,8 +1,8 @@
import { decodeCursor, encodeCursor } from "@bete/shared";
import { createChildLogger, type Logger } from "@bete/shared/logger"; import { createChildLogger, type Logger } from "@bete/shared/logger";
import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import { getDatabase } from "../../shared/database/drizzle.js"; import { getDatabase } from "../../shared/database/drizzle.js";
import type * as schema from "../../shared/database/schema.js"; import type * as schema from "../../shared/database/schema.js";
import { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
import type { import type {
AttachmentRecord, AttachmentRecord,
MessageQuery, MessageQuery,
@@ -18,7 +18,7 @@ import { ModerationActionsDb } from "./moderation-actions.db.js";
import { RetentionDb } from "./retention.db.js"; import { RetentionDb } from "./retention.db.js";
import { ReviewsDb } from "./reviews.db.js"; import { ReviewsDb } from "./reviews.db.js";
export { decodeCursor, encodeCursor } from "../message-capture/pagination.js"; export { decodeCursor, encodeCursor } from "@bete/shared";
export type { AIAnalysisUpdate } from "./messages.db.js"; export type { AIAnalysisUpdate } from "./messages.db.js";
// ─── Lazy singleton ──────────────────────────────────────────────────────── // ─── Lazy singleton ────────────────────────────────────────────────────────
@@ -0,0 +1,413 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import {
and,
asc,
desc,
eq,
inArray,
isNull,
or,
type SQL,
sql,
} from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type { MessageRecord } from "../message-capture/types.js";
// ─── Helpers ──────────────────────────────────────────────────────────────────
function stringifyAIList(
value: string[] | string | null | undefined,
): string | null {
if (value == null) return null;
return Array.isArray(value) ? JSON.stringify(value) : value;
}
// ─── AIAnalysisUpdate interface ──────────────────────────────────────────────
export interface AIAnalysisUpdate {
status: "pending" | "processing" | "clean" | "warn" | "flagged" | "error";
flags?: string | null;
score?: number | null;
analysis?: string | null;
categories?: string[] | string | null;
severity?: MessageRecord["ai_severity"] | null;
confidence?: number | null;
recommendedAction?: MessageRecord["ai_recommended_action"] | null;
analyzedAt?: number | null;
error?: string | null;
}
// ─── MessagesAnalysis Class ───────────────────────────────────────────────────
export class MessagesAnalysis {
protected logger: Logger;
constructor(
protected db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-analysis");
}
// ── AI Analysis Updates ─────────────────────────────────────────────────────
async updateMessageAIAnalysis(
messageId: string,
result: AIAnalysisUpdate,
): Promise<MessageRecord | null> {
this.logger.debug({ messageId }, "updateMessageAIAnalysis entry");
try {
await this.db
.update(messagesTable)
.set({
ai_status: result.status,
ai_moderation_flags: result.flags ?? null,
ai_moderation_score: result.score ?? null,
ai_analysis: result.analysis ?? null,
ai_categories: stringifyAIList(result.categories),
ai_severity: result.severity ?? null,
ai_confidence: result.confidence ?? result.score ?? null,
ai_recommended_action: result.recommendedAction ?? null,
ai_analyzed_at: result.analyzedAt ?? Date.now(),
ai_error: result.error ?? null,
})
.where(eq(messagesTable.id, messageId));
const rows = await this.db
.select()
.from(messagesTable)
.where(eq(messagesTable.id, messageId));
return (rows[0] as MessageRecord) ?? null;
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message AI analysis",
);
throw error;
}
}
async updateMessagesAIAnalysisBulk(
updates: Array<{ messageId: string; result: AIAnalysisUpdate }>,
): Promise<MessageRecord[]> {
this.logger.debug(
{ count: updates.length },
"updateMessagesAIAnalysisBulk entry",
);
if (updates.length === 0) return [];
try {
const now = Date.now();
await this.db.transaction(async (tx) => {
for (const { messageId, result } of updates) {
await tx
.update(messagesTable)
.set({
ai_status: result.status,
ai_moderation_flags: result.flags ?? null,
ai_moderation_score: result.score ?? null,
ai_analysis: result.analysis ?? null,
ai_categories: stringifyAIList(result.categories),
ai_severity: result.severity ?? null,
ai_confidence: result.confidence ?? result.score ?? null,
ai_recommended_action: result.recommendedAction ?? null,
ai_analyzed_at: result.analyzedAt ?? now,
ai_error: result.error ?? null,
})
.where(eq(messagesTable.id, messageId));
}
});
const ids = updates.map(({ messageId }) => messageId);
const rows = await this.db
.select()
.from(messagesTable)
.where(inArray(messagesTable.id, ids));
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
error: error instanceof Error ? error.message : String(error),
},
"Failed to bulk update messages AI analysis",
);
throw error;
}
}
async getPendingAIAnalysisMessages(
limit: number = 25,
): Promise<MessageRecord[]> {
this.logger.debug({ limit }, "getPendingAIAnalysisMessages entry");
try {
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get pending AI analysis messages",
);
throw error;
}
}
// ── Conversation Context ────────────────────────────────────────────────────
async getConversationContextBefore(input: {
channelId: string;
threadId: string | null;
beforeCreatedAt: number;
limit: number;
}): Promise<MessageRecord[]> {
this.logger.debug(
{ channelId: input.channelId, threadId: input.threadId },
"getConversationContextBefore entry",
);
try {
const { channelId, threadId, beforeCreatedAt, limit } = input;
const locationCondition = threadId
? eq(messagesTable.thread_id, threadId)
: eq(messagesTable.channel_id, channelId);
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
locationCondition,
sql`${messagesTable.created_at} < ${beforeCreatedAt}`,
isNull(messagesTable.deleted_at),
),
)
.orderBy(desc(messagesTable.created_at))
.limit(limit);
return (rows as MessageRecord[]).reverse();
} catch (error) {
this.logger.error(
{
channelId: input.channelId,
threadId: input.threadId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get conversation context before",
);
throw error;
}
}
async getPendingMessagesByConversation(
conversationKey: string,
limit: number = 200,
): Promise<MessageRecord[]> {
this.logger.debug(
{ conversationKey, limit },
"getPendingMessagesByConversation entry",
);
try {
const rows = await this.db.transaction(async (tx) => {
const pendingIdsQuery = tx
.select({ id: messagesTable.id })
.from(messagesTable)
.where(
and(
or(
eq(messagesTable.thread_id, conversationKey),
eq(messagesTable.channel_id, conversationKey),
),
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit)
.for("update", { skipLocked: true });
const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>;
if (pendingIds.length === 0) return [];
return await tx
.update(messagesTable)
.set({ ai_status: "processing", ai_analyzed_at: Date.now() })
.where(
inArray(
messagesTable.id,
pendingIds.map((r) => r.id),
),
)
.returning();
});
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
conversationKey,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get pending messages by conversation",
);
throw error;
}
}
async getPendingConversationKeys(limit: number = 500): Promise<string[]> {
this.logger.debug({ limit }, "getPendingConversationKeys entry");
try {
const rows = (await this.db
.selectDistinct({
thread_id: messagesTable.thread_id,
channel_id: messagesTable.channel_id,
})
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.limit(limit)) as Array<{
thread_id: string | null;
channel_id: string;
}>;
const keys: string[] = [];
for (const row of rows) {
const key = row.thread_id || row.channel_id;
if (key && !keys.includes(key)) {
keys.push(key);
}
}
return keys;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get pending conversation keys",
);
throw error;
}
}
async getConversationKeysWithIncompleteAnalysis(
limit: number = 200,
): Promise<string[]> {
this.logger.debug(
{ limit },
"getConversationKeysWithIncompleteAnalysis entry",
);
try {
const rows = (await this.db
.selectDistinct({
thread_id: messagesTable.thread_id,
channel_id: messagesTable.channel_id,
})
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "error"),
sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`,
sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`,
isNull(messagesTable.deleted_at),
),
)
.limit(limit)) as Array<{
thread_id: string | null;
channel_id: string;
}>;
const keys: string[] = [];
for (const row of rows) {
const key = row.thread_id || row.channel_id;
if (key && !keys.includes(key)) {
keys.push(key);
}
}
return keys;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get conversation keys with incomplete analysis",
);
throw error;
}
}
async getIncompleteMessagesByConversation(
conversationKey: string,
limit: number = 500,
): Promise<MessageRecord[]> {
this.logger.debug(
{ conversationKey, limit },
"getIncompleteMessagesByConversation entry",
);
try {
const rows = await this.db.transaction(async (tx) => {
const pendingIdsQuery = tx
.select({ id: messagesTable.id })
.from(messagesTable)
.where(
and(
or(
eq(messagesTable.thread_id, conversationKey),
eq(messagesTable.channel_id, conversationKey),
),
eq(messagesTable.ai_status, "error"),
sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`,
sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`,
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit)
.for("update", { skipLocked: true });
const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>;
if (pendingIds.length === 0) return [];
return await tx
.update(messagesTable)
.set({ ai_status: "processing", ai_analyzed_at: Date.now() })
.where(
inArray(
messagesTable.id,
pendingIds.map((r) => r.id),
),
)
.returning();
});
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
conversationKey,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get incomplete messages by conversation",
);
throw error;
}
}
}
@@ -0,0 +1,86 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, eq, isNull, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type { MessageRecord } from "../message-capture/types.js";
// ─── MessagesCleanup Class ────────────────────────────────────────────────────
export class MessagesCleanup {
private logger: Logger;
constructor(
private db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-cleanup");
}
async getExpiredMessages(retentionDays: number): Promise<MessageRecord[]> {
this.logger.debug({ retentionDays }, "getExpiredMessages entry");
try {
const cutoffTime = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
sql`${messagesTable.created_at} < ${cutoffTime}`,
isNull(messagesTable.deleted_at),
),
)
.limit(1000);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
retentionDays,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get expired messages",
);
throw error;
}
}
async revertStuckProcessingMessages(
timeoutMs: number = 300000,
): Promise<number> {
this.logger.debug({ timeoutMs }, "revertStuckProcessingMessages entry");
try {
const cutoffTime = Date.now() - timeoutMs;
const rows = await this.db
.update(messagesTable)
.set({ ai_status: "pending", ai_analyzed_at: null })
.where(
and(
eq(messagesTable.ai_status, "processing"),
sql`${messagesTable.ai_analyzed_at} < ${cutoffTime}`,
),
)
.returning({ id: messagesTable.id });
if (Array.isArray(rows) && rows.length > 0) {
this.logger.info(
{
count: rows.length,
messageIds: rows.map((r: { id: string }) => r.id),
},
"Reverted stuck processing messages back to pending",
);
}
return Array.isArray(rows) ? rows.length : 0;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to revert stuck processing messages",
);
return 0;
}
}
}
@@ -0,0 +1,204 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, or, type SQL } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type { MessageRecord } from "../message-capture/types.js";
// ─── Shared Helpers ──────────────────────────────────────────────────────────
export function channelOrThreadCondition(channelId: string): SQL {
return or(
eq(messagesTable.channel_id, channelId),
eq(messagesTable.thread_id, channelId),
) as SQL;
}
// ─── MessagesCrud Class ──────────────────────────────────────────────────────
export class MessagesCrud {
protected logger: Logger;
constructor(
protected db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-crud");
}
// ── INSERT ──────────────────────────────────────────────────────────────────
async insertMessage(message: MessageRecord): Promise<void> {
this.logger.debug({ messageId: message.id }, "insertMessage entry");
try {
await this.db
.insert(messagesTable)
.values(message as any)
.onConflictDoNothing();
} catch (error) {
this.logger.error(
{
messageId: message.id,
error: error instanceof Error ? error.message : String(error),
},
"Failed to insert message",
);
throw error;
}
}
async upsertMessageForCapture(message: MessageRecord): Promise<boolean> {
this.logger.debug(
{ messageId: message.id },
"upsertMessageForCapture entry",
);
try {
const messageWithAIStatus = {
...message,
ai_status: "pending" as const,
};
const rows = await this.db
.insert(messagesTable)
.values(messageWithAIStatus as any)
.onConflictDoNothing()
.returning({ id: messagesTable.id });
return rows.length > 0;
} catch (error) {
this.logger.error(
{
messageId: message.id,
error: error instanceof Error ? error.message : String(error),
},
"Failed to upsert message for capture",
);
throw error;
}
}
// ── UPDATE ──────────────────────────────────────────────────────────────────
async updateMessageAsEdited(
messageId: string,
editedContent: string,
editedAt: number,
): Promise<void> {
this.logger.debug({ messageId }, "updateMessageAsEdited entry");
try {
await this.db
.update(messagesTable)
.set({
edited_content: editedContent,
edited_at: editedAt,
type: "edited",
ai_status: "pending",
ai_moderation_flags: null,
ai_moderation_score: null,
ai_analysis: null,
ai_categories: null,
ai_severity: null,
ai_confidence: null,
ai_recommended_action: null,
ai_analyzed_at: null,
ai_error: null,
})
.where(eq(messagesTable.id, messageId));
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message as edited",
);
throw error;
}
}
async updateMessageAsDeleted(
messageId: string,
deletedAt: number,
): Promise<void> {
this.logger.debug({ messageId }, "updateMessageAsDeleted entry");
try {
await this.db
.update(messagesTable)
.set({
deleted_at: deletedAt,
type: "deleted",
})
.where(eq(messagesTable.id, messageId));
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message as deleted",
);
throw error;
}
}
// ── GET ─────────────────────────────────────────────────────────────────────
async getMessagesByChannel(
channelId: string,
limit: number = 50,
offset: number = 0,
guildId?: string,
): Promise<MessageRecord[]> {
this.logger.debug(
{ channelId, limit, offset, guildId },
"getMessagesByChannel entry",
);
try {
const conditions: SQL[] = [channelOrThreadCondition(channelId)];
if (guildId) {
conditions.push(eq(messagesTable.guild_id, guildId));
}
const rows = await this.db
.select()
.from(messagesTable)
.where(and(...conditions))
.orderBy(desc(messagesTable.created_at), desc(messagesTable.id))
.limit(limit)
.offset(offset);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
channelId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get messages by channel",
);
throw error;
}
}
async getMessageById(messageId: string): Promise<MessageRecord | null> {
this.logger.debug({ messageId }, "getMessageById entry");
try {
const rows = await this.db
.select()
.from(messagesTable)
.where(eq(messagesTable.id, messageId));
return (rows[0] as MessageRecord) ?? null;
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get message by id",
);
throw error;
}
}
}
@@ -1,826 +1,161 @@
import { createChildLogger, type Logger } from "@bete/shared/logger"; import { createChildLogger, type Logger } from "@bete/shared/logger";
import {
and,
asc,
desc,
eq,
inArray,
isNull,
or,
type SQL,
sql,
} from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js"; import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
import type { import type {
MessageQuery, MessageQuery,
MessageRecord, MessageRecord,
PageResult, PageResult,
} from "../message-capture/types.js"; } from "../message-capture/types.js";
import type { AIAnalysisUpdate } from "./messages.analysis.js";
import { MessagesAnalysis } from "./messages.analysis.js";
import { MessagesCleanup } from "./messages.cleanup.js";
import { MessagesCrud } from "./messages.crud.js";
import { MessagesPagination } from "./messages.pagination.js";
import { MessagesSearch } from "./messages.search.js";
// ─── Helpers ──────────────────────────────────────────────────────────────── // Re-export AIAnalysisUpdate for consumers (messageStore.ts imports it)
export type { AIAnalysisUpdate } from "./messages.analysis.js";
function channelOrThreadCondition(channelId: string): SQL { // ─── MessagesDb Facade ────────────────────────────────────────────────────────
return or( // Thin facade that delegates to domain-specific sub-modules.
eq(messagesTable.channel_id, channelId),
eq(messagesTable.thread_id, channelId),
) as SQL;
}
function buildListMessageConditions(query: MessageQuery): SQL[] {
const conditions: SQL[] = [];
if (query.guildId) {
conditions.push(eq(messagesTable.guild_id, query.guildId));
}
if (query.channelId) {
conditions.push(channelOrThreadCondition(query.channelId));
}
if (query.threadId) {
conditions.push(eq(messagesTable.thread_id, query.threadId));
}
if (query.userId) {
conditions.push(eq(messagesTable.user_id, query.userId));
}
if (query.status && query.status.length > 0) {
conditions.push(sql`${messagesTable.ai_status} in ${query.status}`);
}
if (query.q) {
const pattern = `%${query.q.toLowerCase()}%`;
conditions.push(sql`lower(${messagesTable.content}) like ${pattern}`);
}
const cursorData = decodeCursor(query.cursor);
if (cursorData) {
conditions.push(
sql`(${messagesTable.created_at} < ${cursorData.created_at} or (${messagesTable.created_at} = ${cursorData.created_at} and ${messagesTable.id} < ${cursorData.id}))`,
);
}
return conditions;
}
function pageRows<T extends { created_at: number; id: string }>(
rows: unknown[],
limit: number,
): PageResult<T> {
const hasMore = rows.length > limit;
const data = rows.slice(0, limit) as T[];
const lastItem = data[data.length - 1];
const nextCursor =
hasMore && lastItem
? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id })
: null;
return { data, nextCursor };
}
function pageMessages(
rows: unknown[],
limit: number,
): PageResult<MessageRecord> {
return pageRows<MessageRecord>(rows, limit);
}
function stringifyAIList(
value: string[] | string | null | undefined,
): string | null {
if (value == null) return null;
return Array.isArray(value) ? JSON.stringify(value) : value;
}
// ─── AIAnalysisUpdate interface ────────────────────────────────────────────
export interface AIAnalysisUpdate {
status: "pending" | "processing" | "clean" | "warn" | "flagged" | "error";
flags?: string | null;
score?: number | null;
analysis?: string | null;
categories?: string[] | string | null;
severity?: MessageRecord["ai_severity"] | null;
confidence?: number | null;
recommendedAction?: MessageRecord["ai_recommended_action"] | null;
analyzedAt?: number | null;
error?: string | null;
}
// ─── MessagesDb Class ──────────────────────────────────────────────────────
export class MessagesDb { export class MessagesDb {
private logger: Logger; private crud: MessagesCrud;
private analysis: MessagesAnalysis;
private search: MessagesSearch;
private pagination: MessagesPagination;
private cleanup: MessagesCleanup;
constructor( constructor(db: NodePgDatabase<typeof schema>, _parentLogger?: Logger) {
private db: NodePgDatabase<typeof schema>, const logger = _parentLogger ?? createChildLogger("messages-db");
_parentLogger?: Logger, this.crud = new MessagesCrud(db, logger);
) { this.analysis = new MessagesAnalysis(db, logger);
this.logger = createChildLogger("messages-db"); this.search = new MessagesSearch(db, logger);
this.pagination = new MessagesPagination(db, logger);
this.cleanup = new MessagesCleanup(db, logger);
} }
// ── CRUD ────────────────────────────────────────────────────────────── // ── CRUD ────────────────────────────────────────────────────────────────
async insertMessage(message: MessageRecord): Promise<void> { insertMessage(message: MessageRecord): Promise<void> {
this.logger.debug({ messageId: message.id }, "insertMessage entry"); return this.crud.insertMessage(message);
try {
await this.db
.insert(messagesTable)
.values(message as any)
.onConflictDoNothing();
} catch (error) {
this.logger.error(
{
messageId: message.id,
error: error instanceof Error ? error.message : String(error),
},
"Failed to insert message",
);
throw error;
}
} }
async upsertMessageForCapture(message: MessageRecord): Promise<boolean> { upsertMessageForCapture(message: MessageRecord): Promise<boolean> {
this.logger.debug( return this.crud.upsertMessageForCapture(message);
{ messageId: message.id },
"upsertMessageForCapture entry",
);
try {
const messageWithAIStatus = {
...message,
ai_status: "pending" as const,
};
const rows = await this.db
.insert(messagesTable)
.values(messageWithAIStatus as any)
.onConflictDoNothing()
.returning({ id: messagesTable.id });
return rows.length > 0;
} catch (error) {
this.logger.error(
{
messageId: message.id,
error: error instanceof Error ? error.message : String(error),
},
"Failed to upsert message for capture",
);
throw error;
}
} }
async updateMessageAsEdited( updateMessageAsEdited(
messageId: string, messageId: string,
editedContent: string, editedContent: string,
editedAt: number, editedAt: number,
): Promise<void> { ): Promise<void> {
this.logger.debug({ messageId }, "updateMessageAsEdited entry"); return this.crud.updateMessageAsEdited(messageId, editedContent, editedAt);
try {
await this.db
.update(messagesTable)
.set({
edited_content: editedContent,
edited_at: editedAt,
type: "edited",
ai_status: "pending",
ai_moderation_flags: null,
ai_moderation_score: null,
ai_analysis: null,
ai_categories: null,
ai_severity: null,
ai_confidence: null,
ai_recommended_action: null,
ai_analyzed_at: null,
ai_error: null,
})
.where(eq(messagesTable.id, messageId));
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message as edited",
);
throw error;
}
} }
async updateMessageAsDeleted( updateMessageAsDeleted(messageId: string, deletedAt: number): Promise<void> {
messageId: string, return this.crud.updateMessageAsDeleted(messageId, deletedAt);
deletedAt: number,
): Promise<void> {
this.logger.debug({ messageId }, "updateMessageAsDeleted entry");
try {
await this.db
.update(messagesTable)
.set({
deleted_at: deletedAt,
type: "deleted",
})
.where(eq(messagesTable.id, messageId));
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message as deleted",
);
throw error;
}
} }
async getMessagesByChannel( getMessagesByChannel(
channelId: string, channelId: string,
limit: number = 50, limit?: number,
offset: number = 0, offset?: number,
guildId?: string, guildId?: string,
): Promise<MessageRecord[]> { ): Promise<MessageRecord[]> {
this.logger.debug( return this.crud.getMessagesByChannel(channelId, limit, offset, guildId);
{ channelId, limit, offset, guildId },
"getMessagesByChannel entry",
);
try {
const conditions: SQL[] = [
or(
eq(messagesTable.channel_id, channelId),
eq(messagesTable.thread_id, channelId),
) as SQL,
];
if (guildId) {
conditions.push(eq(messagesTable.guild_id, guildId));
}
const rows = await this.db
.select()
.from(messagesTable)
.where(and(...conditions))
.orderBy(desc(messagesTable.created_at), desc(messagesTable.id))
.limit(limit)
.offset(offset);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
channelId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get messages by channel",
);
throw error;
}
} }
async getMessageById(messageId: string): Promise<MessageRecord | null> { getMessageById(messageId: string): Promise<MessageRecord | null> {
this.logger.debug({ messageId }, "getMessageById entry"); return this.crud.getMessageById(messageId);
try {
const rows = await this.db
.select()
.from(messagesTable)
.where(eq(messagesTable.id, messageId));
return (rows[0] as MessageRecord) ?? null;
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get message by id",
);
throw error;
}
} }
// ── AI Analysis ─────────────────────────────────────────────────────── // ── AI Analysis ─────────────────────────────────────────────────────────
async updateMessageAIAnalysis( updateMessageAIAnalysis(
messageId: string, messageId: string,
result: AIAnalysisUpdate, result: AIAnalysisUpdate,
): Promise<MessageRecord | null> { ): Promise<MessageRecord | null> {
this.logger.debug({ messageId }, "updateMessageAIAnalysis entry"); return this.analysis.updateMessageAIAnalysis(messageId, result);
try {
await this.db
.update(messagesTable)
.set({
ai_status: result.status,
ai_moderation_flags: result.flags ?? null,
ai_moderation_score: result.score ?? null,
ai_analysis: result.analysis ?? null,
ai_categories: stringifyAIList(result.categories),
ai_severity: result.severity ?? null,
ai_confidence: result.confidence ?? result.score ?? null,
ai_recommended_action: result.recommendedAction ?? null,
ai_analyzed_at: result.analyzedAt ?? Date.now(),
ai_error: result.error ?? null,
})
.where(eq(messagesTable.id, messageId));
const rows = await this.db
.select()
.from(messagesTable)
.where(eq(messagesTable.id, messageId));
return (rows[0] as MessageRecord) ?? null;
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message AI analysis",
);
throw error;
}
} }
async updateMessagesAIAnalysisBulk( updateMessagesAIAnalysisBulk(
updates: Array<{ messageId: string; result: AIAnalysisUpdate }>, updates: Array<{ messageId: string; result: AIAnalysisUpdate }>,
): Promise<MessageRecord[]> { ): Promise<MessageRecord[]> {
this.logger.debug( return this.analysis.updateMessagesAIAnalysisBulk(updates);
{ count: updates.length },
"updateMessagesAIAnalysisBulk entry",
);
if (updates.length === 0) return [];
try {
const now = Date.now();
await this.db.transaction(async (tx) => {
for (const { messageId, result } of updates) {
await tx
.update(messagesTable)
.set({
ai_status: result.status,
ai_moderation_flags: result.flags ?? null,
ai_moderation_score: result.score ?? null,
ai_analysis: result.analysis ?? null,
ai_categories: stringifyAIList(result.categories),
ai_severity: result.severity ?? null,
ai_confidence: result.confidence ?? result.score ?? null,
ai_recommended_action: result.recommendedAction ?? null,
ai_analyzed_at: result.analyzedAt ?? now,
ai_error: result.error ?? null,
})
.where(eq(messagesTable.id, messageId));
}
});
const ids = updates.map(({ messageId }) => messageId);
const rows = await this.db
.select()
.from(messagesTable)
.where(inArray(messagesTable.id, ids));
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
error: error instanceof Error ? error.message : String(error),
},
"Failed to bulk update messages AI analysis",
);
throw error;
}
} }
async getPendingAIAnalysisMessages( getPendingAIAnalysisMessages(limit?: number): Promise<MessageRecord[]> {
limit: number = 25, return this.analysis.getPendingAIAnalysisMessages(limit);
): Promise<MessageRecord[]> {
this.logger.debug({ limit }, "getPendingAIAnalysisMessages entry");
try {
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get pending AI analysis messages",
);
throw error;
}
} }
// ── Listing / Pagination ────────────────────────────────────────────── getConversationContextBefore(input: {
async listMessages(query: MessageQuery): Promise<PageResult<MessageRecord>> {
this.logger.debug({ query }, "listMessages entry");
try {
const conditions = buildListMessageConditions(query);
const rows = await this.db
.select()
.from(messagesTable)
.where(conditions.length > 0 ? and(...conditions) : undefined)
.orderBy(desc(messagesTable.created_at), desc(messagesTable.id))
.limit(query.limit + 1);
return pageMessages(rows, query.limit);
} catch (error) {
this.logger.error(
{
query,
error: error instanceof Error ? error.message : String(error),
},
"Failed to list messages",
);
throw error;
}
}
async listReviewMessages(
query: Omit<MessageQuery, "status">,
): Promise<PageResult<MessageRecord>> {
return this.listMessages({
...query,
status: ["warn", "flagged", "error"],
});
}
// ── Conversation Context ──────────────────────────────────────────────
async getConversationContextBefore(input: {
channelId: string; channelId: string;
threadId: string | null; threadId: string | null;
beforeCreatedAt: number; beforeCreatedAt: number;
limit: number; limit: number;
}): Promise<MessageRecord[]> { }): Promise<MessageRecord[]> {
this.logger.debug( return this.analysis.getConversationContextBefore(input);
{ channelId: input.channelId, threadId: input.threadId },
"getConversationContextBefore entry",
);
try {
const { channelId, threadId, beforeCreatedAt, limit } = input;
const locationCondition = threadId
? eq(messagesTable.thread_id, threadId)
: eq(messagesTable.channel_id, channelId);
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
locationCondition,
sql`${messagesTable.created_at} < ${beforeCreatedAt}`,
isNull(messagesTable.deleted_at),
),
)
.orderBy(desc(messagesTable.created_at))
.limit(limit);
return (rows as MessageRecord[]).reverse();
} catch (error) {
this.logger.error(
{
channelId: input.channelId,
threadId: input.threadId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get conversation context before",
);
throw error;
}
} }
async getPendingMessagesByConversation( getPendingMessagesByConversation(
conversationKey: string, conversationKey: string,
limit: number = 200, limit?: number,
): Promise<MessageRecord[]> { ): Promise<MessageRecord[]> {
this.logger.debug( return this.analysis.getPendingMessagesByConversation(
{ conversationKey, limit }, conversationKey,
"getPendingMessagesByConversation entry", limit,
); );
try {
const rows = await this.db.transaction(async (tx) => {
const pendingIdsQuery = tx
.select({ id: messagesTable.id })
.from(messagesTable)
.where(
and(
or(
eq(messagesTable.thread_id, conversationKey),
eq(messagesTable.channel_id, conversationKey),
),
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit)
.for("update", { skipLocked: true });
const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>;
if (pendingIds.length === 0) return [];
return await tx
.update(messagesTable)
.set({ ai_status: "processing", ai_analyzed_at: Date.now() })
.where(
inArray(
messagesTable.id,
pendingIds.map((r) => r.id),
),
)
.returning();
});
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
conversationKey,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get pending messages by conversation",
);
throw error;
}
} }
// ── Conversation Keys ───────────────────────────────────────────────── getPendingConversationKeys(limit?: number): Promise<string[]> {
return this.analysis.getPendingConversationKeys(limit);
async getPendingConversationKeys(limit: number = 500): Promise<string[]> {
this.logger.debug({ limit }, "getPendingConversationKeys entry");
try {
const rows = (await this.db
.selectDistinct({
thread_id: messagesTable.thread_id,
channel_id: messagesTable.channel_id,
})
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.limit(limit)) as Array<{
thread_id: string | null;
channel_id: string;
}>;
const keys: string[] = [];
for (const row of rows) {
const key = row.thread_id || row.channel_id;
if (key && !keys.includes(key)) {
keys.push(key);
}
}
return keys;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get pending conversation keys",
);
throw error;
}
} }
async getConversationKeysWithIncompleteAnalysis( getConversationKeysWithIncompleteAnalysis(limit?: number): Promise<string[]> {
limit: number = 200, return this.analysis.getConversationKeysWithIncompleteAnalysis(limit);
): Promise<string[]> {
this.logger.debug(
{ limit },
"getConversationKeysWithIncompleteAnalysis entry",
);
try {
const rows = (await this.db
.selectDistinct({
thread_id: messagesTable.thread_id,
channel_id: messagesTable.channel_id,
})
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "error"),
sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`,
sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`,
isNull(messagesTable.deleted_at),
),
)
.limit(limit)) as Array<{
thread_id: string | null;
channel_id: string;
}>;
const keys: string[] = [];
for (const row of rows) {
const key = row.thread_id || row.channel_id;
if (key && !keys.includes(key)) {
keys.push(key);
}
}
return keys;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get conversation keys with incomplete analysis",
);
throw error;
}
} }
async getIncompleteMessagesByConversation( getIncompleteMessagesByConversation(
conversationKey: string, conversationKey: string,
limit: number = 500, limit?: number,
): Promise<MessageRecord[]> { ): Promise<MessageRecord[]> {
this.logger.debug( return this.analysis.getIncompleteMessagesByConversation(
{ conversationKey, limit }, conversationKey,
"getIncompleteMessagesByConversation entry", limit,
); );
try {
const rows = await this.db.transaction(async (tx) => {
const pendingIdsQuery = tx
.select({ id: messagesTable.id })
.from(messagesTable)
.where(
and(
or(
eq(messagesTable.thread_id, conversationKey),
eq(messagesTable.channel_id, conversationKey),
),
eq(messagesTable.ai_status, "error"),
sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`,
sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`,
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit)
.for("update", { skipLocked: true });
const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>;
if (pendingIds.length === 0) return [];
return await tx
.update(messagesTable)
.set({ ai_status: "processing", ai_analyzed_at: Date.now() })
.where(
inArray(
messagesTable.id,
pendingIds.map((r) => r.id),
),
)
.returning();
});
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
conversationKey,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get incomplete messages by conversation",
);
throw error;
}
} }
// ── Search ──────────────────────────────────────────────────────────── // ── Search ──────────────────────────────────────────────────────────────
async searchMessages(input: { searchMessages(input: {
query: string; query: string;
channelId?: string; channelId?: string;
guildId?: string; guildId?: string;
limit?: number; limit?: number;
}): Promise<MessageRecord[]> { }): Promise<MessageRecord[]> {
this.logger.debug({ query: input.query }, "searchMessages entry"); return this.search.searchMessages(input);
try {
const { query, channelId, guildId, limit = 20 } = input;
const searchPattern = `%${query}%`;
const conditions: (SQL | undefined)[] = [
isNull(messagesTable.deleted_at),
];
if (guildId) {
conditions.push(eq(messagesTable.guild_id, guildId));
}
if (channelId) {
conditions.push(channelOrThreadCondition(channelId));
}
conditions.push(
or(
sql`${messagesTable.content} LIKE ${searchPattern}`,
sql`${messagesTable.edited_content} LIKE ${searchPattern}`,
),
);
const validConditions = conditions.filter(
(c): c is SQL => c !== undefined,
);
const rows = await this.db
.select()
.from(messagesTable)
.where(and(...validConditions))
.orderBy(desc(messagesTable.created_at))
.limit(limit);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
query: input.query,
channelId: input.channelId,
guildId: input.guildId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to search messages",
);
throw error;
}
} }
// ── Retention / Recovery ────────────────────────────────────────────── // ── Pagination ──────────────────────────────────────────────────────────
async getExpiredMessages(retentionDays: number): Promise<MessageRecord[]> { listMessages(query: MessageQuery): Promise<PageResult<MessageRecord>> {
this.logger.debug({ retentionDays }, "getExpiredMessages entry"); return this.pagination.listMessages(query);
try {
const cutoffTime = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
sql`${messagesTable.created_at} < ${cutoffTime}`,
isNull(messagesTable.deleted_at),
),
)
.limit(1000);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
retentionDays,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get expired messages",
);
throw error;
}
} }
async revertStuckProcessingMessages( listReviewMessages(
timeoutMs: number = 300000, query: Omit<MessageQuery, "status">,
): Promise<number> { ): Promise<PageResult<MessageRecord>> {
this.logger.debug({ timeoutMs }, "revertStuckProcessingMessages entry"); return this.pagination.listReviewMessages(query);
try { }
const cutoffTime = Date.now() - timeoutMs;
const rows = await this.db // ── Cleanup ─────────────────────────────────────────────────────────────
.update(messagesTable)
.set({ ai_status: "pending", ai_analyzed_at: null })
.where(
and(
eq(messagesTable.ai_status, "processing"),
sql`${messagesTable.ai_analyzed_at} < ${cutoffTime}`,
),
)
.returning({ id: messagesTable.id });
if (Array.isArray(rows) && rows.length > 0) { getExpiredMessages(retentionDays: number): Promise<MessageRecord[]> {
this.logger.info( return this.cleanup.getExpiredMessages(retentionDays);
{ }
count: rows.length,
messageIds: rows.map((r: { id: string }) => r.id),
},
"Reverted stuck processing messages back to pending",
);
}
return Array.isArray(rows) ? rows.length : 0; revertStuckProcessingMessages(timeoutMs?: number): Promise<number> {
} catch (error) { return this.cleanup.revertStuckProcessingMessages(timeoutMs);
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to revert stuck processing messages",
);
return 0;
}
} }
} }
@@ -0,0 +1,100 @@
import { decodeCursor, encodeCursor, pageResult } from "@bete/shared";
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, type SQL, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type {
MessageQuery,
MessageRecord,
PageResult,
} from "../message-capture/types.js";
import { channelOrThreadCondition } from "./messages.crud.js";
// ─── Helpers ──────────────────────────────────────────────────────────────────
export function buildListMessageConditions(query: MessageQuery): SQL[] {
const conditions: SQL[] = [];
if (query.guildId) {
conditions.push(eq(messagesTable.guild_id, query.guildId));
}
if (query.channelId) {
conditions.push(channelOrThreadCondition(query.channelId));
}
if (query.threadId) {
conditions.push(eq(messagesTable.thread_id, query.threadId));
}
if (query.userId) {
conditions.push(eq(messagesTable.user_id, query.userId));
}
if (query.status && query.status.length > 0) {
conditions.push(sql`${messagesTable.ai_status} in ${query.status}`);
}
if (query.q) {
const pattern = `%${query.q.toLowerCase()}%`;
conditions.push(sql`lower(${messagesTable.content}) like ${pattern}`);
}
const cursorData = decodeCursor(query.cursor);
if (cursorData) {
conditions.push(
sql`(${messagesTable.created_at} < ${cursorData.created_at} or (${messagesTable.created_at} = ${cursorData.created_at} and ${messagesTable.id} < ${cursorData.id}))`,
);
}
return conditions;
}
const pageRows = pageResult;
// ─── MessagesPagination Class ────────────────────────────────────────────────
export class MessagesPagination {
private logger: Logger;
constructor(
private db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-pagination");
}
async listMessages(query: MessageQuery): Promise<PageResult<MessageRecord>> {
this.logger.debug({ query }, "listMessages entry");
try {
const conditions = buildListMessageConditions(query);
const rows = await this.db
.select()
.from(messagesTable)
.where(conditions.length > 0 ? and(...conditions) : undefined)
.orderBy(desc(messagesTable.created_at), desc(messagesTable.id))
.limit(query.limit + 1);
return pageRows<MessageRecord>(rows, query.limit);
} catch (error) {
this.logger.error(
{
query,
error: error instanceof Error ? error.message : String(error),
},
"Failed to list messages",
);
throw error;
}
}
async listReviewMessages(
query: Omit<MessageQuery, "status">,
): Promise<PageResult<MessageRecord>> {
return this.listMessages({
...query,
status: ["warn", "flagged", "error"],
});
}
}
@@ -0,0 +1,76 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, isNull, or, type SQL, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type { MessageRecord } from "../message-capture/types.js";
import { channelOrThreadCondition } from "./messages.crud.js";
// ─── MessagesSearch Class ─────────────────────────────────────────────────────
export class MessagesSearch {
private logger: Logger;
constructor(
private db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-search");
}
async searchMessages(input: {
query: string;
channelId?: string;
guildId?: string;
limit?: number;
}): Promise<MessageRecord[]> {
this.logger.debug({ query: input.query }, "searchMessages entry");
try {
const { query, channelId, guildId, limit = 20 } = input;
const searchPattern = `%${query}%`;
const conditions: (SQL | undefined)[] = [
isNull(messagesTable.deleted_at),
];
if (guildId) {
conditions.push(eq(messagesTable.guild_id, guildId));
}
if (channelId) {
conditions.push(channelOrThreadCondition(channelId));
}
conditions.push(
or(
sql`${messagesTable.content} LIKE ${searchPattern}`,
sql`${messagesTable.edited_content} LIKE ${searchPattern}`,
),
);
const validConditions = conditions.filter(
(c): c is SQL => c !== undefined,
);
const rows = await this.db
.select()
.from(messagesTable)
.where(and(...validConditions))
.orderBy(desc(messagesTable.created_at))
.limit(limit);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
query: input.query,
channelId: input.channelId,
guildId: input.guildId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to search messages",
);
throw error;
}
}
}
@@ -1,28 +1,11 @@
import { decodeCursor, encodeCursor, pageResult } from "@bete/shared";
import { createChildLogger, type Logger } from "@bete/shared/logger"; import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, type SQL, sql } from "drizzle-orm"; import { and, desc, eq, type SQL, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js"; import type * as schema from "../../shared/database/schema.js";
import { moderationActionsTable } from "../../shared/database/schema.js"; import { moderationActionsTable } from "../../shared/database/schema.js";
import { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
import type { ModerationAction, PageResult } from "../message-capture/types.js"; import type { ModerationAction, PageResult } from "../message-capture/types.js";
// ─── Helpers ────────────────────────────────────────────────────────────────
function pageRows<T extends { created_at: number; id: string }>(
rows: unknown[],
limit: number,
): PageResult<T> {
const hasMore = rows.length > limit;
const data = rows.slice(0, limit) as T[];
const lastItem = data[data.length - 1];
const nextCursor =
hasMore && lastItem
? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id })
: null;
return { data, nextCursor };
}
// ─── ModerationActionsDb Class ────────────────────────────────────────────── // ─── ModerationActionsDb Class ──────────────────────────────────────────────
export class ModerationActionsDb { export class ModerationActionsDb {
@@ -126,7 +109,7 @@ export class ModerationActionsDb {
) )
.limit(limit + 1); .limit(limit + 1);
return pageRows<ModerationAction>(rows, limit); return pageResult<ModerationAction>(rows, limit);
} catch (error) { } catch (error) {
this.logger.error( this.logger.error(
{ error: error instanceof Error ? error.message : String(error) }, { error: error instanceof Error ? error.message : String(error) },
@@ -1,21 +1,36 @@
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("pagination");
export interface CursorData { export interface CursorData {
created_at: number; created_at: number;
id: string; id: string;
} }
export function encodeCursor(data: CursorData): string { export function encodeCursor(data: CursorData): string {
return Buffer.from(JSON.stringify(data)).toString("base64"); const encoded = Buffer.from(JSON.stringify(data)).toString("base64");
logger.debug({ id: data.id, createdAt: data.created_at }, "Encoded cursor");
return encoded;
} }
export function decodeCursor(cursor?: string): CursorData | null { export function decodeCursor(cursor?: string): CursorData | null {
if (!cursor) return null; if (!cursor) {
logger.debug("No cursor provided to decode");
return null;
}
try { try {
const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8")); const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8"));
if (typeof data.created_at === "number" && typeof data.id === "string") { if (typeof data.created_at === "number" && typeof data.id === "string") {
logger.debug(
{ id: data.id, createdAt: data.created_at },
"Decoded cursor",
);
return data; return data;
} }
logger.warn({ cursor }, "Decoded cursor has invalid shape");
return null; return null;
} catch { } catch (err) {
logger.warn({ cursor, error: String(err) }, "Failed to decode cursor");
return null; return null;
} }
} }
@@ -1,28 +1,11 @@
import { decodeCursor, encodeCursor, pageResult } from "@bete/shared";
import { createChildLogger, type Logger } from "@bete/shared/logger"; import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, type SQL, sql } from "drizzle-orm"; import { and, desc, eq, type SQL, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js"; import type * as schema from "../../shared/database/schema.js";
import { messageReviewsTable } from "../../shared/database/schema.js"; import { messageReviewsTable } from "../../shared/database/schema.js";
import { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
import type { MessageReview, PageResult } from "../message-capture/types.js"; import type { MessageReview, PageResult } from "../message-capture/types.js";
// ─── Helpers ────────────────────────────────────────────────────────────────
function pageRows<T extends { created_at: number; id: string }>(
rows: unknown[],
limit: number,
): PageResult<T> {
const hasMore = rows.length > limit;
const data = rows.slice(0, limit) as T[];
const lastItem = data[data.length - 1];
const nextCursor =
hasMore && lastItem
? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id })
: null;
return { data, nextCursor };
}
// ─── ReviewsDb Class ──────────────────────────────────────────────────────── // ─── ReviewsDb Class ────────────────────────────────────────────────────────
export class ReviewsDb { export class ReviewsDb {
@@ -128,7 +111,7 @@ export class ReviewsDb {
) )
.limit(limit + 1); .limit(limit + 1);
return pageRows<MessageReview>(rows, limit); return pageResult<MessageReview>(rows, limit);
} catch (error) { } catch (error) {
this.logger.error( this.logger.error(
{ error: error instanceof Error ? error.message : String(error) }, { error: error instanceof Error ? error.message : String(error) },
@@ -1,4 +1,7 @@
import { spawn } from "node:child_process"; import { spawn } from "node:child_process";
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("ffmpeg-process");
export interface MuxFfmpegArgsOptions { export interface MuxFfmpegArgsOptions {
inputs: string[]; inputs: string[];
@@ -42,19 +45,24 @@ export function buildMuxFfmpegArgs(options: MuxFfmpegArgsOptions): string[] {
*/ */
export function runFfmpeg(args: string[]): Promise<void> { export function runFfmpeg(args: string[]): Promise<void> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
logger.debug({ args }, "Starting ffmpeg");
const proc = spawn("ffmpeg", args, { const proc = spawn("ffmpeg", args, {
stdio: ["ignore", "inherit", "inherit"], stdio: ["ignore", "inherit", "inherit"],
}); });
proc.on("close", (code) => { proc.on("close", (code) => {
if (code === 0) { if (code === 0) {
logger.debug("ffmpeg completed successfully");
resolve(); resolve();
} else { } else {
logger.warn({ exitCode: code }, "ffmpeg exited with non-zero code");
reject(new Error(`ffmpeg exited with code ${code}`)); reject(new Error(`ffmpeg exited with code ${code}`));
} }
}); });
proc.on("error", (err) => { proc.on("error", (err) => {
logger.error({ error: err.message }, "ffmpeg process error");
reject(err); reject(err);
}); });
}); });
@@ -1,4 +1,7 @@
import { Transform, TransformCallback } from "node:stream"; import { Transform, TransformCallback } from "node:stream";
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("packet-filter");
/** /**
* Transform stream to filter out audio packets that are too small. * Transform stream to filter out audio packets that are too small.
@@ -31,6 +34,11 @@ export class PacketFilter extends Transform {
callback(); callback();
} }
/** Returns the number of packets filtered out and processed. */
getStats(): { filtered: number; total: number } {
return { filtered: this.filteredCount, total: this.totalCount };
}
_flush(callback: TransformCallback): void { _flush(callback: TransformCallback): void {
callback(); callback();
} }
@@ -1,6 +1,9 @@
import { createChildLogger } from "@bete/shared/logger";
import { EndBehaviorType, type VoiceReceiver } from "@discordjs/voice"; import { EndBehaviorType, type VoiceReceiver } from "@discordjs/voice";
import { config } from "../../../shared/config/config.js"; import { config } from "../../../shared/config/config.js";
const logger = createChildLogger("audio-stream");
export interface AudioStreamHandlers { export interface AudioStreamHandlers {
onPacket: (chunk: Buffer) => void; onPacket: (chunk: Buffer) => void;
onEnd: () => void; onEnd: () => void;
@@ -12,6 +15,8 @@ export function subscribeToAudioStream(
userId: string, userId: string,
handlers: AudioStreamHandlers, handlers: AudioStreamHandlers,
): NodeJS.ReadableStream { ): NodeJS.ReadableStream {
logger.debug({ userId }, "Subscribing to audio stream");
const audioStream = receiver.subscribe(userId, { const audioStream = receiver.subscribe(userId, {
end: { end: {
behavior: EndBehaviorType.AfterSilence, behavior: EndBehaviorType.AfterSilence,
@@ -20,8 +25,14 @@ export function subscribeToAudioStream(
}); });
audioStream.on("data", handlers.onPacket); audioStream.on("data", handlers.onPacket);
audioStream.on("end", handlers.onEnd); audioStream.on("end", () => {
audioStream.on("error", handlers.onError); logger.debug({ userId }, "Audio stream ended");
handlers.onEnd();
});
audioStream.on("error", (error: Error) => {
logger.warn({ userId, error: error.message }, "Audio stream error");
handlers.onError(error);
});
return audioStream; return audioStream;
} }
@@ -1,6 +1,10 @@
import path from "node:path"; import path from "node:path";
import { createChildLogger } from "@bete/shared/logger";
import type { Client, VoiceChannel } from "discord.js-selfbot-v13"; import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
import { config } from "../../../shared/config/config.js"; import { config } from "../../../shared/config/config.js";
const logger = createChildLogger("voice-metadata");
import type { import type {
SegmentMetadata, SegmentMetadata,
SegmentState, SegmentState,
@@ -12,12 +16,20 @@ export async function collectUserMetadata(
userId: string, userId: string,
channel: VoiceChannel, channel: VoiceChannel,
): Promise<UserMetadata> { ): Promise<UserMetadata> {
logger.debug({ userId }, "Collecting user metadata");
const user = const user =
client.users.cache.get(userId) || client.users.cache.get(userId) ||
(await client.users.fetch(userId).catch(() => null)); (await client.users.fetch(userId).catch(() => {
logger.warn({ userId }, "Failed to fetch user");
return null;
}));
const member = const member =
channel.guild.members.cache.get(userId) || channel.guild.members.cache.get(userId) ||
(await channel.guild.members.fetch(userId).catch(() => null)); (await channel.guild.members.fetch(userId).catch(() => {
logger.warn({ userId }, "Failed to fetch guild member");
return null;
}));
const username = user?.username ?? "Unknown User"; const username = user?.username ?? "Unknown User";
const roles = const roles =
member?.roles.cache member?.roles.cache
@@ -1,8 +1,11 @@
import fs from "node:fs"; import fs from "node:fs";
import path from "node:path"; import path from "node:path";
import { createChildLogger } from "@bete/shared/logger";
import * as prism from "prism-media"; import * as prism from "prism-media";
import type { SegmentState } from "../../message-capture/types.js"; import type { SegmentState } from "../../message-capture/types.js";
const logger = createChildLogger("segment");
export function buildSegmentPaths( export function buildSegmentPaths(
userDir: string, userDir: string,
startTime: number, startTime: number,
@@ -54,6 +57,11 @@ export class SegmentManager {
oggStream, oggStream,
out, out,
}; };
logger.debug(
{ index, startTime, filename, userDir: this.userDir },
"Segment opened",
);
return this.currentSegment; return this.currentSegment;
} }
@@ -64,6 +72,25 @@ export class SegmentManager {
oggPacketStream.unpipe(segment.oggStream); oggPacketStream.unpipe(segment.oggStream);
segment.oggStream.end(); segment.oggStream.end();
this.currentSegment = null; this.currentSegment = null;
// Get file size after closing
let fileSize = 0;
try {
const stat = fs.statSync(segment.filename);
fileSize = stat.size;
} catch {
// File might not exist yet
}
logger.debug(
{
index: segment.index,
filename: segment.filename,
fileSize,
durationMs: (segment.endTime ?? 0) - segment.startTime,
},
"Segment closed",
);
return segment; return segment;
} }
@@ -77,6 +104,15 @@ export class SegmentManager {
) )
) )
return null; return null;
logger.debug(
{
index: this.currentSegment.index,
filename: this.currentSegment.filename,
durationMs: Date.now() - this.currentSegment.startTime,
},
"Segment rotating",
);
this.close(oggPacketStream); this.close(oggPacketStream);
return this.open(oggPacketStream); return this.open(oggPacketStream);
} }
@@ -1,11 +1,14 @@
import fs, { promises as fsPromises } from "node:fs"; import fs, { promises as fsPromises } from "node:fs";
import path from "node:path"; import path from "node:path";
import { createChildLogger } from "@bete/shared/logger";
import type { UserMetadata } from "../../message-capture/types.js"; import type { UserMetadata } from "../../message-capture/types.js";
import { import {
buildMuxFfmpegArgs, buildMuxFfmpegArgs,
runFfmpeg as defaultRunFfmpeg, runFfmpeg as defaultRunFfmpeg,
} from "../ffmpegProcess.js"; } from "../ffmpegProcess.js";
const logger = createChildLogger("recording-session");
export type SessionRecordingStatus = export type SessionRecordingStatus =
| "pending" | "pending"
| "completed" | "completed"
@@ -86,6 +89,16 @@ export function createRecordingSession(
const participants = new Map<string, SessionParticipant>(); const participants = new Map<string, SessionParticipant>();
const segments: SessionSegmentRef[] = []; const segments: SessionSegmentRef[] = [];
logger.info(
{
sessionId,
guildId: options.guildId,
channelId: options.channelId,
channelName: options.channelName,
},
"Recording session created",
);
return { return {
sessionId, sessionId,
recordingsDir: options.recordingsDir, recordingsDir: options.recordingsDir,
@@ -108,6 +121,10 @@ export function createRecordingSession(
durationMs: input.endTime - input.startTime, durationMs: input.endTime - input.startTime,
offsetMs: input.startTime - options.startTime, offsetMs: input.startTime - options.startTime,
}); });
logger.debug(
{ sessionId, userId: input.user.userId, segmentCount: segments.length },
"Segment registered in session",
);
}, },
snapshot(endTime: number): SessionRecordingMetadata { snapshot(endTime: number): SessionRecordingMetadata {
@@ -132,6 +149,11 @@ export function buildSessionMuxFilter(
segments: Array<{ startTime: number }>, segments: Array<{ startTime: number }>,
sessionStartTime: number, sessionStartTime: number,
): string { ): string {
if (segments.length === 0) {
logger.debug("Building mux filter with no segments");
return "";
}
const filters = segments.map((segment, index) => { const filters = segments.map((segment, index) => {
const delayMs = Math.max(0, segment.startTime - sessionStartTime); const delayMs = Math.max(0, segment.startTime - sessionStartTime);
return `[${index}:a]adelay=${delayMs}|${delayMs}[pad${index}]`; return `[${index}:a]adelay=${delayMs}|${delayMs}[pad${index}]`;
@@ -140,6 +162,11 @@ export function buildSessionMuxFilter(
filters.push( filters.push(
`${inputs}amix=inputs=${segments.length}:dropout_transition=0[out]`, `${inputs}amix=inputs=${segments.length}:dropout_transition=0[out]`,
); );
logger.debug(
{ segmentCount: segments.length, filter: filters.join(";") },
"Built mux filter",
);
return filters.join(";"); return filters.join(";");
} }
@@ -166,26 +193,66 @@ export async function finalizeRecordingSession(
await mkdir(sessionDir); await mkdir(sessionDir);
const metadata = session.snapshot(endTime); const metadata = session.snapshot(endTime);
logger.info(
{
sessionId: session.sessionId,
segmentCount: metadata.segments.length,
outputFile,
},
"Finalizing recording session",
);
if (metadata.segments.length === 0) { if (metadata.segments.length === 0) {
await writeJson(metadataFile, { ...metadata, status: "empty" }); await writeJson(metadataFile, { ...metadata, status: "empty" });
logger.info(
{ sessionId: session.sessionId },
"Recording session finalized with no segments",
);
return; return;
} }
try { try {
await runFfmpeg( const ffmpegArgs = buildMuxFfmpegArgs({
buildMuxFfmpegArgs({ inputs: metadata.segments.map((segment) => segment.oggPath),
inputs: metadata.segments.map((segment) => segment.oggPath), filter: buildSessionMuxFilter(metadata.segments, metadata.startTime),
filter: buildSessionMuxFilter(metadata.segments, metadata.startTime), output: outputFile,
output: outputFile, codec: "libopus",
codec: "libopus", });
}),
logger.debug(
{ sessionId: session.sessionId, ffmpegArgs },
"Running FFmpeg mux for session",
); );
await runFfmpeg(ffmpegArgs);
// Get output file size
let outputSize = 0;
try {
const outStat = await fsPromises.stat(outputFile);
outputSize = outStat.size;
} catch {
// File might not exist yet, ignore
}
await writeJson(metadataFile, { await writeJson(metadataFile, {
...metadata, ...metadata,
status: "completed", status: "completed",
outputFile, outputFile,
}); });
logger.info(
{ sessionId: session.sessionId, outputFile, outputSize },
"Recording session finalized successfully",
);
} catch (error) { } catch (error) {
logger.error(
{
sessionId: session.sessionId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to finalize recording session via FFmpeg",
);
await writeJson(metadataFile, { await writeJson(metadataFile, {
...metadata, ...metadata,
status: "failed", status: "failed",
@@ -1,5 +1,8 @@
import { createChildLogger } from "@bete/shared/logger";
import { retryWithBackoff } from "@bete/shared/utils"; import { retryWithBackoff } from "@bete/shared/utils";
const logger = createChildLogger("tele-upload");
export interface TeleUploadResponse { export interface TeleUploadResponse {
download_url: string; download_url: string;
public_id?: string; public_id?: string;
@@ -40,6 +43,8 @@ export async function uploadToTele(input: {
const { buffer, filename, contentType, uploadUrl, timeoutMs, retries } = const { buffer, filename, contentType, uploadUrl, timeoutMs, retries } =
input; input;
logger.debug({ filename, uploadUrl }, "Starting tele upload");
const response = await retryWithBackoff( const response = await retryWithBackoff(
async () => { async () => {
const fileBlob = new Blob([new Uint8Array(buffer)], { const fileBlob = new Blob([new Uint8Array(buffer)], {
@@ -1,3 +1,4 @@
import { pgAttachmentsTable, pgMessagesTable } from "@bete/shared";
import { import {
bigint as pgBigint, bigint as pgBigint,
boolean as pgBoolean, boolean as pgBoolean,
@@ -5,7 +6,6 @@ import {
index as pgIndex, index as pgIndex,
integer as pgInteger, integer as pgInteger,
jsonb as pgJsonb, jsonb as pgJsonb,
real as pgReal,
pgTable, pgTable,
text as pgText, text as pgText,
timestamp as pgTimestamp, timestamp as pgTimestamp,
@@ -41,137 +41,7 @@ export const pgMuxerJobsTable = pgTable(
}), }),
); );
/** // (pgMessagesTable and pgAttachmentsTable are imported from @bete/shared)
* Messages Table (PostgreSQL)
* Stores text messages with AI moderation analysis
*/
export const pgMessagesTable = pgTable(
"messages",
{
id: pgText("id").primaryKey(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
username: pgText("username").notNull(),
avatar_url: pgText("avatar_url"),
content: pgText("content").notNull(),
edited_content: pgText("edited_content"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
edited_at: pgBigint("edited_at", { mode: "number" }),
deleted_at: pgBigint("deleted_at", { mode: "number" }),
type: pgText("type", { enum: ["text", "edited", "deleted"] })
.notNull()
.default("text"),
metadata: pgText("metadata"),
ai_status: pgText("ai_status", {
enum: ["pending", "processing", "clean", "warn", "flagged", "error"],
})
.notNull()
.default("pending"),
ai_moderation_flags: pgText("ai_moderation_flags"),
ai_moderation_score: pgReal("ai_moderation_score"),
ai_analysis: pgText("ai_analysis"),
ai_categories: pgText("ai_categories"),
ai_severity: pgText("ai_severity", {
enum: ["none", "low", "medium", "high", "critical"],
}),
ai_confidence: pgReal("ai_confidence"),
ai_recommended_action: pgText("ai_recommended_action", {
enum: ["none", "monitor", "warn", "review", "delete", "escalate"],
}),
ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }),
ai_error: pgText("ai_error"),
},
(table) => ({
channelIdx: pgIndex("idx_messages_channel").on(table.channel_id),
userIdx: pgIndex("idx_messages_user").on(table.user_id),
createdIdx: pgIndex("idx_messages_created").on(table.created_at),
threadIdx: pgIndex("idx_messages_thread").on(table.thread_id),
channelCreatedIdx: pgIndex("idx_messages_channel_created").on(
table.channel_id,
table.created_at,
table.id,
),
threadCreatedIdx: pgIndex("idx_messages_thread_created").on(
table.thread_id,
table.created_at,
table.id,
),
aiStatusCreatedIdx: pgIndex("idx_messages_ai_status_created").on(
table.ai_status,
table.created_at,
table.id,
),
guildAiStatusCreatedIdx: pgIndex("idx_messages_guild_ai_status_created").on(
table.guild_id,
table.ai_status,
table.created_at,
table.id,
),
guildCreatedDeletedIdx: pgIndex("idx_messages_guild_created_deleted").on(
table.guild_id,
table.created_at,
table.deleted_at,
table.id,
),
channelAiStatusCreatedIdx: pgIndex(
"idx_messages_channel_ai_status_created",
).on(table.channel_id, table.ai_status, table.created_at, table.id),
threadAiStatusCreatedIdx: pgIndex(
"idx_messages_thread_ai_status_created",
).on(table.thread_id, table.ai_status, table.created_at, table.id),
}),
);
/**
* Attachments Table (PostgreSQL)
* Stores attachment metadata with upload status tracking
*/
export const pgAttachmentsTable = pgTable(
"attachments",
{
id: pgText("id").primaryKey(),
message_id: pgText("message_id").notNull(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
filename: pgText("filename").notNull(),
size: pgInteger("size").notNull(),
type: pgText("type").notNull(),
discord_url: pgText("discord_url").notNull(),
uploaded_url: pgText("uploaded_url"),
upload_status: pgText("upload_status", {
enum: ["pending", "uploaded", "failed"],
})
.notNull()
.default("pending"),
upload_error: pgText("upload_error"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
uploaded_at: pgBigint("uploaded_at", { mode: "number" }),
},
(table) => ({
channelIdx: pgIndex("idx_attachments_channel").on(table.channel_id),
messageIdx: pgIndex("idx_attachments_message").on(table.message_id),
statusIdx: pgIndex("idx_attachments_status").on(table.upload_status),
channelCreatedIdx: pgIndex("idx_attachments_channel_created").on(
table.channel_id,
table.created_at,
table.id,
),
threadCreatedIdx: pgIndex("idx_attachments_thread_created").on(
table.thread_id,
table.created_at,
table.id,
),
messageFk: pgForeignKey({
columns: [table.message_id],
foreignColumns: [pgMessagesTable.id],
name: "fk_attachments_message_id",
}).onDelete("cascade"),
}),
);
/** /**
* UI State Table (PostgreSQL) * UI State Table (PostgreSQL)
@@ -7,6 +7,10 @@ import {
skipMedia, skipMedia,
stopMedia, stopMedia,
} from "../../../shared/api/client"; } from "../../../shared/api/client";
import { useAsyncAction } from "../../../shared/hooks/useAsyncAction.js";
import { createLogger } from "../../../shared/lib/logger.js";
const logger = createLogger("use-media-control");
const emptyMediaState: MediaState = { const emptyMediaState: MediaState = {
playing: false, playing: false,
@@ -17,8 +21,7 @@ const emptyMediaState: MediaState = {
export function useMediaControl() { export function useMediaControl() {
const [mediaState, setMediaState] = useState<MediaState>(emptyMediaState); const [mediaState, setMediaState] = useState<MediaState>(emptyMediaState);
const [loading, setLoading] = useState(false); const { loading, error, execute, clearError } = useAsyncAction();
const [error, setError] = useState<string | null>(null);
const refreshMedia = useCallback(async () => { const refreshMedia = useCallback(async () => {
const state = await getMediaStatus(); const state = await getMediaStatus();
@@ -28,71 +31,62 @@ export function useMediaControl() {
const enqueue = useCallback( const enqueue = useCallback(
async (source: string, mode: "music" | "screen") => { async (source: string, mode: "music" | "screen") => {
setLoading(true); const result = await execute(() => queueMedia(source, mode));
setError(null); if (result) {
try { setMediaState(result);
const state = await queueMedia(source, mode); logger.info("Media queued", { source, mode });
setMediaState(state); } else {
return state; logger.error("Failed to queue media", { source, mode });
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
} }
return result;
}, },
[], [execute],
); );
const skip = useCallback(async () => { const skip = useCallback(async () => {
setLoading(true); const result = await execute(() => skipMedia());
setError(null); if (result) {
try { setMediaState(result);
const state = await skipMedia(); logger.info("Media skipped");
setMediaState(state); } else {
return state; logger.error("Failed to skip media");
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
} }
}, []); return result;
}, [execute]);
const stop = useCallback(async () => { const stop = useCallback(async () => {
setLoading(true); const result = await execute(() => stopMedia());
setError(null); if (result) {
try { setMediaState(result);
const state = await stopMedia(); logger.info("Media stopped");
setMediaState(state); } else {
return state; logger.error("Failed to stop media");
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
} }
}, []); return result;
}, [execute]);
const setVolume = useCallback(async (volume: number) => { const setVolume = useCallback(
setError(null); async (volume: number) => {
try { clearError();
const state = await setMediaVolume(volume); try {
setMediaState(state); const state = await setMediaVolume(volume);
return state; setMediaState(state);
} catch (err) { logger.info("Volume set", { volume });
const message = err instanceof Error ? err.message : String(err); return state;
setError(message); } catch (err) {
throw err; const message = err instanceof Error ? err.message : String(err);
} logger.error("Failed to set volume", { volume, error: message });
}, []); throw err;
}
},
[clearError],
);
useEffect(() => { useEffect(() => {
refreshMedia().catch((err) => refreshMedia().catch((err) =>
setError(err instanceof Error ? err.message : String(err)), logger.error("Failed to refresh media state on mount", {
error: String(err),
}),
); );
}, [refreshMedia]); }, [refreshMedia]);
@@ -8,6 +8,10 @@ import {
getVoiceChannels, getVoiceChannels,
getVoiceStatus, getVoiceStatus,
} from "../../../shared/api/client"; } from "../../../shared/api/client";
import { useAsyncAction } from "../../../shared/hooks/useAsyncAction.js";
import { createLogger } from "../../../shared/lib/logger.js";
const logger = createLogger("use-voice-control");
export function useVoiceControl() { export function useVoiceControl() {
const [guilds, setGuilds] = useState<Guild[]>([]); const [guilds, setGuilds] = useState<Guild[]>([]);
@@ -19,15 +23,14 @@ export function useVoiceControl() {
activeChannelId: null, activeChannelId: null,
activeChannelName: null, activeChannelName: null,
}); });
const [loading, setLoading] = useState(false); const { loading, error, execute, clearError } = useAsyncAction();
const [error, setError] = useState<string | null>(null);
const refreshGuilds = useCallback(async () => { const refreshGuilds = useCallback(async () => {
setError(null); clearError();
const nextGuilds = await getGuilds(); const nextGuilds = await getGuilds();
setGuilds(nextGuilds); setGuilds(nextGuilds);
return nextGuilds; return nextGuilds;
}, []); }, [clearError]);
const refreshVoiceStatus = useCallback(async () => { const refreshVoiceStatus = useCallback(async () => {
const status = await getVoiceStatus(); const status = await getVoiceStatus();
@@ -55,44 +58,39 @@ export function useVoiceControl() {
return channels; return channels;
}, []); }, []);
const joinVoice = useCallback(async (guildId: string, channelId: string) => { const joinVoice = useCallback(
setLoading(true); async (guildId: string, channelId: string) => {
setError(null); const result = await execute(() => connectVoice(guildId, channelId));
try { if (result) {
const status = await connectVoice(guildId, channelId); setVoiceStatus(result);
setVoiceStatus(status); logger.info("Connected to voice", { guildId, channelId });
return status; } else {
} catch (err) { logger.error("Failed to connect to voice", { guildId, channelId });
const message = err instanceof Error ? err.message : String(err); }
setError(message); return result;
throw err; },
} finally { [execute],
setLoading(false); );
}
}, []);
const leaveVoice = useCallback(async () => { const leaveVoice = useCallback(async () => {
setLoading(true); const result = await execute(() => disconnectVoice());
setError(null); if (result) {
try { setVoiceStatus(result);
const status = await disconnectVoice(); logger.info("Disconnected from voice");
setVoiceStatus(status); } else {
return status; logger.error("Failed to disconnect from voice");
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
} }
}, []); return result;
}, [execute]);
useEffect(() => { useEffect(() => {
refreshGuilds().catch((err) => refreshGuilds().catch((err) =>
setError(err instanceof Error ? err.message : String(err)), logger.error("Failed to refresh guilds on mount", { error: String(err) }),
); );
refreshVoiceStatus().catch((err) => refreshVoiceStatus().catch((err) =>
setError(err instanceof Error ? err.message : String(err)), logger.error("Failed to refresh voice status on mount", {
error: String(err),
}),
); );
}, [refreshGuilds, refreshVoiceStatus]); }, [refreshGuilds, refreshVoiceStatus]);
@@ -5,6 +5,9 @@ import {
reanalyzeErrorBatch, reanalyzeErrorBatch,
reanalyzeMessage, reanalyzeMessage,
} from "../../../shared/api/client"; } from "../../../shared/api/client";
import { createLogger } from "../../../shared/lib/logger.js";
const logger = createLogger("use-messages");
const PAGE_SIZE = 100; const PAGE_SIZE = 100;
@@ -54,6 +57,7 @@ export function useMessages() {
} catch (err) { } catch (err) {
const message = err instanceof Error ? err.message : String(err); const message = err instanceof Error ? err.message : String(err);
setError(message); setError(message);
logger.error("Failed to fetch messages", { guildId, error: message });
throw err; throw err;
} finally { } finally {
setLoading(false); setLoading(false);
@@ -72,6 +76,9 @@ export function useMessages() {
setMessages((prev) => [...prev, ...result.data]); setMessages((prev) => [...prev, ...result.data]);
setCursor(result.nextCursor); setCursor(result.nextCursor);
setHasMore(!!result.nextCursor); setHasMore(!!result.nextCursor);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to load more messages", { error: message });
} finally { } finally {
setLoadingMore(false); setLoadingMore(false);
} }
@@ -106,6 +113,8 @@ export function useMessages() {
prev.map((message) => (message.id === id ? snapshot : message)), prev.map((message) => (message.id === id ? snapshot : message)),
); );
} }
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to reanalyze message", { id, error: message });
throw err; throw err;
} }
}, []); }, []);
@@ -124,10 +133,17 @@ export function useMessages() {
: message, : message,
), ),
); );
const { count } = await reanalyzeErrorBatch({ try {
guildId: currentGuild.current ?? undefined, const { count } = await reanalyzeErrorBatch({
}); guildId: currentGuild.current ?? undefined,
return count; });
logger.info("Reanalyze all errors complete", { count });
return count;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to reanalyze error batch", { error: message });
throw err;
}
}, []); }, []);
return { return {
+9 -1
View File
@@ -1,6 +1,9 @@
// ─── Shared HTTP client — all API endpoints in one file ────────────────────── // ─── Shared HTTP client — all API endpoints in one file ──────────────────────
import type { MessageRecord, PageResult } from "@bete/shared"; import type { MessageRecord, PageResult } from "@bete/shared";
import { createLogger } from "../lib/logger.js";
const logger = createLogger("api");
const BE_API_URL = import.meta.env.VITE_BE_API_URL || "http://localhost:3001"; const BE_API_URL = import.meta.env.VITE_BE_API_URL || "http://localhost:3001";
const BE_WS_URL = import.meta.env.VITE_BE_WS_URL || "ws://localhost:3001"; const BE_WS_URL = import.meta.env.VITE_BE_WS_URL || "ws://localhost:3001";
@@ -20,6 +23,8 @@ class ApiError extends Error {
export async function request<T>(path: string, init?: RequestInit): Promise<T> { export async function request<T>(path: string, init?: RequestInit): Promise<T> {
const password = localStorage.getItem("admin-password"); const password = localStorage.getItem("admin-password");
const url = path.startsWith("http") ? path : `${BE_API_URL}${path}`; const url = path.startsWith("http") ? path : `${BE_API_URL}${path}`;
logger.debug("Request", { method: init?.method ?? "GET", url });
const res = await fetch(url, { const res = await fetch(url, {
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
@@ -38,10 +43,13 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
} catch { } catch {
// ignore parse errors // ignore parse errors
} }
logger.error("Request failed", { url, status: res.status, code, message });
throw new ApiError(code, message, res.status); throw new ApiError(code, message, res.status);
} }
return res.json() as Promise<T>; const result = (await res.json()) as T;
logger.debug("Response", { url, status: res.status });
return result;
} }
export function getWebSocketURL(): string { export function getWebSocketURL(): string {
@@ -0,0 +1,37 @@
// ─── Generic async action state hook ──────────────────────────────────────
import { useCallback, useState } from "react";
interface AsyncActionState {
loading: boolean;
error: string | null;
}
export function useAsyncAction() {
const [state, setState] = useState<AsyncActionState>({
loading: false,
error: null,
});
const execute = useCallback(
async <T>(fn: () => Promise<T>): Promise<T | null> => {
setState({ loading: true, error: null });
try {
const result = await fn();
setState({ loading: false, error: null });
return result;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setState({ loading: false, error: message });
return null;
}
},
[],
);
const clearError = useCallback(() => {
setState((prev) => ({ ...prev, error: null }));
}, []);
return { ...state, execute, clearError };
}
@@ -1,5 +1,8 @@
// ─── Audio playback hook — receives PCM from WebSocket and plays through Web Audio API ── // ─── Audio playback hook — receives PCM from WebSocket and plays through Web Audio API ──
import { useCallback, useRef, useState } from "react"; import { useCallback, useRef, useState } from "react";
import { createLogger } from "../lib/logger.js";
const logger = createLogger("use-audio-playback");
const SAMPLE_RATE = 24000; const SAMPLE_RATE = 24000;
const CHANNELS = 1; const CHANNELS = 1;
@@ -15,56 +18,64 @@ export function useAudioPlayback() {
const handleIncomingPcm = useCallback( const handleIncomingPcm = useCallback(
(data: { userId: string; pcm: string }) => { (data: { userId: string; pcm: string }) => {
// Decode base64 PCM data // Decode base64 PCM data
const binaryString = atob(data.pcm); try {
const bytes = new Uint8Array(binaryString.length); const binaryString = atob(data.pcm);
for (let i = 0; i < binaryString.length; i++) { const bytes = new Uint8Array(binaryString.length);
bytes[i] = binaryString.charCodeAt(i); for (let i = 0; i < binaryString.length; i++) {
} bytes[i] = binaryString.charCodeAt(i);
const int16Array = new Int16Array(bytes.buffer); }
const int16Array = new Int16Array(bytes.buffer);
// Calculate audio levels for visualization // Calculate audio levels for visualization
let sum = 0; let sum = 0;
for (const sample of int16Array) sum += Math.abs(sample / 32768); for (const sample of int16Array) sum += Math.abs(sample / 32768);
const average = int16Array.length ? sum / int16Array.length : 0; const average = int16Array.length ? sum / int16Array.length : 0;
setLevels((prev) => setLevels((prev) =>
prev.map((_, index) => prev.map((_, index) =>
Math.max( Math.max(
0.04, 0.04,
average * average *
(0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) * (0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) *
5, 5,
),
), ),
), );
);
const audioContext = audioContextRef.current; const audioContext = audioContextRef.current;
if (!isListening || !audioContext) return; if (!isListening || !audioContext) return;
// Convert to float32 for Web Audio API // Convert to float32 for Web Audio API
const float32Array = new Float32Array(int16Array.length); const float32Array = new Float32Array(int16Array.length);
for (let i = 0; i < int16Array.length; i++) for (let i = 0; i < int16Array.length; i++)
float32Array[i] = int16Array[i] / 32768; float32Array[i] = int16Array[i] / 32768;
const audioBuffer = audioContext.createBuffer( const audioBuffer = audioContext.createBuffer(
CHANNELS, CHANNELS,
float32Array.length, float32Array.length,
SAMPLE_RATE, SAMPLE_RATE,
); );
audioBuffer.getChannelData(0).set(float32Array); audioBuffer.getChannelData(0).set(float32Array);
const source = audioContext.createBufferSource(); const source = audioContext.createBufferSource();
source.buffer = audioBuffer; source.buffer = audioBuffer;
source.connect(audioContext.destination); source.connect(audioContext.destination);
// Schedule playback per user to avoid overlaps // Schedule playback per user to avoid overlaps
const currentTime = audioContext.currentTime; const currentTime = audioContext.currentTime;
let nextStart = userTimelinesRef.current.get(data.userId) || 0; let nextStart = userTimelinesRef.current.get(data.userId) || 0;
if (nextStart < currentTime) nextStart = currentTime + 0.05; if (nextStart < currentTime) nextStart = currentTime + 0.05;
source.start(nextStart); source.start(nextStart);
userTimelinesRef.current.set( userTimelinesRef.current.set(
data.userId, data.userId,
nextStart + audioBuffer.duration, nextStart + audioBuffer.duration,
); );
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to decode PCM audio", {
userId: data.userId,
error: message,
});
}
}, },
[isListening], [isListening],
); );
@@ -74,6 +85,7 @@ export function useAudioPlayback() {
await audioContextRef.current?.suspend(); await audioContextRef.current?.suspend();
userTimelinesRef.current.clear(); userTimelinesRef.current.clear();
setIsListening(false); setIsListening(false);
logger.info("Audio playback paused");
return; return;
} }
const AudioContextCtor = const AudioContextCtor =
@@ -85,6 +97,7 @@ export function useAudioPlayback() {
}); });
await audioContextRef.current.resume(); await audioContextRef.current.resume();
setIsListening(true); setIsListening(true);
logger.info("Audio playback started");
}, [isListening]); }, [isListening]);
return { return {
@@ -0,0 +1,22 @@
// ─── Client-side structured logger ────────────────────────────────────────
const LOG_PREFIX = "[Bete]";
export function createLogger(context: string) {
const prefix = `${LOG_PREFIX} [${context}]`;
return {
debug: (msg: string, data?: Record<string, unknown>) => {
if (import.meta.env.DEV) console.debug(prefix, msg, data ?? "");
},
info: (msg: string, data?: Record<string, unknown>) => {
console.info(prefix, msg, data ?? "");
},
warn: (msg: string, data?: Record<string, unknown>) => {
console.warn(prefix, msg, data ?? "");
},
error: (msg: string, data?: Record<string, unknown>) => {
console.error(prefix, msg, data ?? "");
},
};
}
+21 -4
View File
@@ -1,5 +1,8 @@
// ─── WebSocket singleton with reconnect, typed events, and observable status ─ // ─── WebSocket singleton with reconnect, typed events, and observable status ─
import { useCallback, useEffect, useRef, useState } from "react"; import { useCallback, useEffect, useRef, useState } from "react";
import { createLogger } from "../lib/logger.js";
const logger = createLogger("socket");
export type WsStatus = "connecting" | "connected" | "disconnected" | "error"; export type WsStatus = "connecting" | "connected" | "disconnected" | "error";
@@ -26,6 +29,7 @@ export interface WsHandlers {
let _wsInstance: WebSocket | null = null; let _wsInstance: WebSocket | null = null;
let _reconnectTimer: ReturnType<typeof setTimeout> | null = null; let _reconnectTimer: ReturnType<typeof setTimeout> | null = null;
let _closed = false; let _closed = false;
let _reconnectAttempts = 0;
const _listeners = new Set<WsHandlers>(); const _listeners = new Set<WsHandlers>();
const _statusCallbacks = new Set<(s: WsStatus) => void>(); const _statusCallbacks = new Set<(s: WsStatus) => void>();
@@ -41,12 +45,23 @@ function doConnect(): WebSocket {
const ws = new WebSocket(url); const ws = new WebSocket(url);
ws.binaryType = "arraybuffer"; ws.binaryType = "arraybuffer";
dispatchStatus("connecting"); dispatchStatus("connecting");
logger.info("Connecting", { url });
ws.addEventListener("open", () => dispatchStatus("connected")); ws.addEventListener("open", () => {
ws.addEventListener("error", () => dispatchStatus("error")); _reconnectAttempts = 0;
ws.addEventListener("close", () => { dispatchStatus("connected");
logger.info("Connected");
});
ws.addEventListener("error", () => {
dispatchStatus("error");
logger.error("WebSocket error");
});
ws.addEventListener("close", (event) => {
dispatchStatus("disconnected"); dispatchStatus("disconnected");
logger.info("Disconnected", { code: event.code, reason: event.reason });
if (!_closed && _listeners.size > 0) { if (!_closed && _listeners.size > 0) {
_reconnectAttempts++;
logger.warn("Reconnecting", { attempt: _reconnectAttempts });
_reconnectTimer = setTimeout(() => doReconnect(), 2500); _reconnectTimer = setTimeout(() => doReconnect(), 2500);
} }
}); });
@@ -108,7 +123,9 @@ function doConnect(): WebSocket {
} }
} }
} catch { } catch {
// ignore malformed messages logger.error("Failed to parse message", {
raw: event.data.slice(0, 200),
});
} }
}); });