refactor: atomic, DRY, and logging improvements across codebase

- Split llmModerationClient.ts (2170 lines) into 5 focused sub-modules
- Split aiAnalyzer.ts (1282 lines) into 4 modular pipelines
- Split messages.db.ts (826 lines) into 5 domain-specific modules
- Moved shared schema to @bete/shared, eliminated backend duplication
- Added createChildLogger to all voice-recording and AI moderation modules
- Extracted tryCommandThenFallback, normalizeMediaState, DEFAULT_VOICE_STATUS
- Created shared pagination.ts utility, eliminated 5+ cursor-pagination duplications
- Created shared messageMapper.ts for row mapping
- Standardized backend error handling with asyncHandler
- Added frontend createLogger utility and useAsyncAction hook
- Added structured logging to frontend hooks, socket, and API client

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-06-09 19:46:08 +07:00
co-authored by Claude Opus 4.8
parent b68789fffc
commit 07032ab521
61 changed files with 3808 additions and 3043 deletions
@@ -1,5 +1,9 @@
import { createChildLogger } from "@bete/shared/logger";
import { getPool } from "../../shared/database/index.js";
import {
type MappedMessage,
mapMessageRow,
} from "../../shared/utils/messageMapper.js";
const logger = createChildLogger("analysis.repository");
@@ -10,61 +14,8 @@ export interface AnalysisSearchQuery {
limit?: number;
}
export interface AnalysisSearchResult {
id: string;
guild_id: string;
channel_id: string;
thread_id: string | null;
user_id: string;
username: string;
avatar_url: string | null;
content: string;
edited_content: string | null;
created_at: number;
edited_at: number | null;
deleted_at: number | null;
type: string;
metadata: string | null;
ai_status: string | null;
ai_moderation_flags: string | null;
ai_moderation_score: number | null;
ai_analysis: string | null;
ai_categories: string | null;
ai_severity: string | null;
ai_confidence: number | null;
ai_recommended_action: string | null;
ai_analyzed_at: number | null;
ai_error: string | null;
}
function mapSearchResult(row: Record<string, unknown>): AnalysisSearchResult {
return {
id: String(row.id ?? ""),
guild_id: String(row.guild_id ?? ""),
channel_id: String(row.channel_id ?? ""),
thread_id: (row.thread_id as string | null) ?? null,
user_id: String(row.user_id ?? ""),
username: String(row.username ?? ""),
avatar_url: (row.avatar_url as string | null) ?? null,
content: String(row.content ?? ""),
edited_content: (row.edited_content as string | null) ?? null,
created_at: Number(row.created_at ?? 0),
edited_at: (row.edited_at as number | null) ?? null,
deleted_at: (row.deleted_at as number | null) ?? null,
type: String(row.type ?? "text"),
metadata: (row.metadata as string | null) ?? null,
ai_status: (row.ai_status as string | null) ?? null,
ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null,
ai_moderation_score: (row.ai_moderation_score as number | null) ?? null,
ai_analysis: (row.ai_analysis as string | null) ?? null,
ai_categories: (row.ai_categories as string | null) ?? null,
ai_severity: (row.ai_severity as string | null) ?? null,
ai_confidence: (row.ai_confidence as number | null) ?? null,
ai_recommended_action: (row.ai_recommended_action as string | null) ?? null,
ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null,
ai_error: (row.ai_error as string | null) ?? null,
};
}
// AnalysisSearchResult is identical to MappedMessage — reuse the shared mapper
export type AnalysisSearchResult = MappedMessage;
export class AnalysisRepository {
async search(query: AnalysisSearchQuery): Promise<AnalysisSearchResult[]> {
@@ -107,7 +58,7 @@ export class AnalysisRepository {
[...params, limit],
);
return rows.map((r) => mapSearchResult(r as Record<string, unknown>));
return rows.map((r) => mapMessageRow(r as Record<string, unknown>));
}
}
@@ -1,16 +1,12 @@
import type { NextFunction, Request, Response } from "express";
import type { Request, Response } from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import { healthService } from "./health.service.js";
export function handleHealthCheck(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
export const handleHealthCheck = asyncHandler(
async (req: Request, res: Response) => {
const verbose = req.query.verbose === "true";
const result = await healthService.getHealth(verbose);
const status = result.status === "healthy" ? 200 : 503;
res.status(status).json(result);
})(req, res, next);
}
},
);
@@ -1,5 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response } from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import { mascotChatService } from "./mascot-chat.service.js";
const logger = createChildLogger("mascot-chat.controller");
@@ -8,8 +9,8 @@ interface AuthenticatedRequest extends Request {
userId?: string;
}
export async function handleMascotChat(req: Request, res: Response) {
try {
export const handleMascotChat = asyncHandler(
async (req: Request, res: Response) => {
const { message, context } = req.body;
if (!message || typeof message !== "string") {
@@ -49,17 +50,11 @@ export async function handleMascotChat(req: Request, res: Response) {
response,
timestamp: new Date().toISOString(),
});
} catch (error) {
logger.error({ error }, "Error processing mascot chat");
res.status(500).json({
error: "INTERNAL_SERVER_ERROR",
message: "Failed to process mascot chat",
});
}
}
},
);
export async function getMascotChatHistory(req: Request, res: Response) {
try {
export const getMascotChatHistory = asyncHandler(
async (req: Request, res: Response) => {
const userId = (req as AuthenticatedRequest).userId || "anonymous";
const limit = Math.min(parseInt(req.query.limit as string) || 50, 100);
@@ -69,17 +64,11 @@ export async function getMascotChatHistory(req: Request, res: Response) {
history,
total: history.length,
});
} catch (error) {
logger.error({ error }, "Error fetching chat history");
res.status(500).json({
error: "INTERNAL_SERVER_ERROR",
message: "Failed to fetch chat history",
});
}
}
},
);
export async function clearMascotChatHistory(req: Request, res: Response) {
try {
export const clearMascotChatHistory = asyncHandler(
async (req: Request, res: Response) => {
const userId = (req as AuthenticatedRequest).userId || "anonymous";
await mascotChatService.clearChatHistory(userId);
@@ -87,11 +76,5 @@ export async function clearMascotChatHistory(req: Request, res: Response) {
res.status(200).json({
message: "Chat history cleared successfully",
});
} catch (error) {
logger.error({ error }, "Error clearing chat history");
res.status(500).json({
error: "INTERNAL_SERVER_ERROR",
message: "Failed to clear chat history",
});
}
}
},
);
@@ -1,5 +1,4 @@
import express, { type Router } from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import {
clearMascotChatHistory,
getMascotChatHistory,
@@ -9,9 +8,9 @@ import {
export function createMascotChatRouter(): Router {
const router = express.Router();
router.post("/mascot/chat", asyncHandler(handleMascotChat));
router.get("/mascot/chat/history", asyncHandler(getMascotChatHistory));
router.delete("/mascot/chat/history", asyncHandler(clearMascotChatHistory));
router.post("/mascot/chat", handleMascotChat);
router.get("/mascot/chat/history", getMascotChatHistory);
router.delete("/mascot/chat/history", clearMascotChatHistory);
return router;
}
@@ -5,7 +5,10 @@ import {
COMMAND_MEDIA_VOLUME,
MEDIA_STATUS_KEY,
} from "@bete/shared";
import { createChildLogger } from "@bete/shared/logger";
import {
createChildLogger,
tryCommandThenFallback,
} from "../../shared/commandHelper.js";
import { publishCommand, readRedisStatus } from "../../shared/redis/index.js";
const logger = createChildLogger("media.service");
@@ -43,6 +46,35 @@ const DEFAULT_STATE: MediaState = {
queue: [],
};
// ---------------------------------------------------------------------------
// Normalisation — handle both boolean (new) and string (legacy) playing values
// ---------------------------------------------------------------------------
function normalizeMediaState(raw: Record<string, unknown>): MediaState {
const rawPlaying = raw.playing;
const playing =
rawPlaying === true ||
rawPlaying === "playing" ||
rawPlaying === "buffering";
return {
playing,
musicVolume: Number(raw.musicVolume ?? 1.0),
current: (raw.current as MediaItem | null) ?? null,
queue: (raw.queue as MediaItem[]) ?? [],
};
}
type MediaReplyData = Record<string, unknown> | MediaState;
function fromReply(data: MediaReplyData): MediaState {
return normalizeMediaState(data as Record<string, unknown>);
}
async function readStatusFallback(): Promise<MediaState> {
const cached = await readRedisStatus(MEDIA_STATUS_KEY);
return cached ? normalizeMediaState(cached) : DEFAULT_STATE;
}
// ---------------------------------------------------------------------------
// Service methods
// ---------------------------------------------------------------------------
@@ -52,24 +84,7 @@ const DEFAULT_STATE: MediaState = {
*/
export async function getStatus(): Promise<MediaState> {
logger.debug("getStatus called");
const cached = await readRedisStatus(MEDIA_STATUS_KEY);
if (cached) {
const rawPlaying = cached.playing;
// Handle both boolean (new) and string (legacy from String(discordPlayer.getStatus()))
const playing =
rawPlaying === true ||
rawPlaying === "playing" ||
rawPlaying === "buffering";
return {
playing,
musicVolume: Number(cached.musicVolume ?? 1.0),
current: (cached.current as MediaItem | null) ?? null,
queue: (cached.queue as MediaItem[]) ?? [],
};
}
return DEFAULT_STATE;
return readStatusFallback();
}
/**
@@ -80,26 +95,16 @@ export async function queue(
mode: "music" | "screen" = "music",
): Promise<MediaState> {
logger.info({ source, mode }, "queue called");
const reply = await publishCommand<MediaState>(
COMMAND_MEDIA_QUEUE,
{ source, mode },
DEFAULT_COMMAND_TIMEOUT_MS,
return tryCommandThenFallback(
() =>
publishCommand<MediaState>(
COMMAND_MEDIA_QUEUE,
{ source, mode },
DEFAULT_COMMAND_TIMEOUT_MS,
),
() => readStatusFallback(),
"queue",
);
if (reply?.success && reply.data) {
return {
playing: reply.data.playing,
musicVolume: reply.data.musicVolume,
current: reply.data.current ?? null,
queue: reply.data.queue ?? [],
};
}
logger.warn(
{ source, mode },
"discord-gateway unreachable, returning current media status",
);
return getStatus();
}
/**
@@ -107,23 +112,16 @@ export async function queue(
*/
export async function skip(): Promise<MediaState> {
logger.info("skip called");
const reply = await publishCommand<MediaState>(
COMMAND_MEDIA_SKIP,
{},
DEFAULT_COMMAND_TIMEOUT_MS,
return tryCommandThenFallback(
() =>
publishCommand<MediaState>(
COMMAND_MEDIA_SKIP,
{},
DEFAULT_COMMAND_TIMEOUT_MS,
),
() => readStatusFallback(),
"skip",
);
if (reply?.success && reply.data) {
return {
playing: reply.data.playing,
musicVolume: reply.data.musicVolume,
current: reply.data.current ?? null,
queue: reply.data.queue ?? [],
};
}
logger.warn("discord-gateway unreachable, returning current media status");
return getStatus();
}
/**
@@ -131,23 +129,16 @@ export async function skip(): Promise<MediaState> {
*/
export async function stop(): Promise<MediaState> {
logger.info("stop called");
const reply = await publishCommand<MediaState>(
COMMAND_MEDIA_STOP,
{},
DEFAULT_COMMAND_TIMEOUT_MS,
return tryCommandThenFallback(
() =>
publishCommand<MediaState>(
COMMAND_MEDIA_STOP,
{},
DEFAULT_COMMAND_TIMEOUT_MS,
),
() => readStatusFallback(),
"stop",
);
if (reply?.success && reply.data) {
return {
playing: reply.data.playing,
musicVolume: reply.data.musicVolume,
current: reply.data.current ?? null,
queue: reply.data.queue ?? [],
};
}
logger.warn("discord-gateway unreachable, returning current media status");
return getStatus();
}
/**
@@ -155,24 +146,14 @@ export async function stop(): Promise<MediaState> {
*/
export async function setVolume(volume: number): Promise<MediaState> {
logger.info({ volume }, "setVolume called");
const reply = await publishCommand<MediaState>(
COMMAND_MEDIA_VOLUME,
{ volume },
DEFAULT_COMMAND_TIMEOUT_MS,
return tryCommandThenFallback(
() =>
publishCommand<MediaState>(
COMMAND_MEDIA_VOLUME,
{ volume },
DEFAULT_COMMAND_TIMEOUT_MS,
),
() => readStatusFallback(),
"setVolume",
);
if (reply?.success && reply.data) {
return {
playing: reply.data.playing,
musicVolume: reply.data.musicVolume,
current: reply.data.current ?? null,
queue: reply.data.queue ?? [],
};
}
logger.warn(
{ volume },
"discord-gateway unreachable, returning current media status",
);
return getStatus();
}
@@ -1,14 +1,9 @@
import type { PageResult } from "@bete/shared";
import { pgAttachmentsTable, pgMessagesTable } from "@bete/shared";
import { createChildLogger } from "@bete/shared/logger";
import { and, desc, eq, inArray, lt, ne, type SQL } from "drizzle-orm";
import {
bigint as pgBigint,
integer as pgInteger,
real as pgReal,
pgTable,
text as pgText,
} from "drizzle-orm/pg-core";
import { getDatabase } from "../../shared/database/index.js";
import { mapMessageRow } from "../../shared/utils/messageMapper.js";
import type {
MessageCreate,
MessageQuery,
@@ -17,71 +12,6 @@ import type {
const logger = createChildLogger("messages.repository");
/**
* Local table definitions mirroring services/discord-gateway/src/shared/database/schema.ts.
* These are query-building references only — schema source of truth remains in discord-gateway.
*/
const messages = pgTable("messages", {
id: pgText("id").primaryKey(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
username: pgText("username").notNull(),
avatar_url: pgText("avatar_url"),
content: pgText("content").notNull(),
edited_content: pgText("edited_content"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
edited_at: pgBigint("edited_at", { mode: "number" }),
deleted_at: pgBigint("deleted_at", { mode: "number" }),
type: pgText("type", {
enum: ["text", "edited", "deleted"],
})
.notNull()
.default("text"),
metadata: pgText("metadata"),
ai_status: pgText("ai_status", {
enum: ["pending", "processing", "clean", "warn", "flagged", "error"],
})
.notNull()
.default("pending"),
ai_moderation_flags: pgText("ai_moderation_flags"),
ai_moderation_score: pgReal("ai_moderation_score"),
ai_analysis: pgText("ai_analysis"),
ai_categories: pgText("ai_categories"),
ai_severity: pgText("ai_severity", {
enum: ["none", "low", "medium", "high", "critical"],
}),
ai_confidence: pgReal("ai_confidence"),
ai_recommended_action: pgText("ai_recommended_action", {
enum: ["none", "monitor", "warn", "review", "delete", "escalate"],
}),
ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }),
ai_error: pgText("ai_error"),
});
const attachments = pgTable("attachments", {
id: pgText("id").primaryKey(),
message_id: pgText("message_id").notNull(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
filename: pgText("filename").notNull(),
size: pgInteger("size").notNull(),
type: pgText("type").notNull(),
discord_url: pgText("discord_url").notNull(),
uploaded_url: pgText("uploaded_url"),
upload_status: pgText("upload_status", {
enum: ["pending", "uploaded", "failed"],
})
.notNull()
.default("pending"),
upload_error: pgText("upload_error"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
uploaded_at: pgBigint("uploaded_at", { mode: "number" }),
});
export interface AttachmentResult {
id: string;
message_id: string;
@@ -100,35 +30,6 @@ export interface AttachmentResult {
uploaded_at: number | null;
}
function mapMessageRow(row: Record<string, unknown>) {
return {
id: String(row.id ?? ""),
guild_id: String(row.guild_id ?? ""),
channel_id: String(row.channel_id ?? ""),
thread_id: (row.thread_id as string | null) ?? null,
user_id: String(row.user_id ?? ""),
username: String(row.username ?? ""),
avatar_url: (row.avatar_url as string | null) ?? null,
content: String(row.content ?? ""),
edited_content: (row.edited_content as string | null) ?? null,
created_at: Number(row.created_at ?? 0),
edited_at: (row.edited_at as number | null) ?? null,
deleted_at: (row.deleted_at as number | null) ?? null,
type: String(row.type ?? "text"),
metadata: (row.metadata as string | null) ?? null,
ai_status: (row.ai_status as string | null) ?? null,
ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null,
ai_moderation_score: (row.ai_moderation_score as number | null) ?? null,
ai_analysis: (row.ai_analysis as string | null) ?? null,
ai_categories: (row.ai_categories as string | null) ?? null,
ai_severity: (row.ai_severity as string | null) ?? null,
ai_confidence: (row.ai_confidence as number | null) ?? null,
ai_recommended_action: (row.ai_recommended_action as string | null) ?? null,
ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null,
ai_error: (row.ai_error as string | null) ?? null,
};
}
export class MessagesRepository {
async findMany(
query: MessageQuery,
@@ -138,27 +39,27 @@ export class MessagesRepository {
const conditions: SQL[] = [];
if (query.guildId) {
conditions.push(eq(messages.guild_id, query.guildId));
conditions.push(eq(pgMessagesTable.guild_id, query.guildId));
}
if (query.channelId) {
conditions.push(eq(messages.channel_id, query.channelId));
conditions.push(eq(pgMessagesTable.channel_id, query.channelId));
}
if (query.userId) {
conditions.push(eq(messages.user_id, query.userId));
conditions.push(eq(pgMessagesTable.user_id, query.userId));
}
if (query.status) {
conditions.push(eq(messages.ai_status, query.status));
conditions.push(eq(pgMessagesTable.ai_status, query.status));
}
if (query.cursor) {
conditions.push(lt(messages.created_at, Number(query.cursor)));
conditions.push(lt(pgMessagesTable.created_at, Number(query.cursor)));
}
const where = conditions.length > 0 ? and(...conditions) : undefined;
const rows = await db
.select()
.from(messages)
.from(pgMessagesTable)
.where(where)
.orderBy(desc(messages.created_at))
.orderBy(desc(pgMessagesTable.created_at))
.limit(limit + 1);
const data = rows
@@ -175,8 +76,8 @@ export class MessagesRepository {
const db = getDatabase();
const [row] = await db
.select()
.from(messages)
.where(eq(messages.id, id))
.from(pgMessagesTable)
.where(eq(pgMessagesTable.id, id))
.limit(1);
if (!row) return null;
@@ -189,17 +90,17 @@ export class MessagesRepository {
): Promise<PageResult<ReturnType<typeof mapMessageRow>>> {
const db = getDatabase();
const limit = query.limit ?? 50;
const conditions: SQL[] = [eq(messages.channel_id, channelId)];
const conditions: SQL[] = [eq(pgMessagesTable.channel_id, channelId)];
if (query.cursor) {
conditions.push(lt(messages.created_at, Number(query.cursor)));
conditions.push(lt(pgMessagesTable.created_at, Number(query.cursor)));
}
const rows = await db
.select()
.from(messages)
.from(pgMessagesTable)
.where(and(...conditions))
.orderBy(desc(messages.created_at))
.orderBy(desc(pgMessagesTable.created_at))
.limit(limit + 1);
const data = rows
@@ -216,7 +117,7 @@ export class MessagesRepository {
const id = crypto.randomUUID();
const [row] = await db
.insert(messages)
.insert(pgMessagesTable)
.values({
id,
guild_id: data.guildId,
@@ -242,7 +143,7 @@ export class MessagesRepository {
async update(id: string, data: MessageUpdate) {
const db = getDatabase();
const setData: Partial<typeof messages.$inferInsert> = {};
const setData: Partial<typeof pgMessagesTable.$inferInsert> = {};
if (data.editedContent !== undefined) {
setData.edited_content = data.editedContent;
@@ -266,9 +167,9 @@ export class MessagesRepository {
if (Object.keys(setData).length === 0) return this.findById(id);
const [row] = await db
.update(messages)
.update(pgMessagesTable)
.set(setData)
.where(eq(messages.id, id))
.where(eq(pgMessagesTable.id, id))
.returning();
if (!row) return null;
@@ -288,20 +189,20 @@ export class MessagesRepository {
messageIds?: string[];
}): Promise<number> {
const db = getDatabase();
const conditions: SQL[] = [eq(messages.ai_status, "error")];
const conditions: SQL[] = [eq(pgMessagesTable.ai_status, "error")];
if (opts.messageIds && opts.messageIds.length > 0) {
conditions.push(inArray(messages.id, opts.messageIds));
conditions.push(inArray(pgMessagesTable.id, opts.messageIds));
}
if (opts.guildId) {
conditions.push(eq(messages.guild_id, opts.guildId));
conditions.push(eq(pgMessagesTable.guild_id, opts.guildId));
}
if (opts.channelId) {
conditions.push(eq(messages.channel_id, opts.channelId));
conditions.push(eq(pgMessagesTable.channel_id, opts.channelId));
}
const result = await db
.update(messages)
.update(pgMessagesTable)
.set({ ai_status: "pending" })
.where(and(...conditions));
@@ -317,9 +218,14 @@ export class MessagesRepository {
async markForReanalysis(id: string): Promise<void> {
const db = getDatabase();
await db
.update(messages)
.update(pgMessagesTable)
.set({ ai_status: "pending" })
.where(and(eq(messages.id, id), ne(messages.ai_status, "pending")));
.where(
and(
eq(pgMessagesTable.id, id),
ne(pgMessagesTable.ai_status, "pending"),
),
);
}
/**
@@ -332,32 +238,32 @@ export class MessagesRepository {
): Promise<Record<string, unknown>[]> {
const db = getDatabase();
const conditions: SQL[] = [
inArray(messages.ai_status, ["warn", "flagged"]),
inArray(pgMessagesTable.ai_status, ["warn", "flagged"]),
];
if (channelId) {
conditions.push(eq(messages.channel_id, channelId));
conditions.push(eq(pgMessagesTable.channel_id, channelId));
}
const rows = await db
.select({
id: messages.id,
guild_id: messages.guild_id,
channel_id: messages.channel_id,
user_id: messages.user_id,
username: messages.username,
avatar_url: messages.avatar_url,
content: messages.content,
type: messages.type,
created_at: messages.created_at,
ai_status: messages.ai_status,
ai_severity: messages.ai_severity,
ai_confidence: messages.ai_confidence,
ai_analysis: messages.ai_analysis,
id: pgMessagesTable.id,
guild_id: pgMessagesTable.guild_id,
channel_id: pgMessagesTable.channel_id,
user_id: pgMessagesTable.user_id,
username: pgMessagesTable.username,
avatar_url: pgMessagesTable.avatar_url,
content: pgMessagesTable.content,
type: pgMessagesTable.type,
created_at: pgMessagesTable.created_at,
ai_status: pgMessagesTable.ai_status,
ai_severity: pgMessagesTable.ai_severity,
ai_confidence: pgMessagesTable.ai_confidence,
ai_analysis: pgMessagesTable.ai_analysis,
})
.from(messages)
.from(pgMessagesTable)
.where(and(...conditions))
.orderBy(desc(messages.created_at))
.orderBy(desc(pgMessagesTable.created_at))
.limit(limit);
return rows as unknown as Record<string, unknown>[];
@@ -365,7 +271,9 @@ export class MessagesRepository {
async delete(id: string): Promise<boolean> {
const db = getDatabase();
const result = await db.delete(messages).where(eq(messages.id, id));
const result = await db
.delete(pgMessagesTable)
.where(eq(pgMessagesTable.id, id));
return (result.rowCount ?? 0) > 0;
}
@@ -376,17 +284,17 @@ export class MessagesRepository {
): Promise<PageResult<AttachmentResult>> {
const db = getDatabase();
const limit = query.limit ?? 50;
const conditions: SQL[] = [eq(attachments.channel_id, channelId)];
const conditions: SQL[] = [eq(pgAttachmentsTable.channel_id, channelId)];
if (query.cursor) {
conditions.push(lt(attachments.created_at, Number(query.cursor)));
conditions.push(lt(pgAttachmentsTable.created_at, Number(query.cursor)));
}
const rows = await db
.select()
.from(attachments)
.from(pgAttachmentsTable)
.where(and(...conditions))
.orderBy(desc(attachments.created_at))
.orderBy(desc(pgAttachmentsTable.created_at))
.limit(limit + 1);
const data = rows.map((r) => ({
@@ -1,21 +1,36 @@
import type { Router } from "express";
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response, Router } from "express";
import express from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import { getGuilds, getTextChannels } from "./voice.service.js";
const logger = createChildLogger("guilds.routes");
export function createGuildsRouter(): Router {
const router = express.Router();
// GET /api/guilds
router.get("/", async (_req, res) => {
const guilds = await getGuilds();
res.json(guilds);
});
router.get(
"/",
asyncHandler(async (_req: Request, res: Response) => {
logger.debug("Fetching guilds");
const guilds = await getGuilds();
res.json(guilds);
}),
);
// GET /api/guilds/:guildId/channels
router.get("/:guildId/channels", async (req, res) => {
const channels = await getTextChannels(req.params.guildId);
res.json(channels);
});
router.get(
"/:guildId/channels",
asyncHandler(async (req: Request, res: Response) => {
const guildId = Array.isArray(req.params.guildId)
? req.params.guildId[0]
: req.params.guildId;
logger.debug({ guildId }, "Fetching text channels");
const channels = await getTextChannels(guildId);
res.json(channels);
}),
);
return router;
}
@@ -1,4 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response } from "express";
import { asyncHandler } from "../../shared/middlewares/index.js";
import { publishCommandNoReply } from "../../shared/redis/index.js";
import {
connectVoice,
@@ -7,10 +9,14 @@ import {
getVoiceStatus,
} from "./voice.service.js";
export async function handleGetVoiceStatus(_req: Request, res: Response) {
const status = await getVoiceStatus();
res.json(status);
}
const logger = createChildLogger("voice.controller");
export const handleGetVoiceStatus = asyncHandler(
async (_req: Request, res: Response) => {
const status = await getVoiceStatus();
res.json(status);
},
);
/** Safely extract a string value that may be a single string or string array. */
function asString(val: unknown): string {
@@ -18,47 +24,52 @@ function asString(val: unknown): string {
return String(val ?? "");
}
export async function handleConnectVoice(req: Request, res: Response) {
const guildId = asString(req.body.guildId);
const channelId = asString(req.body.channelId);
if (!guildId || !channelId) {
return res.status(400).json({
error: "VALIDATION_ERROR",
message: "guildId and channelId are required",
});
}
const status = await connectVoice(guildId, channelId);
res.json(status);
}
export const handleConnectVoice = asyncHandler(
async (req: Request, res: Response) => {
const guildId = asString(req.body.guildId);
const channelId = asString(req.body.channelId);
if (!guildId || !channelId) {
return res.status(400).json({
error: "VALIDATION_ERROR",
message: "guildId and channelId are required",
});
}
logger.debug({ guildId, channelId }, "Connecting to voice channel");
const status = await connectVoice(guildId, channelId);
res.json(status);
},
);
export async function handleDisconnectVoice(_req: Request, res: Response) {
const status = await disconnectVoice();
res.json(status);
}
export const handleDisconnectVoice = asyncHandler(
async (_req: Request, res: Response) => {
logger.debug("Disconnecting from voice");
const status = await disconnectVoice();
res.json(status);
},
);
export async function handleGetVoiceChannels(req: Request, res: Response) {
const guildId = asString(req.params.guildId);
const channels = await getVoiceChannels(guildId);
res.json(channels);
}
export const handleGetVoiceChannels = asyncHandler(
async (req: Request, res: Response) => {
const guildId = asString(req.params.guildId);
logger.debug({ guildId }, "Fetching voice channels");
const channels = await getVoiceChannels(guildId);
res.json(channels);
},
);
export async function handleVoiceCommand(req: Request, res: Response) {
const command = asString(req.body.command);
export const handleVoiceCommand = asyncHandler(
async (req: Request, res: Response) => {
const command = asString(req.body.command);
if (!command) {
return res.status(400).json({
error: "VALIDATION_ERROR",
message: "command is required",
});
}
if (!command) {
return res.status(400).json({
error: "VALIDATION_ERROR",
message: "command is required",
});
}
try {
logger.debug({ command }, "Publishing voice command");
await publishCommandNoReply(command);
res.json({ success: true, command });
} catch (err) {
res.status(500).json({
error: "COMMAND_FAILED",
message: err instanceof Error ? err.message : "Unknown error",
});
}
}
},
);
@@ -6,7 +6,10 @@ import {
COMMAND_VOICE_DISCONNECT,
VOICE_STATUS_KEY,
} from "@bete/shared";
import { createChildLogger } from "@bete/shared/logger";
import {
createChildLogger,
tryCommandThenFallback,
} from "../../shared/commandHelper.js";
import { getPool } from "../../shared/database/index.js";
import { publishCommand, readRedisStatus } from "../../shared/redis/index.js";
@@ -31,29 +34,40 @@ export interface VoiceStatus {
activeChannelName: string | null;
}
export const DEFAULT_VOICE_STATUS: VoiceStatus = {
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
};
function readVoiceStatusFallback(): Promise<VoiceStatus> {
return readRedisStatus(VOICE_STATUS_KEY).then(
(cached) => (cached as unknown as VoiceStatus) ?? DEFAULT_VOICE_STATUS,
);
}
/**
* Get guilds — query from discord-gateway via Redis command for real names.
* Falls back to database (distinct guild_id from messages) if gateway unreachable.
*/
export async function getGuilds(): Promise<Guild[]> {
logger.info("getGuilds called");
const reply = await publishCommand<Guild[]>(COMMAND_GUILDS_LIST, {});
if (reply?.success && reply.data && reply.data.length > 0) return reply.data;
// Fallback: Postgres with synthetic names
logger.warn(
"discord-gateway unreachable, falling back to Postgres for guilds",
return tryCommandThenFallback(
() => publishCommand<Guild[]>(COMMAND_GUILDS_LIST, {}),
async () => {
const pool = getPool();
const { rows } = await pool.query(
`SELECT DISTINCT guild_id FROM messages ORDER BY guild_id`,
);
return rows.map((row: Record<string, unknown>) => ({
id: String(row.guild_id ?? ""),
name: `Guild ${String(row.guild_id).slice(0, 8)}`,
icon: null,
}));
},
"getGuilds",
);
const pool = getPool();
const { rows } = await pool.query(
`SELECT DISTINCT guild_id FROM messages ORDER BY guild_id`,
);
return rows.map((row: Record<string, unknown>) => ({
id: String(row.guild_id ?? ""),
name: `Guild ${String(row.guild_id).slice(0, 8)}`,
icon: null,
}));
}
/**
@@ -62,27 +76,22 @@ export async function getGuilds(): Promise<Guild[]> {
*/
export async function getTextChannels(guildId: string): Promise<Channel[]> {
logger.info({ guildId }, "getTextChannels called");
const reply = await publishCommand<Channel[]>(COMMAND_GUILDS_TEXT_CHANNELS, {
guildId,
});
if (reply?.success && reply.data && reply.data.length > 0) return reply.data;
// Fallback: Postgres with synthetic names
logger.warn(
{ guildId },
"discord-gateway unreachable, falling back to Postgres for text channels",
return tryCommandThenFallback(
() => publishCommand<Channel[]>(COMMAND_GUILDS_TEXT_CHANNELS, { guildId }),
async () => {
const pool = getPool();
const { rows } = await pool.query(
`SELECT DISTINCT channel_id FROM messages WHERE guild_id = $1 ORDER BY channel_id`,
[guildId],
);
return rows.map((row: Record<string, unknown>) => ({
id: String(row.channel_id ?? ""),
name: `Channel ${String(row.channel_id).slice(0, 8)}`,
type: "text" as const,
}));
},
"getTextChannels",
);
const pool = getPool();
const { rows } = await pool.query(
`SELECT DISTINCT channel_id FROM messages WHERE guild_id = $1 ORDER BY channel_id`,
[guildId],
);
return rows.map((row: Record<string, unknown>) => ({
id: String(row.channel_id ?? ""),
name: `Channel ${String(row.channel_id).slice(0, 8)}`,
type: "text" as const,
}));
}
/**
@@ -102,13 +111,7 @@ export async function getVoiceChannels(guildId: string): Promise<Channel[]> {
export async function getVoiceStatus(): Promise<VoiceStatus> {
logger.debug("getVoiceStatus called");
const cached = await readRedisStatus(VOICE_STATUS_KEY);
if (cached) return cached as unknown as VoiceStatus;
return {
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
};
return (cached as unknown as VoiceStatus) ?? DEFAULT_VOICE_STATUS;
}
/**
@@ -119,21 +122,14 @@ export async function connectVoice(
channelId: string,
): Promise<VoiceStatus> {
logger.info({ guildId, channelId }, "connectVoice called");
const reply = await publishCommand<VoiceStatus>(COMMAND_VOICE_CONNECT, {
guildId,
channelId,
});
if (reply?.success && reply.data) return reply.data;
// Fallback: read from Redis status key
const cached = await readRedisStatus(VOICE_STATUS_KEY);
return (
(cached as unknown as VoiceStatus) ?? {
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
}
return tryCommandThenFallback(
() =>
publishCommand<VoiceStatus>(COMMAND_VOICE_CONNECT, {
guildId,
channelId,
}),
() => readVoiceStatusFallback(),
"connectVoice",
);
}
@@ -142,16 +138,9 @@ export async function connectVoice(
*/
export async function disconnectVoice(): Promise<VoiceStatus> {
logger.info("disconnectVoice called");
const reply = await publishCommand<VoiceStatus>(COMMAND_VOICE_DISCONNECT, {});
if (reply?.success && reply.data) return reply.data;
const cached = await readRedisStatus(VOICE_STATUS_KEY);
return (
(cached as unknown as VoiceStatus) ?? {
connected: false,
activeGuildId: null,
activeChannelId: null,
activeChannelName: null,
}
return tryCommandThenFallback(
() => publishCommand<VoiceStatus>(COMMAND_VOICE_DISCONNECT, {}),
() => readVoiceStatusFallback(),
"disconnectVoice",
);
}
@@ -0,0 +1,26 @@
import type { CommandReply } from "@bete/shared";
import { createChildLogger } from "@bete/shared/logger";
import { publishCommand, readRedisStatus } from "./redis/index.js";
export { createChildLogger };
/**
* Attempt a Redis command first; if it fails or times out, fall back.
*
* @param commandFn - Function that issues the publishCommand and returns the reply.
* @param fallbackFn - Async fallback, typically reads from Redis status key.
* @param commandLabel - Label used for logging (e.g. "voice:connect").
*/
export async function tryCommandThenFallback<T>(
commandFn: () => Promise<CommandReply<T> | null>,
fallbackFn: () => Promise<T>,
commandLabel: string,
): Promise<T> {
const logger = createChildLogger(`command-helper:${commandLabel}`);
const reply = await commandFn();
if (reply?.success && reply.data !== undefined && reply.data !== null) {
return reply.data;
}
logger.warn("discord-gateway unreachable, falling back");
return fallbackFn();
}
@@ -0,0 +1,57 @@
// Shared message row mapper for backend repository modules
export interface MappedMessage {
id: string;
guild_id: string;
channel_id: string;
thread_id: string | null;
user_id: string;
username: string;
avatar_url: string | null;
content: string;
edited_content: string | null;
created_at: number;
edited_at: number | null;
deleted_at: number | null;
type: string;
metadata: string | null;
ai_status: string | null;
ai_moderation_flags: string | null;
ai_moderation_score: number | null;
ai_analysis: string | null;
ai_categories: string | null;
ai_severity: string | null;
ai_confidence: number | null;
ai_recommended_action: string | null;
ai_analyzed_at: number | null;
ai_error: string | null;
}
export function mapMessageRow(row: Record<string, unknown>): MappedMessage {
return {
id: String(row.id ?? ""),
guild_id: String(row.guild_id ?? ""),
channel_id: String(row.channel_id ?? ""),
thread_id: (row.thread_id as string | null) ?? null,
user_id: String(row.user_id ?? ""),
username: String(row.username ?? ""),
avatar_url: (row.avatar_url as string | null) ?? null,
content: String(row.content ?? ""),
edited_content: (row.edited_content as string | null) ?? null,
created_at: Number(row.created_at ?? 0),
edited_at: (row.edited_at as number | null) ?? null,
deleted_at: (row.deleted_at as number | null) ?? null,
type: String(row.type ?? "text"),
metadata: (row.metadata as string | null) ?? null,
ai_status: (row.ai_status as string | null) ?? null,
ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null,
ai_moderation_score: (row.ai_moderation_score as number | null) ?? null,
ai_analysis: (row.ai_analysis as string | null) ?? null,
ai_categories: (row.ai_categories as string | null) ?? null,
ai_severity: (row.ai_severity as string | null) ?? null,
ai_confidence: (row.ai_confidence as number | null) ?? null,
ai_recommended_action: (row.ai_recommended_action as string | null) ?? null,
ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null,
ai_error: (row.ai_error as string | null) ?? null,
};
}
+76 -32
View File
@@ -9,6 +9,10 @@
* broadcastMessageCreated(messageData);
*/
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("broadcast");
type BroadcastFn = (data: unknown) => void;
type BroadcastRawFn = (type: string, data: unknown) => void;
type BroadcastBinaryFn = (data: Buffer) => void;
@@ -30,63 +34,103 @@ export interface BroadcastFunctions {
binary: BroadcastBinaryFn;
}
const noop: BroadcastFn = () => {};
const noopRaw: BroadcastRawFn = () => {};
const noopBinary: BroadcastBinaryFn = () => {};
let _fns: BroadcastFunctions | null = null;
let _enabled = true;
/** Enable or disable broadcast logging (disabled by default to reduce noise). */
export function setBroadcastLogging(enabled: boolean): void {
_enabled = enabled;
}
/**
* Inject broadcast functions from the WebSocket server initializer.
* Must be called once during server startup before any broadcast is used.
*/
export function setBroadcastFunctions(fns: BroadcastFunctions): void {
_fns = fns;
logger.info("Broadcast functions initialized");
}
/** Clear injected functions (used during cleanup). */
export function clearBroadcastFunctions(): void {
_fns = null;
logger.info("Broadcast functions cleared");
}
export const broadcastMessageCreated: BroadcastFn = (data) =>
(_fns?.messageCreated ?? noop)(data);
function logBroadcast(name: string, data: unknown): void {
if (!_enabled) return;
// Avoid logging binary or PCM data due to volume
if (name === "voice_pcm_data" || name === "binary") return;
logger.debug({ event: name }, "Broadcasting event");
}
export const broadcastMessageUpdated: BroadcastFn = (data) =>
(_fns?.messageUpdated ?? noop)(data);
export const broadcastMessageCreated: BroadcastFn = (data) => {
logBroadcast("message_created", data);
_fns?.messageCreated?.(data);
};
export const broadcastMessageDeleted: BroadcastFn = (data) =>
(_fns?.messageDeleted ?? noop)(data);
export const broadcastMessageUpdated: BroadcastFn = (data) => {
logBroadcast("message_updated", data);
_fns?.messageUpdated?.(data);
};
export const broadcastAttachmentCreated: BroadcastFn = (data) =>
(_fns?.attachmentCreated ?? noop)(data);
export const broadcastMessageDeleted: BroadcastFn = (data) => {
logBroadcast("message_deleted", data);
_fns?.messageDeleted?.(data);
};
export const broadcastAttachmentUploaded: BroadcastFn = (data) =>
(_fns?.attachmentUploaded ?? noop)(data);
export const broadcastAttachmentCreated: BroadcastFn = (data) => {
logBroadcast("attachment_created", data);
_fns?.attachmentCreated?.(data);
};
export const broadcastMessageAnalyzed: BroadcastFn = (data) =>
(_fns?.messageAnalyzed ?? noop)(data);
export const broadcastAttachmentUploaded: BroadcastFn = (data) => {
logBroadcast("attachment_uploaded", data);
_fns?.attachmentUploaded?.(data);
};
export const broadcastVoiceRecordingStarted: BroadcastFn = (data) =>
(_fns?.voiceRecordingStarted ?? noop)(data);
export const broadcastMessageAnalyzed: BroadcastFn = (data) => {
logBroadcast("message_analyzed", data);
_fns?.messageAnalyzed?.(data);
};
export const broadcastVoiceRecordingStopped: BroadcastFn = (data) =>
(_fns?.voiceRecordingStopped ?? noop)(data);
export const broadcastVoiceRecordingStarted: BroadcastFn = (data) => {
logBroadcast("voice_recording_started", data);
_fns?.voiceRecordingStarted?.(data);
};
export const broadcastVoiceRecordingUploaded: BroadcastFn = (data) =>
(_fns?.voiceRecordingUploaded ?? noop)(data);
export const broadcastVoiceRecordingStopped: BroadcastFn = (data) => {
logBroadcast("voice_recording_stopped", data);
_fns?.voiceRecordingStopped?.(data);
};
export const broadcastVoicePcmData: BroadcastFn = (data) =>
(_fns?.voicePcmData ?? noop)(data);
export const broadcastVoiceRecordingUploaded: BroadcastFn = (data) => {
logBroadcast("voice_recording_uploaded", data);
_fns?.voiceRecordingUploaded?.(data);
};
export const broadcastVoiceActiveUser: BroadcastFn = (data) =>
(_fns?.voiceActiveUser ?? noop)(data);
export const broadcastVoicePcmData: BroadcastFn = (data) => {
// PCM data is high-volume; logging is skipped unconditionally
_fns?.voicePcmData?.(data);
};
export const broadcastAnalysisQueueStatus: BroadcastFn = (data) =>
(_fns?.analysisQueueStatus ?? noop)(data);
export const broadcastVoiceActiveUser: BroadcastFn = (data) => {
logBroadcast("voice_active_user", data);
_fns?.voiceActiveUser?.(data);
};
export const broadcastRaw: BroadcastRawFn = (type, data) =>
(_fns?.raw ?? noopRaw)(type, data);
export const broadcastAnalysisQueueStatus: BroadcastFn = (data) => {
logBroadcast("analysis_queue_status", data);
_fns?.analysisQueueStatus?.(data);
};
export const broadcastBinary: BroadcastBinaryFn = (data) =>
(_fns?.binary ?? noopBinary)(data);
export const broadcastRaw: BroadcastRawFn = (type, data) => {
logBroadcast(type, data);
_fns?.raw?.(type, data);
};
export const broadcastBinary: BroadcastBinaryFn = (data) => {
// Binary data is high-volume; logging is skipped unconditionally
_fns?.binary?.(data);
};
@@ -1,3 +1,4 @@
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/config.js";
import { initializeDatabase } from "../../shared/database/drizzle.js";
import {
@@ -15,6 +16,8 @@ import {
runSimpleTextFallback,
} from "./llmModerationClient.js";
const logger = createChildLogger("aiAnalysisWorker");
let dbInitialized = false;
let dbInitPromise: Promise<any> | null = null;
@@ -70,13 +73,9 @@ export default async function workerRouter(
if (!config.AI_LLM_API_KEY) {
const errorMsg =
"AI_LLM_API_KEY is missing from environment. Worker cannot process moderation requests without credentials.";
console.error(
JSON.stringify({
level: "ERROR",
context: "aiAnalysisWorker",
error: errorMsg,
timestamp: new Date().toISOString(),
}),
logger.error(
{ error: errorMsg },
"AI_LLM_API_KEY is missing from environment",
);
if (job.type === "batch") {
@@ -113,15 +112,9 @@ export default async function workerRouter(
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
const errorStack = error instanceof Error ? error.stack : undefined;
console.error(
JSON.stringify({
level: "ERROR",
context: "aiAnalysisWorker",
type: job.type,
error: errorMessage,
stack: errorStack,
timestamp: new Date().toISOString(),
}),
logger.error(
{ type: job.type, error: errorMessage, stack: errorStack },
"Worker job failed",
);
if (job.type === "batch") {
return {
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,399 @@
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/config.js";
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
import { updateMessagesAIAnalysisBulk } from "../message-capture/messageStore.js";
import type { MessageRecord } from "../message-capture/types.js";
import {
broadcastAnalysisCompleted,
conversationErrorCooldown,
conversationProcessing,
LAST_ERROR,
recordConversationBatchFailure,
resetConversationBatchFailures,
scheduleAutoDelete,
workerPool,
} from "./circuitBreaker.js";
import { estimateTokens } from "./conversationContext.js";
import { enqueueIndividualFallbacks } from "./individualFallbackProcessor.js";
const logger = createChildLogger("batch-processor");
export interface AnalysisWorkerResponse {
ok: boolean;
conversationKey: string;
rows: MessageRecord[];
error?: string;
}
// ---------------------------------------------------------------------------
// Observability
// ---------------------------------------------------------------------------
export let activeRequests = 0;
// ---------------------------------------------------------------------------
// Exported helpers
// ---------------------------------------------------------------------------
/**
* Picks a batch of messages within a token budget.
* `tokensPerMessage` accounts for JSON structure overhead around each entry.
* Uses a rough character-based token estimate (avoids async formatMessageForPrompt
* since this function runs in a synchronous promise chain).
*/
export function pickBatchWithinBudget(
messages: MessageRecord[],
maxTokens: number,
tokensPerMessage: number,
): MessageRecord[] {
const batch: MessageRecord[] = [];
let usedTokens = 0;
for (const msg of messages) {
const content = msg.edited_content ?? msg.content;
// Accurate token count via tiktoken (+ overhead for JSON structure)
const msgTokens = estimateTokens(content) + tokensPerMessage;
if (usedTokens + msgTokens <= maxTokens) {
batch.push(msg);
usedTokens += msgTokens;
}
}
return batch;
}
// ---------------------------------------------------------------------------
// Age-restricted message helpers
// ---------------------------------------------------------------------------
export function isAgeRestrictedMessage(message: MessageRecord): boolean {
return isAgeRestrictedMetadata(message.metadata);
}
export function buildAgeRestrictedSkipResult(): {
status: "clean";
flags: string | null;
score: number;
analysis: string;
categories: string[];
severity: "none";
confidence: number;
recommendedAction: "none";
analyzedAt: number;
error: null;
} {
return {
status: "clean",
flags: JSON.stringify(["age_restricted"]),
score: 0,
analysis: "Skipped moderation for age-restricted content.",
categories: ["age_restricted"],
severity: "none",
confidence: 1,
recommendedAction: "none",
analyzedAt: Date.now(),
error: null,
};
}
export async function skipAgeRestrictedMessages(
messages: MessageRecord[],
): Promise<MessageRecord[]> {
const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage);
if (ageRestrictedMessages.length === 0) {
return messages;
}
const skippedRows = await updateMessagesAIAnalysisBulk(
ageRestrictedMessages.map((message) => ({
messageId: message.id,
result: buildAgeRestrictedSkipResult(),
})),
);
for (const row of skippedRows) {
broadcastAnalysisCompleted(row);
}
const skippedIds = new Set(
ageRestrictedMessages.map((message) => message.id),
);
return messages.filter((message) => !skippedIds.has(message.id));
}
// ---------------------------------------------------------------------------
// Batch pipeline
// ---------------------------------------------------------------------------
async function postBatchReputationUpdate(rows: MessageRecord[]): Promise<void> {
for (const row of rows) {
if (row.ai_status === "clean") {
import("./userReputationStore.js")
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
.catch((e) =>
logger.error({ error: e }, "Failed to record clean message streak"),
);
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
import("./userReputationStore.js")
.then((store) =>
store.recordInfraction(
row.user_id,
row.guild_id,
row.ai_severity as "low" | "medium" | "high" | "critical",
),
)
.catch((e) =>
logger.error({ error: e }, "Failed to record infraction penalty"),
);
}
}
}
export async function processBatch(
conversationKey: string,
messages: MessageRecord[],
processingStartedAt: number,
): Promise<void> {
if (messages.length === 0) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
return;
}
const cooldownUntil = conversationErrorCooldown.get(conversationKey) ?? 0;
if (Date.now() < cooldownUntil) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
return;
}
activeRequests++;
let shouldScheduleNext = false;
try {
const result = (await workerPool.run({
type: "batch",
conversationKey,
messages,
})) as AnalysisWorkerResponse;
// Do not broadcast or auto-delete if it's an API failure that will be reverted.
for (const row of result.rows) {
let isApiFailure = false;
if (row.ai_status === "error") {
try {
const flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
isApiFailure = flags.includes("analysis_api_failed");
} catch {}
}
if (!isApiFailure) {
broadcastAnalysisCompleted(row);
scheduleAutoDelete(row);
}
}
// Post-batch reputation updates (fire-and-forget)
postBatchReputationUpdate(
result.rows.filter((r) => {
if (r.ai_status === "error") {
try {
const flags = JSON.parse(r.ai_moderation_flags ?? "[]") as string[];
return !flags.includes("analysis_api_failed");
} catch {
return false;
}
}
return true;
}),
);
if (!result.ok) {
recordConversationBatchFailure(conversationKey);
// Batch failed entirely -- fall back all messages to individual queue
logger.warn(
{
conversationKey,
messageCount: messages.length,
error: result.error,
},
"Batch failed entirely -- routing all messages to individual fallback queue",
);
enqueueIndividualFallbacks(messages);
LAST_ERROR.value = result.error ?? "Analysis worker failed";
conversationErrorCooldown.set(
conversationKey,
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
);
logger.error(
{
conversationKey,
error: LAST_ERROR.value,
messageCount: messages.length,
messageIds: messages.map((m) => m.id),
cooldownUntil: new Date(
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
).toISOString(),
timestamp: new Date().toISOString(),
},
"Batch analysis failed, will retry after cooldown",
);
return;
}
// Batch succeeded -- check for messages the LLM silently dropped or failed
const incompleteMessages: MessageRecord[] = [];
const parseFailedMessages: MessageRecord[] = [];
const apiFailedMessages: MessageRecord[] = [];
for (const msg of messages) {
const row = result.rows.find((r) => r.id === msg.id);
if (!row) {
incompleteMessages.push(msg);
continue;
}
if (row.ai_status === "error") {
let flags: string[] = [];
try {
flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
} catch {}
if (flags.includes("analysis_incomplete")) {
incompleteMessages.push(msg);
} else if (flags.includes("analysis_parse_failed")) {
parseFailedMessages.push(msg);
} else if (flags.includes("analysis_api_failed")) {
apiFailedMessages.push(msg);
}
}
}
const messagesForIndividualQueue = [
...incompleteMessages,
...parseFailedMessages,
];
if (messagesForIndividualQueue.length > 0) {
logger.warn(
{
conversationKey,
count: messagesForIndividualQueue.length,
ids: messagesForIndividualQueue.map((m) => m.id),
totalBatchSize: messages.length,
},
"Batch returned incomplete or unparseable results -- fanning out to individual fallback queue",
);
enqueueIndividualFallbacks(messagesForIndividualQueue);
}
if (apiFailedMessages.length > 0) {
logger.warn(
{
conversationKey,
count: apiFailedMessages.length,
ids: apiFailedMessages.map((m) => m.id),
},
"Batch returned API failures -- reverting to pending to put back in queue",
);
// Revert to pending so they are picked up again
const revertedRows = await updateMessagesAIAnalysisBulk(
apiFailedMessages.map((msg) => ({
messageId: msg.id,
result: {
status: "pending",
flags: null,
score: null,
analysis: null,
categories: null,
severity: null,
confidence: null,
recommendedAction: null,
analyzedAt: null,
error: null,
},
})),
).catch((err) => {
logger.error(
{ error: String(err) },
"Failed to revert API failures to pending",
);
return [];
});
for (const row of revertedRows) {
broadcastAnalysisCompleted(row);
}
// Trigger conversation cooldown
recordConversationBatchFailure(conversationKey);
const existingCooldown =
conversationErrorCooldown.get(conversationKey) ?? 0;
const newCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS;
if (newCooldown > existingCooldown) {
conversationErrorCooldown.set(conversationKey, newCooldown);
}
// Release the processing lock immediately so the cooldown timer controls retry
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
// Do NOT schedule next -- let the cooldown gate it
shouldScheduleNext = false;
}
if (apiFailedMessages.length === 0) {
resetConversationBatchFailures(conversationKey);
conversationErrorCooldown.delete(conversationKey);
}
shouldScheduleNext = true;
} catch (error) {
recordConversationBatchFailure(conversationKey);
logger.warn(
{ conversationKey, messageCount: messages.length },
"Batch threw exception -- routing all messages to individual fallback queue",
);
enqueueIndividualFallbacks(messages);
LAST_ERROR.value = error instanceof Error ? error.message : String(error);
const errorStack = error instanceof Error ? error.stack : undefined;
const existingCatchCooldown =
conversationErrorCooldown.get(conversationKey) ?? 0;
const newCatchCooldown = Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS;
if (newCatchCooldown > existingCatchCooldown) {
conversationErrorCooldown.set(conversationKey, newCatchCooldown);
}
logger.error(
{
conversationKey,
error: LAST_ERROR.value,
stack: errorStack,
messageCount: messages.length,
messageIds: messages.map((m) => m.id),
cooldownUntil: new Date(
Date.now() + config.AI_ANALYSIS_ERROR_COOLDOWN_MS,
).toISOString(),
timestamp: new Date().toISOString(),
},
"Analysis worker failed, will retry after cooldown",
);
} finally {
activeRequests--;
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
if (shouldScheduleNext) {
setImmediate(() => {
// Dynamic import to avoid circular dependency at module scope
import("./batchScheduler.js").then((m) =>
m.scheduleConversationAnalysis(conversationKey),
);
});
}
}
}
@@ -0,0 +1,140 @@
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/config.js";
import { getPendingMessagesByConversation } from "../message-capture/messageStore.js";
import type { MessageRecord } from "../message-capture/types.js";
import {
pickBatchWithinBudget,
processBatch,
skipAgeRestrictedMessages,
} from "./batchProcessor.js";
import {
conversationConsecutiveErrors,
conversationDebounceTimers,
conversationErrorCooldown,
conversationProcessing,
isConversationProcessingLocked,
MAX_CONSECUTIVE_ERRORS,
} from "./circuitBreaker.js";
const logger = createChildLogger("batch-scheduler");
// ---------------------------------------------------------------------------
// Scheduling
// ---------------------------------------------------------------------------
/**
* Schedules a debounced analysis run for a conversation.
*
* FIX #3: The async work inside setTimeout is now wrapped in an explicit
* .catch() so DB errors don't produce unhandled promise rejections.
* FIX #6: Calls pickBatchWithinBudget after fetching messages so token budget
* is respected before handing the batch to the LLM.
* FIX #7: Unified single-timer path -- always clear-and-reset one timer per
* conversation key regardless of whether a cooldown is active. The delay is
* simply max(cooldownRemainder+500, debounce) so the same timer serves both
* the "throttled by error cooldown" and "normal debounce" cases, eliminating
* the previous two-path logic that could leave both timers live simultaneously.
*/
export function scheduleConversationAnalysis(conversationKey: string): void {
if (isConversationProcessingLocked(conversationKey)) {
return;
}
const convoCooldown = conversationErrorCooldown.get(conversationKey) ?? 0;
const convoErrors = conversationConsecutiveErrors.get(conversationKey) ?? 0;
// Hard-block: circuit breaker threshold reached AND cooldown still active.
if (convoErrors >= MAX_CONSECUTIVE_ERRORS && Date.now() < convoCooldown) {
return;
}
// Unified delay: honour the cooldown window if active, otherwise use the
// normal debounce interval. Always clear-and-reset so only ONE timer is
// ever pending per conversation key regardless of call source.
const now = Date.now();
const delayMs =
convoCooldown > now
? convoCooldown - now + 500
: config.AI_ANALYSIS_DEBOUNCE_MS;
const existingTimer = conversationDebounceTimers.get(conversationKey);
if (existingTimer) {
clearTimeout(existingTimer);
}
const timer = setTimeout(() => {
conversationDebounceTimers.delete(conversationKey);
// FIX TOCTOU: Set lock synchronously BEFORE the async DB fetch starts
if (isConversationProcessingLocked(conversationKey)) {
return;
}
const processingStartedAt = Date.now();
conversationProcessing.set(conversationKey, processingStartedAt);
// FIX #3: explicit .catch() -- no async arrow function to avoid unhandled rejection.
getPendingMessagesByConversation(
conversationKey,
config.AI_ANALYSIS_MAX_BATCH_SIZE,
)
.then(async (messages: MessageRecord[]) => {
if (messages.length === 0) {
if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey);
}
return;
}
const processableMessages = await skipAgeRestrictedMessages(messages);
if (processableMessages.length === 0) {
if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey);
}
return;
}
// FIX #6: trim to token budget before sending to LLM.
let trimmed = pickBatchWithinBudget(
processableMessages,
config.AI_ANALYSIS_MAX_TARGET_TOKENS,
50,
);
// FIX #10: if every message individually exceeds the token budget,
// fall back to the first message alone.
if (trimmed.length === 0 && processableMessages.length > 0) {
trimmed = processableMessages.slice(0, 1);
logger.warn(
{
conversationKey,
messageId: processableMessages[0]?.id,
tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS,
},
"All messages exceed token budget -- processing first message alone to avoid stuck-pending deadlock",
);
}
return processBatch(conversationKey, trimmed, processingStartedAt);
})
.catch((err: unknown) => {
if (
conversationProcessing.get(conversationKey) === processingStartedAt
) {
conversationProcessing.delete(conversationKey);
}
logger.error(
{
conversationKey,
error: err instanceof Error ? err.message : String(err),
},
"Failed to fetch or dispatch pending messages for scheduled analysis",
);
});
}, delayMs);
conversationDebounceTimers.set(conversationKey, timer);
}
@@ -1,3 +1,4 @@
import { createChildLogger } from "@bete/shared/logger";
import { eq } from "drizzle-orm";
import { getDatabase } from "../../shared/database/drizzle.js";
import {
@@ -5,6 +6,8 @@ import {
channelCulturesTable,
} from "../../shared/database/schema.js";
const logger = createChildLogger("channelCultureStore");
/**
* Fetch the AI-generated culture summary for a channel.
*/
@@ -18,6 +21,11 @@ export async function getChannelCulture(
.where(eq(channelCulturesTable.channel_id, channelId))
.limit(1);
if (existing[0]) {
logger.debug({ channelId }, "Channel culture lookup: found");
} else {
logger.debug({ channelId }, "Channel culture lookup: not found");
}
return existing[0] || null;
}
@@ -46,4 +54,9 @@ export async function updateChannelCulture(
last_analyzed_at: Date.now(),
},
});
logger.debug(
{ channelId, guildId, cultureSummary },
"Channel culture updated",
);
}
@@ -0,0 +1,235 @@
import { existsSync } from "node:fs";
import { availableParallelism } from "node:os";
import { fileURLToPath } from "node:url";
import { createChildLogger } from "@bete/shared/logger";
import type { Client } from "discord.js-selfbot-v13";
import { LRUCache } from "lru-cache";
import { Piscina } from "piscina";
import { config } from "../../shared/config/config.js";
import type { EventBroadcaster } from "../event-broadcaster/index.js";
import type { MessageRecord } from "../message-capture/types.js";
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
const logger = createChildLogger("circuit-breaker");
// ---------------------------------------------------------------------------
// Piscina worker pool (shared by batch + individual pipelines)
// ---------------------------------------------------------------------------
function getAnalysisWorkerUrl(): URL {
const candidates = [
new URL("./aiAnalysisWorker.js", import.meta.url),
new URL("../aiAnalysisWorker.js", import.meta.url),
new URL("./aiAnalysisWorker.ts", import.meta.url),
];
for (const candidate of candidates) {
if (existsSync(fileURLToPath(candidate))) {
return candidate;
}
}
return candidates[2];
}
export const workerPool = new Piscina({
filename: fileURLToPath(getAnalysisWorkerUrl()),
execArgv: process.execArgv,
maxThreads: config.PISCINA_MAX_THREADS ?? availableParallelism(),
});
/**
* Gets the conversation key for a message (thread_id or channel_id).
*/
export function getConversationKey(message: MessageRecord): string {
return message.thread_id || message.channel_id;
}
// ---------------------------------------------------------------------------
// Shared observable state
// ---------------------------------------------------------------------------
/** Redis EventBroadcaster -- set externally so sub-modules can publish events. */
export let _redisEventBroadcaster: EventBroadcaster | undefined;
/** Discord client reference -- needed for auto-delete actions. */
export let moderationClient: Client | undefined;
export function setSharedEventBroadcaster(
eb: EventBroadcaster | undefined,
): void {
_redisEventBroadcaster = eb;
}
export function setModerationClient(mc: Client | undefined): void {
moderationClient = mc;
}
/**
* Per-message in-flight guard for the auto-delete side-effect.
* (LRU-backed to prevent unbounded growth)
*/
export const autoDeleteInFlight = new LRUCache<string, true>({ max: 10000 });
/** Last recorded error across all pipelines. */
export const LAST_ERROR: { value: string | null } = { value: null };
// ---------------------------------------------------------------------------
// Batch circuit breaker state
// ---------------------------------------------------------------------------
export const conversationConsecutiveErrors = new LRUCache<string, number>({
max: 10000,
});
export const MAX_CONSECUTIVE_ERRORS = 5;
export const CONVERSATION_CB_COOLDOWN_MS = 60000;
export const conversationErrorCooldown = new LRUCache<string, number>({
max: 10000,
});
// ---------------------------------------------------------------------------
// Scheduling / timing state (shared so sub-modules can access without cycles)
// ---------------------------------------------------------------------------
/** Debounce timer handle per conversation key. */
export const conversationDebounceTimers = new LRUCache<string, NodeJS.Timeout>({
max: 10000,
dispose: (value) => {
clearTimeout(value);
},
});
/** Timestamp of when processing started per conversation key. */
export const conversationProcessing = new LRUCache<string, number>({
max: 10000,
});
// ---------------------------------------------------------------------------
// Conversation lock helper
// ---------------------------------------------------------------------------
export function isConversationProcessingLocked(
conversationKey: string,
): boolean {
const startedAt = conversationProcessing.get(conversationKey);
return Boolean(
startedAt &&
Date.now() - startedAt < config.AI_ANALYSIS_PROCESSING_TIMEOUT_MS,
);
}
// ---------------------------------------------------------------------------
// Alert system
// ---------------------------------------------------------------------------
export type CircuitBreakerAlert = {
type: "conversation_cb" | "individual_cb" | "sustained_error";
conversationKey?: string;
consecutiveErrors: number;
message: string;
lastError?: string | null;
};
const alertHandlers: Array<(alert: CircuitBreakerAlert) => void> = [];
/**
* Register an alert handler (e.g., for webhook integration).
*/
export function onCircuitBreakerAlert(
handler: (alert: CircuitBreakerAlert) => void,
): void {
alertHandlers.push(handler);
}
export function fireAlert(alert: CircuitBreakerAlert): void {
logger.warn(alert, `CB Alert: ${alert.type} -- ${alert.message}`);
for (const handler of alertHandlers) {
try {
handler(alert);
} catch {
// handler errors are non-critical
}
}
}
// ---------------------------------------------------------------------------
// Circuit breaker helpers
// ---------------------------------------------------------------------------
export function recordConversationBatchFailure(conversationKey: string): void {
const nextCount =
(conversationConsecutiveErrors.get(conversationKey) ?? 0) + 1;
conversationConsecutiveErrors.set(conversationKey, nextCount);
if (nextCount >= MAX_CONSECUTIVE_ERRORS) {
conversationErrorCooldown.set(
conversationKey,
Date.now() + CONVERSATION_CB_COOLDOWN_MS,
);
fireAlert({
type: "conversation_cb",
conversationKey,
consecutiveErrors: nextCount,
message: `Conversation ${conversationKey} circuit breaker triggered after ${nextCount} consecutive errors`,
lastError: LAST_ERROR.value,
});
conversationConsecutiveErrors.set(conversationKey, 0);
}
}
export function resetConversationBatchFailures(conversationKey: string): void {
conversationConsecutiveErrors.delete(conversationKey);
}
// ---------------------------------------------------------------------------
// Broadcast & auto-delete helpers
// ---------------------------------------------------------------------------
export function broadcastAnalysisCompleted(row: MessageRecord): void {
if (_redisEventBroadcaster) {
_redisEventBroadcaster.messageAnalyzed(row).catch((err: unknown) =>
logger.warn(
{
messageId: row.id,
error: err instanceof Error ? err.message : String(err),
},
"Failed to publish message_analyzed via Redis EventBroadcaster",
),
);
}
}
export function scheduleAutoDelete(row: MessageRecord): void {
if (row.ai_status !== "flagged" && row.ai_status !== "warn") return;
if (autoDeleteInFlight.has(row.id)) {
logger.debug(
{ messageId: row.id },
"Auto-delete skipped: already in-flight for this message",
);
return;
}
autoDeleteInFlight.set(row.id, true);
const run = () => {
attemptAutoDeleteFlaggedMessage(moderationClient, row)
.catch((error: unknown) => {
logger.error(
{
messageId: row.id,
error: error instanceof Error ? error.message : String(error),
},
"Unexpected auto-delete error",
);
})
.finally(() => {
autoDeleteInFlight.delete(row.id);
});
};
if (config.AUTO_DELETE_FLAGGED_DELAY_MS > 0) {
setTimeout(run, config.AUTO_DELETE_FLAGGED_DELAY_MS);
return;
}
setImmediate(run);
}
@@ -1,6 +1,9 @@
import { createChildLogger } from "@bete/shared/logger";
import pLimit from "p-limit";
import { config } from "../../shared/config/config.js";
const logger = createChildLogger("concurrencyLimiter");
/**
* Concurrency limiter for LLM API calls.
*
@@ -9,6 +12,42 @@ import { config } from "../../shared/config/config.js";
*/
const llmSemaphore = pLimit(config.AI_LLM_MAX_CONCURRENT ?? 5);
let activeCount = 0;
let pendingCount = 0;
// Track queue state changes for logging
function updateCounts(): void {
// p-limit exposes queueSize and activeCount via constructor internals,
// but we track via our wrapper to avoid depending on internals.
}
export async function withLlmConcurrency<T>(fn: () => Promise<T>): Promise<T> {
return llmSemaphore(fn);
const queuedAt = activeCount + pendingCount;
pendingCount++;
logger.debug(
{ activeCount, pendingCount, maxConcurrent: config.AI_LLM_MAX_CONCURRENT },
"Queuing LLM request",
);
return llmSemaphore(async () => {
pendingCount--;
activeCount++;
if (activeCount >= (config.AI_LLM_MAX_CONCURRENT ?? 5)) {
logger.warn(
{ activeCount, maxConcurrent: config.AI_LLM_MAX_CONCURRENT },
"LLM concurrency limit reached",
);
}
try {
return await fn();
} finally {
activeCount--;
logger.debug(
{ activeCount, pendingCount },
"LLM request completed, concurrency slot released",
);
}
});
}
@@ -1,7 +1,10 @@
import { createChildLogger } from "@bete/shared/logger";
import { encoding_for_model as encodingForModel } from "tiktoken";
import { formatMediaEvidenceForPrompt } from "../message-capture/messageMetadata.js";
import type { MessageRecord } from "../message-capture/types.js";
const logger = createChildLogger("conversationContext");
export interface ConversationContextInput {
contextBefore: MessageRecord[];
targets: MessageRecord[];
@@ -29,7 +32,12 @@ function formatTimestamp(ms: number): string {
*/
export function estimateTokens(text: string): number {
// Use tiktoken for accurate token counting (+15 overhead for JSON structure)
return getEncoder().encode(text).length + 15;
const tokens = getEncoder().encode(text).length + 15;
logger.debug(
{ tokenEstimate: tokens, textLength: text.length },
"Estimated tokens for text",
);
return tokens;
}
/**
@@ -81,5 +89,14 @@ export function buildConversationContext(
}
}
logger.debug(
{
targetCount: targets.length,
contextCount: selectedContextLines.length,
usedTokens,
maxTokens,
},
"Conversation context built",
);
return selectedContextLines;
}
@@ -0,0 +1,76 @@
import { createChildLogger } from "@bete/shared/logger";
const log = createChildLogger("imageMimeSniffer");
/**
* Sniff the first bytes of a buffer to determine if it is a supported image
* format. Returns the canonical MIME type string on success, or null if the
* bytes are not a recognizable image.
*/
export function sniffImageMimeType(buf: Buffer): string | null {
if (buf.length < 12) return null;
if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {
return "image/jpeg";
}
if (
buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47 &&
buf[4] === 0x0d &&
buf[5] === 0x0a &&
buf[6] === 0x1a &&
buf[7] === 0x0a
) {
return "image/png";
}
if (
buf[0] === 0x47 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x38
) {
return "image/gif";
}
if (
buf[0] === 0x52 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x46 &&
buf[8] === 0x57 &&
buf[9] === 0x45 &&
buf[10] === 0x42 &&
buf[11] === 0x50
) {
return "image/webp";
}
if (
buf.length >= 12 &&
buf[4] === 0x66 &&
buf[5] === 0x74 &&
buf[6] === 0x79 &&
buf[7] === 0x70
) {
const brand = buf.subarray(8, 12).toString("ascii");
if (brand.startsWith("avif") || brand.startsWith("avis")) {
return "image/avif";
}
if (
brand.startsWith("mif1") ||
brand.startsWith("heic") ||
brand.startsWith("heis")
) {
return "image/heic";
}
}
return null;
}
// Keep log referenced so TS does not tree-shake the logger init
log.debug("imageMimeSniffer loaded");
@@ -0,0 +1,346 @@
import { createChildLogger } from "@bete/shared/logger";
import { LRUCache } from "lru-cache";
import { config } from "../../shared/config/config.js";
import { updateMessagesAIAnalysisBulk } from "../message-capture/messageStore.js";
import type {
AnalysisResult,
MessageRecord,
} from "../message-capture/types.js";
import {
broadcastAnalysisCompleted,
fireAlert,
getConversationKey,
LAST_ERROR,
scheduleAutoDelete,
workerPool,
} from "./circuitBreaker.js";
import { logModerationError } from "./responseLogger.js";
const logger = createChildLogger("individual-fallback");
// ---------------------------------------------------------------------------
// Individual fallback queue state
// ---------------------------------------------------------------------------
/** IDs currently being processed one-by-one (LRU-backed, max 10k entries). */
export const individualInFlight = new LRUCache<string, true>({ max: 10000 });
/**
* Per-conversation count of in-flight individual messages.
* (LRU-backed to prevent unbounded growth)
*/
export const individualInFlightByConversation = new LRUCache<string, number>({
max: 10000,
});
/** Last-touched timestamp for pruning stale entries (LRU-backed). */
export const individualInFlightLastTouched = new LRUCache<string, number>({
max: 10000,
});
/** Counter for observability. */
export let activeIndividualRequests = 0;
// ---------------------------------------------------------------------------
// Individual fallback circuit breaker (independent of batch CB)
// ---------------------------------------------------------------------------
let individualConsecutiveErrors = 0;
export let individualCooldownUntil = 0;
const INDIVIDUAL_COOLDOWN_MS = 60000;
// ---------------------------------------------------------------------------
// Individual fallback pipeline
// ---------------------------------------------------------------------------
/**
* Processes a single message via the Piscina worker pool (offloaded from
* main thread to avoid blocking the event loop).
*/
async function processIndividualFallback(
message: MessageRecord,
): Promise<void> {
const { id: messageId } = message;
const conversationKey = getConversationKey(message);
activeIndividualRequests++;
individualInFlightByConversation.set(
conversationKey,
(individualInFlightByConversation.get(conversationKey) ?? 0) + 1,
);
individualInFlightLastTouched.set(conversationKey, Date.now());
let exhaustedOnIncomplete = false;
try {
// Run the LLM-heavy work in the worker thread
const workerResult = (await workerPool.run({
type: "individual",
message,
skipNormalAnalysis: false,
} as unknown)) as
| { ok: true; results: AnalysisResult[] }
| { ok: false; results: AnalysisResult[]; error: string };
let analysisResult: { results: AnalysisResult[] } | null = null;
let usedSimpleFallback = false;
if (workerResult.ok) {
const stillIncomplete = workerResult.results.some((r) =>
r.flags.includes("analysis_incomplete"),
);
if (stillIncomplete) {
exhaustedOnIncomplete = true;
analysisResult = null;
} else {
analysisResult = workerResult;
}
}
// Step 2: If normal analysis failed, try SIMPLE fallback via worker
if (!analysisResult) {
logger.info(
{ messageId },
"Normal analysis failed -- trying simple text fallback via worker",
);
const simpleResult = (await workerPool.run({
type: "individual",
message,
skipNormalAnalysis: true,
} as unknown)) as
| { ok: true; results: AnalysisResult[] }
| { ok: false; results: AnalysisResult[]; error: string };
if (simpleResult.ok) {
analysisResult = simpleResult;
usedSimpleFallback = true;
exhaustedOnIncomplete = false;
}
}
if (!analysisResult) {
throw new Error(
`Both normal and simple analysis failed for message ${messageId}`,
);
}
if (usedSimpleFallback) {
logger.info(
{ messageId, status: analysisResult.results[0]?.status },
"Used simple text fallback for individual message (via worker)",
);
}
// Main thread: DB writes + broadcast
const updates = analysisResult.results.map((r) => ({
messageId: r.messageId,
result: {
status: r.status,
flags: JSON.stringify(r.flags),
score: r.score,
analysis: r.analysis,
categories: r.categories,
severity: r.severity,
confidence: r.confidence,
recommendedAction: r.recommendedAction,
analyzedAt: Date.now(),
error: null,
},
}));
const rows = await updateMessagesAIAnalysisBulk(updates);
for (const row of rows) {
broadcastAnalysisCompleted(row);
scheduleAutoDelete(row);
// Update reputation autonomously
if (row.ai_status === "clean") {
import("./userReputationStore.js")
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
.catch((e) =>
logger.error(
{ error: e },
"Failed to record clean message streak in fallback",
),
);
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
import("./userReputationStore.js")
.then((store) =>
store.recordInfraction(
row.user_id,
row.guild_id,
row.ai_severity as "low" | "medium" | "high" | "critical",
),
)
.catch((e) =>
logger.error(
{ error: e },
"Failed to record infraction penalty in fallback",
),
);
}
}
const resultSummary = analysisResult.results[0];
logModerationError([messageId], config.AI_LLM_MODEL, new Error("Success"), {
phase: "individual_fallback",
status: resultSummary?.status,
flags: resultSummary?.flags,
severity: resultSummary?.severity,
confidence: resultSummary?.confidence,
});
individualConsecutiveErrors = 0;
logger.debug(
{ messageId, status: analysisResult.results[0]?.status },
"Individual fallback analysis complete (via worker)",
);
} catch (error) {
individualConsecutiveErrors++;
if (
individualConsecutiveErrors >= config.AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD
) {
individualCooldownUntil = Date.now() + INDIVIDUAL_COOLDOWN_MS;
fireAlert({
type: "individual_cb",
consecutiveErrors: individualConsecutiveErrors,
message: `Individual fallback circuit breaker triggered after ${individualConsecutiveErrors} consecutive errors`,
lastError: LAST_ERROR.value,
});
}
LAST_ERROR.value = error instanceof Error ? error.message : String(error);
logModerationError(
[messageId],
config.AI_LLM_MODEL,
error as Error | string,
{
phase: "individual_fallback",
conversationKey,
exhaustedOnIncomplete,
},
);
if (exhaustedOnIncomplete) {
await updateMessagesAIAnalysisBulk([
{
messageId,
result: {
status: "error",
flags: JSON.stringify(["individual_analysis_exhausted"]),
score: 0,
analysis:
"Individual fallback exhausted all retries: LLM consistently dropped this message even in single-target mode",
categories: ["individual_analysis_exhausted"],
severity: "none",
confidence: 0,
recommendedAction: "review",
analyzedAt: Date.now(),
error: LAST_ERROR.value,
},
},
]).catch((dbErr: unknown) => {
logger.error(
{ messageId, error: String(dbErr) },
"Failed to write terminal exhausted status",
);
});
logger.warn(
{ messageId },
"Individual fallback exhausted -- marked as individual_analysis_exhausted",
);
} else {
logger.error(
{
messageId,
error: LAST_ERROR.value,
stack: error instanceof Error ? error.stack : undefined,
},
"Individual fallback analysis failed (transient) -- will be retried",
);
}
} finally {
activeIndividualRequests--;
individualInFlight.delete(messageId);
const prev = individualInFlightByConversation.get(conversationKey) ?? 1;
if (prev <= 1) {
individualInFlightByConversation.delete(conversationKey);
individualInFlightLastTouched.delete(conversationKey);
} else {
individualInFlightByConversation.set(conversationKey, prev - 1);
individualInFlightLastTouched.set(conversationKey, Date.now());
}
}
}
// ---------------------------------------------------------------------------
// Enqueue individual fallbacks
// ---------------------------------------------------------------------------
/**
* Fans out message records to the individual fallback queue.
*
* FIX #1: Checks concurrency cap before admitting new work.
* FIX #5: Checks individual circuit breaker before admitting new work.
*/
export function enqueueIndividualFallbacks(messages: MessageRecord[]): void {
// FIX #5: Honour the individual circuit breaker.
if (Date.now() < individualCooldownUntil) {
logger.warn(
{
until: new Date(individualCooldownUntil).toISOString(),
skipped: messages.length,
},
"Individual fallback circuit breaker active -- messages will be recovered later",
);
return;
}
// FIX #5: Enforce concurrency cap
const maxConcurrent = config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT ?? 50;
const availableSlots = Math.max(0, maxConcurrent - activeIndividualRequests);
if (availableSlots <= 0) {
logger.debug(
{ maxConcurrent, active: activeIndividualRequests },
"Individual fallback concurrency cap reached -- messages will be recovered later",
);
return;
}
const newMessages = messages
.filter((m) => !individualInFlight.has(m.id))
.slice(0, availableSlots);
if (newMessages.length === 0) return;
logger.debug(
{
count: newMessages.length,
messageIds: newMessages.map((m) => m.id),
},
"Enqueueing individual fallback analysis for batch-incomplete messages",
);
for (const msg of newMessages) {
individualInFlight.set(msg.id, true);
processIndividualFallback(msg).catch((err: unknown) => {
logger.error(
{ messageId: msg.id, error: String(err) },
"Unexpected uncaught error escaping processIndividualFallback",
);
individualInFlight.delete(msg.id);
const ck = getConversationKey(msg);
const prev = individualInFlightByConversation.get(ck) ?? 1;
if (prev <= 1) {
individualInFlightByConversation.delete(ck);
individualInFlightLastTouched.delete(ck);
} else {
individualInFlightByConversation.set(ck, prev - 1);
individualInFlightLastTouched.set(ck, Date.now());
}
});
}
}
@@ -0,0 +1,81 @@
import { createChildLogger } from "@bete/shared/logger";
const log = createChildLogger("jsonExtractor");
/**
* Helper to extract JSON from a potentially conversational or markdown-wrapped string.
*/
export function extractJson(content: string): unknown {
const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g;
const matches = content.matchAll(codeBlockRegex);
for (const match of matches) {
const codeContent = match[1].trim();
try {
const parsed = JSON.parse(codeContent);
if (parsed && typeof parsed === "object") {
return parsed;
}
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to parse JSON from code block — trying next block",
);
}
}
for (let start = 0; start < content.length; start++) {
const firstChar = content[start];
if (firstChar !== "{" && firstChar !== "[") continue;
const stack = [firstChar];
let inString = false;
let escaped = false;
for (let i = start + 1; i < content.length; i++) {
const char = content[i];
if (inString) {
if (escaped) {
escaped = false;
} else if (char === "\\") {
escaped = true;
} else if (char === '"') {
inString = false;
}
continue;
}
if (char === '"') {
inString = true;
continue;
}
if (char === "{" || char === "[") {
stack.push(char);
continue;
}
const last = stack[stack.length - 1];
if ((char === "}" && last === "{") || (char === "]" && last === "[")) {
stack.pop();
if (stack.length === 0) {
const candidate = content.slice(start, i + 1);
try {
const parsed = JSON.parse(candidate);
if (parsed && typeof parsed === "object") {
return parsed;
}
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to parse JSON candidate — trying next position",
);
}
break;
}
}
}
}
throw new Error("No JSON object found in response");
}
@@ -2,7 +2,6 @@ import { createChildLogger } from "@bete/shared/logger";
import { delay, retryWithBackoff } from "@bete/shared/utils";
import { LRUCache } from "lru-cache";
import type { ChatCompletion } from "openai/resources/chat/completions";
import { z } from "zod";
import { config } from "../../shared/config/config.js";
import { resizeImageForVision } from "../attachment-upload/imageResizer.js";
import { extractMessageMediaEvidence } from "../message-capture/messageMetadata.js";
@@ -46,33 +45,31 @@ import {
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
import { initializeUserReputation } from "./userReputationStore.js";
const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]);
const RecommendedActionSchema = z.enum([
"none",
"monitor",
"warn",
"review",
"delete",
"escalate",
]);
export { sniffImageMimeType } from "./imageMimeSniffer.js";
export { extractJson } from "./jsonExtractor.js";
export {
parseModerationResponse,
sanitizeErrorMessage,
} from "./moderationResponseParser.js";
// Re-export all symbols from sub-modules to preserve public API
export {
ModerationResponseSchema,
RecommendedActionSchema,
ResultItemSchema,
SeveritySchema,
} from "./moderationSchemas.js";
export {
clampScore,
DEFERRAL_ANALYSIS_PATTERN,
DEFERRAL_EXCEPTION_PATTERN,
deriveRecommendedAction,
deriveSeverity,
hasDeferralAnalysis,
} from "./severityDeriver.js";
const ResultItemSchema = z.object({
message_id: z.union([z.string(), z.number()]).transform(String),
status: z.enum(["clean", "warn", "flagged"]),
flags: z.array(z.string()).optional(),
score: z.number(),
analysis: z.string().nullable().optional(),
categories: z.array(z.string()).optional(),
severity: SeveritySchema.optional(),
confidence: z.number().optional(),
recommended_action: RecommendedActionSchema.optional(),
policy_version: z.string().optional(),
evidence: z.array(z.string()).optional(),
});
const ModerationResponseSchema = z.object({
results: z.array(ResultItemSchema),
});
import { sniffImageMimeType } from "./imageMimeSniffer.js";
// Internal imports for functions used locally in the facade
import { parseModerationResponse } from "./moderationResponseParser.js";
const log = createChildLogger("llmModerationClient");
@@ -112,371 +109,6 @@ async function buildCorrectedFewShotExamples(): Promise<string> {
}
}
/**
* Enhanced deferral detection pattern (R9).
*
* Only matches patterns where the model explicitly states it cannot make
* a decision and needs human review. Removed overly broad patterns that
* caused false positives:
* - "admin (perlu|harus|sebaiknya)" → common in regular sentences
* - "bisa (berpotensi|mengandung)" → decisive statements, not deferral
* - "maaf|sorry" → opinions/apologies, not deferral
* - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral
*/
const DEFERRAL_ANALYSIS_PATTERN =
/(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i;
/**
* Exceptions: patterns that look like deferral but are actually decisive.
* Expanded to catch more variations where the model gives a clear verdict.
*/
const DEFERRAL_EXCEPTION_PATTERN =
/tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i;
function hasDeferralAnalysis(analysis: string): boolean {
if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false;
return DEFERRAL_ANALYSIS_PATTERN.test(analysis);
}
function clampScore(value: number | undefined, fallback = 0): number {
return Math.max(
0,
Math.min(1, Number.isFinite(value) ? (value as number) : fallback),
);
}
function deriveSeverity(
status: "clean" | "warn" | "flagged",
score: number,
): z.infer<typeof SeveritySchema> {
if (status === "clean") return "none";
if (status === "warn") return score >= 0.65 ? "medium" : "low";
if (score >= 0.9) return "critical";
return score >= 0.75 ? "high" : "medium";
}
function deriveRecommendedAction(
status: "clean" | "warn" | "flagged",
severity: z.infer<typeof SeveritySchema>,
): z.infer<typeof RecommendedActionSchema> {
if (status === "clean") return "none";
if (status === "warn") return severity === "medium" ? "review" : "warn";
if (severity === "critical") return "escalate";
if (severity === "high") return "delete";
return "review";
}
/**
* Helper to extract JSON from a potentially conversational or markdown-wrapped string.
*/
export function extractJson(content: string): unknown {
const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g;
const matches = content.matchAll(codeBlockRegex);
for (const match of matches) {
const codeContent = match[1].trim();
try {
const parsed = JSON.parse(codeContent);
if (parsed && typeof parsed === "object") {
return parsed;
}
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to parse JSON from code block — trying next block",
);
}
}
for (let start = 0; start < content.length; start++) {
const firstChar = content[start];
if (firstChar !== "{" && firstChar !== "[") continue;
const stack = [firstChar];
let inString = false;
let escaped = false;
for (let i = start + 1; i < content.length; i++) {
const char = content[i];
if (inString) {
if (escaped) {
escaped = false;
} else if (char === "\\") {
escaped = true;
} else if (char === '"') {
inString = false;
}
continue;
}
if (char === '"') {
inString = true;
continue;
}
if (char === "{" || char === "[") {
stack.push(char);
continue;
}
const last = stack[stack.length - 1];
if ((char === "}" && last === "{") || (char === "]" && last === "[")) {
stack.pop();
if (stack.length === 0) {
const candidate = content.slice(start, i + 1);
try {
const parsed = JSON.parse(candidate);
if (parsed && typeof parsed === "object") {
return parsed;
}
} catch (err) {
log.debug(
{ err: err instanceof Error ? err.message : String(err) },
"Failed to parse JSON candidate — trying next position",
);
}
break;
}
}
}
}
throw new Error("No JSON object found in response");
}
/**
* Sanitize error messages for client-facing output (R10).
* Internal details are logged but the caller gets a generic message.
*/
function sanitizeErrorMessage(internalMsg: string, messageId: string): string {
// Log the full error for debugging
log.warn(
{ messageId, internalError: internalMsg },
"Internal moderation error (sanitized for client)",
);
// Return generic message without internal details
return `Analisis gagal dan memerlukan pemeriksaan manual. Error code: MOD_${Date.now().toString(36).slice(0, 6)}`;
}
export function parseModerationResponse(
content: string,
targetIds: string[],
): AnalysisResult[] {
let parsed: any;
try {
parsed = JSON.parse(content);
} catch (e) {
parsed = extractJson(content);
}
if (Array.isArray(parsed)) {
parsed = { results: parsed };
} else if (parsed && typeof parsed === "object" && !("results" in parsed)) {
if ("message_id" in parsed) {
parsed = { results: [parsed] };
} else {
const arrayKey = Object.keys(parsed).find((key) => {
const val = parsed[key];
return (
Array.isArray(val) &&
val.length > 0 &&
val.every(
(item: unknown) =>
typeof item === "object" &&
item !== null &&
"message_id" in (item as Record<string, unknown>),
)
);
});
if (arrayKey) {
parsed.results = parsed[arrayKey];
} else {
parsed = { results: [parsed] };
}
}
}
const parseResult = ModerationResponseSchema.safeParse(parsed);
if (!parseResult.success) {
throw new Error(`Zod validation failed: ${parseResult.error.message}`);
}
const response = parseResult.data;
const foundIds = new Set<string>();
const targetIdSet = new Set(targetIds);
const results: (AnalysisResult | null)[] = response.results.map((result) => {
const {
message_id,
status,
flags,
score,
analysis,
categories,
severity,
confidence,
recommended_action,
policy_version,
evidence,
} = result;
const finalId = message_id.trim();
if (!targetIdSet.has(finalId)) {
return null;
}
if (foundIds.has(finalId)) {
throw new Error(
`Duplicate message_id in moderation response: ${finalId}`,
);
}
foundIds.add(finalId);
const coalescedAnalysis = analysis ?? "";
if (hasDeferralAnalysis(coalescedAnalysis)) {
throw new Error(
`Deferral analysis is not allowed for message ${finalId}; return a direct moderation decision`,
);
}
const normalizedScore = clampScore(score);
const normalizedConfidence = clampScore(confidence, normalizedScore);
const normalizedSeverity =
severity ?? deriveSeverity(status, normalizedScore);
return {
messageId: finalId,
status: status as "clean" | "warn" | "flagged",
flags: flags ?? [],
score: normalizedScore,
analysis: coalescedAnalysis,
categories: categories ?? flags ?? [],
severity: normalizedSeverity,
confidence: normalizedConfidence,
recommendedAction:
recommended_action ??
deriveRecommendedAction(status, normalizedSeverity),
policyVersion: policy_version ?? "default-2026-05-30",
evidence: evidence ?? [],
};
});
const filteredResults = results.filter(
(r): r is AnalysisResult => r !== null,
);
const missingIds = targetIds.filter((id) => !foundIds.has(id));
if (missingIds.length > 0) {
log.warn(
{ missingIds, foundCount: foundIds.size, totalCount: targetIds.length },
"Some target IDs missing in response - marking as incomplete",
);
for (const missingId of missingIds) {
filteredResults.push({
messageId: missingId,
status: "error",
flags: ["analysis_incomplete"],
score: 0,
analysis: sanitizeErrorMessage(
"Analysis incomplete - LLM did not process this message",
missingId,
),
categories: ["analysis_incomplete"],
severity: "none",
confidence: 0,
recommendedAction: "review",
policyVersion: "default-2026-05-30",
evidence: [],
});
}
}
return filteredResults;
}
interface ModerationInput {
targets: MessageRecord[];
contextText: string;
attachments?: AttachmentRecord[];
}
interface ModerationOutput {
results: AnalysisResult[];
raw: unknown;
}
/**
* Sniff the first bytes of a buffer to determine if it is a supported image
* format. Returns the canonical MIME type string on success, or null if the
* bytes are not a recognizable image.
*/
function sniffImageMimeType(buf: Buffer): string | null {
if (buf.length < 12) return null;
if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {
return "image/jpeg";
}
if (
buf[0] === 0x89 &&
buf[1] === 0x50 &&
buf[2] === 0x4e &&
buf[3] === 0x47 &&
buf[4] === 0x0d &&
buf[5] === 0x0a &&
buf[6] === 0x1a &&
buf[7] === 0x0a
) {
return "image/png";
}
if (
buf[0] === 0x47 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x38
) {
return "image/gif";
}
if (
buf[0] === 0x52 &&
buf[1] === 0x49 &&
buf[2] === 0x46 &&
buf[3] === 0x46 &&
buf[8] === 0x57 &&
buf[9] === 0x45 &&
buf[10] === 0x42 &&
buf[11] === 0x50
) {
return "image/webp";
}
if (
buf.length >= 12 &&
buf[4] === 0x66 &&
buf[5] === 0x74 &&
buf[6] === 0x79 &&
buf[7] === 0x70
) {
const brand = buf.subarray(8, 12).toString("ascii");
if (brand.startsWith("avif") || brand.startsWith("avis")) {
return "image/avif";
}
if (
brand.startsWith("mif1") ||
brand.startsWith("heic") ||
brand.startsWith("heis")
) {
return "image/heic";
}
}
return null;
}
// ---------------------------------------------------------------------------
// Shared types for image resolution
// ---------------------------------------------------------------------------
@@ -1495,6 +1127,17 @@ async function runMediaBatch(
// Main entry point — splits text-only vs media, runs both paths in parallel
// ---------------------------------------------------------------------------
interface ModerationInput {
targets: MessageRecord[];
contextText: string;
attachments?: AttachmentRecord[];
}
interface ModerationOutput {
results: AnalysisResult[];
raw: unknown;
}
/**
* Runs LLM-based moderation analysis on messages.
*
@@ -0,0 +1,178 @@
import { createChildLogger } from "@bete/shared/logger";
import type { AnalysisResult } from "../message-capture/types.js";
import { extractJson } from "./jsonExtractor.js";
import { ModerationResponseSchema } from "./moderationSchemas.js";
import {
clampScore,
DEFERRAL_ANALYSIS_PATTERN,
DEFERRAL_EXCEPTION_PATTERN,
deriveRecommendedAction,
deriveSeverity,
hasDeferralAnalysis,
} from "./severityDeriver.js";
const log = createChildLogger("moderationResponseParser");
/**
* Re-export deferral patterns for backward compatibility.
* See severityDeriver.ts for the full regex definitions.
*/
export {
DEFERRAL_ANALYSIS_PATTERN,
DEFERRAL_EXCEPTION_PATTERN,
} from "./severityDeriver.js";
/**
* Sanitize error messages for client-facing output (R10).
* Internal details are logged but the caller gets a generic message.
*/
export function sanitizeErrorMessage(
internalMsg: string,
messageId: string,
): string {
// Log the full error for debugging
log.warn(
{ messageId, internalError: internalMsg },
"Internal moderation error (sanitized for client)",
);
// Return generic message without internal details
return `Analisis gagal dan memerlukan pemeriksaan manual. Error code: MOD_${Date.now().toString(36).slice(0, 6)}`;
}
export function parseModerationResponse(
content: string,
targetIds: string[],
): AnalysisResult[] {
let parsed: any;
try {
parsed = JSON.parse(content);
} catch (e) {
parsed = extractJson(content);
}
if (Array.isArray(parsed)) {
parsed = { results: parsed };
} else if (parsed && typeof parsed === "object" && !("results" in parsed)) {
if ("message_id" in parsed) {
parsed = { results: [parsed] };
} else {
const arrayKey = Object.keys(parsed).find((key) => {
const val = parsed[key];
return (
Array.isArray(val) &&
val.length > 0 &&
val.every(
(item: unknown) =>
typeof item === "object" &&
item !== null &&
"message_id" in (item as Record<string, unknown>),
)
);
});
if (arrayKey) {
parsed.results = parsed[arrayKey];
} else {
parsed = { results: [parsed] };
}
}
}
const parseResult = ModerationResponseSchema.safeParse(parsed);
if (!parseResult.success) {
throw new Error(`Zod validation failed: ${parseResult.error.message}`);
}
const response = parseResult.data;
const foundIds = new Set<string>();
const targetIdSet = new Set(targetIds);
const results: (AnalysisResult | null)[] = response.results.map((result) => {
const {
message_id,
status,
flags,
score,
analysis,
categories,
severity,
confidence,
recommended_action,
policy_version,
evidence,
} = result;
const finalId = message_id.trim();
if (!targetIdSet.has(finalId)) {
return null;
}
if (foundIds.has(finalId)) {
throw new Error(
`Duplicate message_id in moderation response: ${finalId}`,
);
}
foundIds.add(finalId);
const coalescedAnalysis = analysis ?? "";
if (hasDeferralAnalysis(coalescedAnalysis)) {
throw new Error(
`Deferral analysis is not allowed for message ${finalId}; return a direct moderation decision`,
);
}
const normalizedScore = clampScore(score);
const normalizedConfidence = clampScore(confidence, normalizedScore);
const normalizedSeverity =
severity ?? deriveSeverity(status, normalizedScore);
return {
messageId: finalId,
status: status as "clean" | "warn" | "flagged",
flags: flags ?? [],
score: normalizedScore,
analysis: coalescedAnalysis,
categories: categories ?? flags ?? [],
severity: normalizedSeverity,
confidence: normalizedConfidence,
recommendedAction:
recommended_action ??
deriveRecommendedAction(status, normalizedSeverity),
policyVersion: policy_version ?? "default-2026-05-30",
evidence: evidence ?? [],
};
});
const filteredResults = results.filter(
(r): r is AnalysisResult => r !== null,
);
const missingIds = targetIds.filter((id) => !foundIds.has(id));
if (missingIds.length > 0) {
log.warn(
{ missingIds, foundCount: foundIds.size, totalCount: targetIds.length },
"Some target IDs missing in response - marking as incomplete",
);
for (const missingId of missingIds) {
filteredResults.push({
messageId: missingId,
status: "error",
flags: ["analysis_incomplete"],
score: 0,
analysis: sanitizeErrorMessage(
"Analysis incomplete - LLM did not process this message",
missingId,
),
categories: ["analysis_incomplete"],
severity: "none",
confidence: 0,
recommendedAction: "review",
policyVersion: "default-2026-05-30",
evidence: [],
});
}
}
return filteredResults;
}
@@ -0,0 +1,41 @@
import { createChildLogger } from "@bete/shared/logger";
import { z } from "zod";
const log = createChildLogger("moderationSchemas");
export const SeveritySchema = z.enum([
"none",
"low",
"medium",
"high",
"critical",
]);
export const RecommendedActionSchema = z.enum([
"none",
"monitor",
"warn",
"review",
"delete",
"escalate",
]);
export const ResultItemSchema = z.object({
message_id: z.union([z.string(), z.number()]).transform(String),
status: z.enum(["clean", "warn", "flagged"]),
flags: z.array(z.string()).optional(),
score: z.number(),
analysis: z.string().nullable().optional(),
categories: z.array(z.string()).optional(),
severity: SeveritySchema.optional(),
confidence: z.number().optional(),
recommended_action: RecommendedActionSchema.optional(),
policy_version: z.string().optional(),
evidence: z.array(z.string()).optional(),
});
export const ModerationResponseSchema = z.object({
results: z.array(ResultItemSchema),
});
// Keep log referenced so TS does not tree-shake the logger init
log.debug("moderationSchemas loaded");
@@ -0,0 +1,64 @@
import { createChildLogger } from "@bete/shared/logger";
import type { z } from "zod";
import {
RecommendedActionSchema,
SeveritySchema,
} from "./moderationSchemas.js";
const log = createChildLogger("severityDeriver");
/**
* Enhanced deferral detection pattern (R9).
*
* Only matches patterns where the model explicitly states it cannot make
* a decision and needs human review. Removed overly broad patterns that
* caused false positives:
* - "admin (perlu|harus|sebaiknya)" → common in regular sentences
* - "bisa (berpotensi|mengandung)" → decisive statements, not deferral
* - "maaf|sorry" → opinions/apologies, not deferral
* - "saya tidak yakin|tahu|paham" → expressing uncertainty, not deferral
*/
export const DEFERRAL_ANALYSIS_PATTERN =
/(?:kurang (?:konteks|bukti|informasi|data) (?:untuk (?:menilai|menentukan|memutuskan)|untuk moderasi)|perlu (?:dicek|diperiksa|ditinjau|dikaji|dievaluasi) (?:oleh )?(?:admin|moderator|manusia|human review)|tidak (?:bisa|dapat|mampu) (?:menentukan|menilai|memastikan|menyimpulkan|memberi keputusan|memoderasi).*(?:karena (?:konteks tidak jelas|informasi tidak cukup|bukti kurang|konteks kurang|tidak cukup konteks)|data tidak cukup|informasi tidak lengkap)|cannot determine|insufficient (?:context|evidence|information) (?:to |for )?(?:moderate|judge|evaluate|decide|classify)|(?:sepertinya|tampaknya) (?:perlu|harus) (?:ditinjau|diperiksa|dicek) (?:oleh )?(?:admin|moderator)|tidak cukup (?:bukti|informasi|konteks) (?:untuk (?:memberikan|membuat|menentukan)|memutuskan))/i;
/**
* Exceptions: patterns that look like deferral but are actually decisive.
* Expanded to catch more variations where the model gives a clear verdict.
*/
export const DEFERRAL_EXCEPTION_PATTERN =
/tidak bisa menentukan.*(?:karena|sebab|dengan alasan|sebab tidak ada).*(?:clean|tidak (?:ada|terdapat|menunjukkan).*(?:pelanggaran|masalah|indikasi|konten)|aman|bersih|normal)/i;
export function hasDeferralAnalysis(analysis: string): boolean {
if (DEFERRAL_EXCEPTION_PATTERN.test(analysis)) return false;
return DEFERRAL_ANALYSIS_PATTERN.test(analysis);
}
export function clampScore(value: number | undefined, fallback = 0): number {
return Math.max(
0,
Math.min(1, Number.isFinite(value) ? (value as number) : fallback),
);
}
export function deriveSeverity(
status: "clean" | "warn" | "flagged",
score: number,
): z.infer<typeof SeveritySchema> {
if (status === "clean") return "none";
if (status === "warn") return score >= 0.65 ? "medium" : "low";
if (score >= 0.9) return "critical";
return score >= 0.75 ? "high" : "medium";
}
export function deriveRecommendedAction(
status: "clean" | "warn" | "flagged",
severity: z.infer<typeof SeveritySchema>,
): z.infer<typeof RecommendedActionSchema> {
if (status === "clean") return "none";
if (status === "warn") return severity === "medium" ? "review" : "warn";
if (severity === "critical") return "escalate";
if (severity === "high") return "delete";
return "review";
}
log.debug("severityDeriver loaded");
@@ -1,3 +1,7 @@
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("stickerPrompt");
/**
* Sticker-specific prompt templates for AI moderation.
*
@@ -17,6 +21,7 @@ export function buildStickerVisionPrompt(
stickerName: string,
messageId: string,
): string {
logger.debug({ stickerName, messageId }, "Building sticker vision prompt");
return [
`Analisis sticker Discord berikut sebagai evidence moderasi.`,
`Sticker "${stickerName}" berasal dari pesan id=${messageId}.`,
@@ -49,6 +54,10 @@ export function buildStickerTextOnlyWarning(
stickerName: string,
stickerUrl: string,
): string {
logger.debug(
{ stickerName, stickerUrl },
"Building sticker text-only warning",
);
return (
`[sticker: "${stickerName}" (${stickerUrl}) — GAMBAR GAGAL DIUNDUH. ` +
`"${stickerName}" adalah sticker kartun/meme Discord. ` +
@@ -68,6 +77,7 @@ export function buildCustomEmojiVisionPrompt(
emojiName: string,
messageId: string,
): string {
logger.debug({ emojiName, messageId }, "Building custom emoji vision prompt");
return [
`Analisis custom emoji Discord berikut sebagai evidence moderasi.`,
`Emoji "${emojiName}" berasal dari pesan id=${messageId}.`,
@@ -87,6 +97,7 @@ export function buildCustomEmojiVisionPrompt(
* Fallback text for when a custom emoji image failed to download.
*/
export function buildCustomEmojiTextOnlyFallback(emojiName: string): string {
logger.debug({ emojiName }, "Building custom emoji text-only fallback");
return (
`[custom_emoji: "${emojiName}" — GAMBAR GAGAL DIUNDUH. ` +
`"${emojiName}" adalah custom emoji Discord (ikon kecil). ` +
@@ -105,6 +116,7 @@ export function buildGeneralImageVisionPrompt(
sourceLabel: string,
_messageId: string,
): string {
logger.debug({ sourceLabel }, "Building general image vision prompt");
return [
`Deskripsikan gambar ini secara objektif dan spesifik.`,
`${sourceLabel}`,
@@ -1,3 +1,4 @@
import { createChildLogger } from "@bete/shared/logger";
import { and, desc, eq } from "drizzle-orm";
import { getDatabase } from "../../shared/database/drizzle.js";
import {
@@ -6,6 +7,8 @@ import {
userReputationsTable,
} from "../../shared/database/schema.js";
const logger = createChildLogger("userReputationStore");
/**
* Ensures a user reputation record exists.
*/
@@ -21,6 +24,7 @@ export async function initializeUserReputation(
.limit(1);
if (existing.length > 0) {
logger.debug({ userId }, "Reputation record already exists");
return existing[0];
}
@@ -40,6 +44,7 @@ export async function initializeUserReputation(
if (!inserted) {
// If concurrent insert happened
logger.debug({ userId }, "Concurrent reputation insert detected, retrying");
const retry = await db
.select()
.from(userReputationsTable)
@@ -48,6 +53,10 @@ export async function initializeUserReputation(
return retry[0];
}
logger.debug(
{ userId, trustScore: inserted.trust_score },
"Initialized user reputation",
);
return inserted;
}
@@ -64,6 +73,14 @@ export async function getUserReputation(
.where(eq(userReputationsTable.user_id, userId))
.limit(1);
if (existing[0]) {
logger.debug(
{ userId, trustScore: existing[0].trust_score },
"Fetched user reputation",
);
} else {
logger.debug({ userId }, "No reputation record found, returning null");
}
return existing[0] || null;
}
@@ -93,6 +110,11 @@ export async function recordCleanMessage(
updated_at: Date.now(),
})
.where(eq(userReputationsTable.user_id, userId));
logger.debug(
{ userId, previousScore: rep.trust_score, newScore, newStreak },
"Clean message recorded, reputation updated",
);
}
/**
@@ -133,6 +155,17 @@ export async function recordInfraction(
updated_at: Date.now(),
})
.where(eq(userReputationsTable.user_id, userId));
logger.info(
{
userId,
severity,
penalty,
newScore,
totalInfractions: rep.total_infractions + 1,
},
"Infraction recorded",
);
}
/**
@@ -1,8 +1,8 @@
import { decodeCursor, encodeCursor } from "@bete/shared";
import { createChildLogger, type Logger } from "@bete/shared/logger";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import { getDatabase } from "../../shared/database/drizzle.js";
import type * as schema from "../../shared/database/schema.js";
import { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
import type {
AttachmentRecord,
MessageQuery,
@@ -18,7 +18,7 @@ import { ModerationActionsDb } from "./moderation-actions.db.js";
import { RetentionDb } from "./retention.db.js";
import { ReviewsDb } from "./reviews.db.js";
export { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
export { decodeCursor, encodeCursor } from "@bete/shared";
export type { AIAnalysisUpdate } from "./messages.db.js";
// ─── Lazy singleton ────────────────────────────────────────────────────────
@@ -0,0 +1,413 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import {
and,
asc,
desc,
eq,
inArray,
isNull,
or,
type SQL,
sql,
} from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type { MessageRecord } from "../message-capture/types.js";
// ─── Helpers ──────────────────────────────────────────────────────────────────
function stringifyAIList(
value: string[] | string | null | undefined,
): string | null {
if (value == null) return null;
return Array.isArray(value) ? JSON.stringify(value) : value;
}
// ─── AIAnalysisUpdate interface ──────────────────────────────────────────────
export interface AIAnalysisUpdate {
status: "pending" | "processing" | "clean" | "warn" | "flagged" | "error";
flags?: string | null;
score?: number | null;
analysis?: string | null;
categories?: string[] | string | null;
severity?: MessageRecord["ai_severity"] | null;
confidence?: number | null;
recommendedAction?: MessageRecord["ai_recommended_action"] | null;
analyzedAt?: number | null;
error?: string | null;
}
// ─── MessagesAnalysis Class ───────────────────────────────────────────────────
export class MessagesAnalysis {
protected logger: Logger;
constructor(
protected db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-analysis");
}
// ── AI Analysis Updates ─────────────────────────────────────────────────────
async updateMessageAIAnalysis(
messageId: string,
result: AIAnalysisUpdate,
): Promise<MessageRecord | null> {
this.logger.debug({ messageId }, "updateMessageAIAnalysis entry");
try {
await this.db
.update(messagesTable)
.set({
ai_status: result.status,
ai_moderation_flags: result.flags ?? null,
ai_moderation_score: result.score ?? null,
ai_analysis: result.analysis ?? null,
ai_categories: stringifyAIList(result.categories),
ai_severity: result.severity ?? null,
ai_confidence: result.confidence ?? result.score ?? null,
ai_recommended_action: result.recommendedAction ?? null,
ai_analyzed_at: result.analyzedAt ?? Date.now(),
ai_error: result.error ?? null,
})
.where(eq(messagesTable.id, messageId));
const rows = await this.db
.select()
.from(messagesTable)
.where(eq(messagesTable.id, messageId));
return (rows[0] as MessageRecord) ?? null;
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message AI analysis",
);
throw error;
}
}
async updateMessagesAIAnalysisBulk(
updates: Array<{ messageId: string; result: AIAnalysisUpdate }>,
): Promise<MessageRecord[]> {
this.logger.debug(
{ count: updates.length },
"updateMessagesAIAnalysisBulk entry",
);
if (updates.length === 0) return [];
try {
const now = Date.now();
await this.db.transaction(async (tx) => {
for (const { messageId, result } of updates) {
await tx
.update(messagesTable)
.set({
ai_status: result.status,
ai_moderation_flags: result.flags ?? null,
ai_moderation_score: result.score ?? null,
ai_analysis: result.analysis ?? null,
ai_categories: stringifyAIList(result.categories),
ai_severity: result.severity ?? null,
ai_confidence: result.confidence ?? result.score ?? null,
ai_recommended_action: result.recommendedAction ?? null,
ai_analyzed_at: result.analyzedAt ?? now,
ai_error: result.error ?? null,
})
.where(eq(messagesTable.id, messageId));
}
});
const ids = updates.map(({ messageId }) => messageId);
const rows = await this.db
.select()
.from(messagesTable)
.where(inArray(messagesTable.id, ids));
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
error: error instanceof Error ? error.message : String(error),
},
"Failed to bulk update messages AI analysis",
);
throw error;
}
}
async getPendingAIAnalysisMessages(
limit: number = 25,
): Promise<MessageRecord[]> {
this.logger.debug({ limit }, "getPendingAIAnalysisMessages entry");
try {
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get pending AI analysis messages",
);
throw error;
}
}
// ── Conversation Context ────────────────────────────────────────────────────
async getConversationContextBefore(input: {
channelId: string;
threadId: string | null;
beforeCreatedAt: number;
limit: number;
}): Promise<MessageRecord[]> {
this.logger.debug(
{ channelId: input.channelId, threadId: input.threadId },
"getConversationContextBefore entry",
);
try {
const { channelId, threadId, beforeCreatedAt, limit } = input;
const locationCondition = threadId
? eq(messagesTable.thread_id, threadId)
: eq(messagesTable.channel_id, channelId);
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
locationCondition,
sql`${messagesTable.created_at} < ${beforeCreatedAt}`,
isNull(messagesTable.deleted_at),
),
)
.orderBy(desc(messagesTable.created_at))
.limit(limit);
return (rows as MessageRecord[]).reverse();
} catch (error) {
this.logger.error(
{
channelId: input.channelId,
threadId: input.threadId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get conversation context before",
);
throw error;
}
}
async getPendingMessagesByConversation(
conversationKey: string,
limit: number = 200,
): Promise<MessageRecord[]> {
this.logger.debug(
{ conversationKey, limit },
"getPendingMessagesByConversation entry",
);
try {
const rows = await this.db.transaction(async (tx) => {
const pendingIdsQuery = tx
.select({ id: messagesTable.id })
.from(messagesTable)
.where(
and(
or(
eq(messagesTable.thread_id, conversationKey),
eq(messagesTable.channel_id, conversationKey),
),
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit)
.for("update", { skipLocked: true });
const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>;
if (pendingIds.length === 0) return [];
return await tx
.update(messagesTable)
.set({ ai_status: "processing", ai_analyzed_at: Date.now() })
.where(
inArray(
messagesTable.id,
pendingIds.map((r) => r.id),
),
)
.returning();
});
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
conversationKey,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get pending messages by conversation",
);
throw error;
}
}
async getPendingConversationKeys(limit: number = 500): Promise<string[]> {
this.logger.debug({ limit }, "getPendingConversationKeys entry");
try {
const rows = (await this.db
.selectDistinct({
thread_id: messagesTable.thread_id,
channel_id: messagesTable.channel_id,
})
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.limit(limit)) as Array<{
thread_id: string | null;
channel_id: string;
}>;
const keys: string[] = [];
for (const row of rows) {
const key = row.thread_id || row.channel_id;
if (key && !keys.includes(key)) {
keys.push(key);
}
}
return keys;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get pending conversation keys",
);
throw error;
}
}
async getConversationKeysWithIncompleteAnalysis(
limit: number = 200,
): Promise<string[]> {
this.logger.debug(
{ limit },
"getConversationKeysWithIncompleteAnalysis entry",
);
try {
const rows = (await this.db
.selectDistinct({
thread_id: messagesTable.thread_id,
channel_id: messagesTable.channel_id,
})
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "error"),
sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`,
sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`,
isNull(messagesTable.deleted_at),
),
)
.limit(limit)) as Array<{
thread_id: string | null;
channel_id: string;
}>;
const keys: string[] = [];
for (const row of rows) {
const key = row.thread_id || row.channel_id;
if (key && !keys.includes(key)) {
keys.push(key);
}
}
return keys;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get conversation keys with incomplete analysis",
);
throw error;
}
}
async getIncompleteMessagesByConversation(
conversationKey: string,
limit: number = 500,
): Promise<MessageRecord[]> {
this.logger.debug(
{ conversationKey, limit },
"getIncompleteMessagesByConversation entry",
);
try {
const rows = await this.db.transaction(async (tx) => {
const pendingIdsQuery = tx
.select({ id: messagesTable.id })
.from(messagesTable)
.where(
and(
or(
eq(messagesTable.thread_id, conversationKey),
eq(messagesTable.channel_id, conversationKey),
),
eq(messagesTable.ai_status, "error"),
sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`,
sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`,
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit)
.for("update", { skipLocked: true });
const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>;
if (pendingIds.length === 0) return [];
return await tx
.update(messagesTable)
.set({ ai_status: "processing", ai_analyzed_at: Date.now() })
.where(
inArray(
messagesTable.id,
pendingIds.map((r) => r.id),
),
)
.returning();
});
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
conversationKey,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get incomplete messages by conversation",
);
throw error;
}
}
}
@@ -0,0 +1,86 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, eq, isNull, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type { MessageRecord } from "../message-capture/types.js";
// ─── MessagesCleanup Class ────────────────────────────────────────────────────
export class MessagesCleanup {
private logger: Logger;
constructor(
private db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-cleanup");
}
async getExpiredMessages(retentionDays: number): Promise<MessageRecord[]> {
this.logger.debug({ retentionDays }, "getExpiredMessages entry");
try {
const cutoffTime = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
sql`${messagesTable.created_at} < ${cutoffTime}`,
isNull(messagesTable.deleted_at),
),
)
.limit(1000);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
retentionDays,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get expired messages",
);
throw error;
}
}
async revertStuckProcessingMessages(
timeoutMs: number = 300000,
): Promise<number> {
this.logger.debug({ timeoutMs }, "revertStuckProcessingMessages entry");
try {
const cutoffTime = Date.now() - timeoutMs;
const rows = await this.db
.update(messagesTable)
.set({ ai_status: "pending", ai_analyzed_at: null })
.where(
and(
eq(messagesTable.ai_status, "processing"),
sql`${messagesTable.ai_analyzed_at} < ${cutoffTime}`,
),
)
.returning({ id: messagesTable.id });
if (Array.isArray(rows) && rows.length > 0) {
this.logger.info(
{
count: rows.length,
messageIds: rows.map((r: { id: string }) => r.id),
},
"Reverted stuck processing messages back to pending",
);
}
return Array.isArray(rows) ? rows.length : 0;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to revert stuck processing messages",
);
return 0;
}
}
}
@@ -0,0 +1,204 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, or, type SQL } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type { MessageRecord } from "../message-capture/types.js";
// ─── Shared Helpers ──────────────────────────────────────────────────────────
export function channelOrThreadCondition(channelId: string): SQL {
return or(
eq(messagesTable.channel_id, channelId),
eq(messagesTable.thread_id, channelId),
) as SQL;
}
// ─── MessagesCrud Class ──────────────────────────────────────────────────────
export class MessagesCrud {
protected logger: Logger;
constructor(
protected db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-crud");
}
// ── INSERT ──────────────────────────────────────────────────────────────────
async insertMessage(message: MessageRecord): Promise<void> {
this.logger.debug({ messageId: message.id }, "insertMessage entry");
try {
await this.db
.insert(messagesTable)
.values(message as any)
.onConflictDoNothing();
} catch (error) {
this.logger.error(
{
messageId: message.id,
error: error instanceof Error ? error.message : String(error),
},
"Failed to insert message",
);
throw error;
}
}
async upsertMessageForCapture(message: MessageRecord): Promise<boolean> {
this.logger.debug(
{ messageId: message.id },
"upsertMessageForCapture entry",
);
try {
const messageWithAIStatus = {
...message,
ai_status: "pending" as const,
};
const rows = await this.db
.insert(messagesTable)
.values(messageWithAIStatus as any)
.onConflictDoNothing()
.returning({ id: messagesTable.id });
return rows.length > 0;
} catch (error) {
this.logger.error(
{
messageId: message.id,
error: error instanceof Error ? error.message : String(error),
},
"Failed to upsert message for capture",
);
throw error;
}
}
// ── UPDATE ──────────────────────────────────────────────────────────────────
async updateMessageAsEdited(
messageId: string,
editedContent: string,
editedAt: number,
): Promise<void> {
this.logger.debug({ messageId }, "updateMessageAsEdited entry");
try {
await this.db
.update(messagesTable)
.set({
edited_content: editedContent,
edited_at: editedAt,
type: "edited",
ai_status: "pending",
ai_moderation_flags: null,
ai_moderation_score: null,
ai_analysis: null,
ai_categories: null,
ai_severity: null,
ai_confidence: null,
ai_recommended_action: null,
ai_analyzed_at: null,
ai_error: null,
})
.where(eq(messagesTable.id, messageId));
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message as edited",
);
throw error;
}
}
async updateMessageAsDeleted(
messageId: string,
deletedAt: number,
): Promise<void> {
this.logger.debug({ messageId }, "updateMessageAsDeleted entry");
try {
await this.db
.update(messagesTable)
.set({
deleted_at: deletedAt,
type: "deleted",
})
.where(eq(messagesTable.id, messageId));
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message as deleted",
);
throw error;
}
}
// ── GET ─────────────────────────────────────────────────────────────────────
async getMessagesByChannel(
channelId: string,
limit: number = 50,
offset: number = 0,
guildId?: string,
): Promise<MessageRecord[]> {
this.logger.debug(
{ channelId, limit, offset, guildId },
"getMessagesByChannel entry",
);
try {
const conditions: SQL[] = [channelOrThreadCondition(channelId)];
if (guildId) {
conditions.push(eq(messagesTable.guild_id, guildId));
}
const rows = await this.db
.select()
.from(messagesTable)
.where(and(...conditions))
.orderBy(desc(messagesTable.created_at), desc(messagesTable.id))
.limit(limit)
.offset(offset);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
channelId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get messages by channel",
);
throw error;
}
}
async getMessageById(messageId: string): Promise<MessageRecord | null> {
this.logger.debug({ messageId }, "getMessageById entry");
try {
const rows = await this.db
.select()
.from(messagesTable)
.where(eq(messagesTable.id, messageId));
return (rows[0] as MessageRecord) ?? null;
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get message by id",
);
throw error;
}
}
}
@@ -1,826 +1,161 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import {
and,
asc,
desc,
eq,
inArray,
isNull,
or,
type SQL,
sql,
} from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
import type {
MessageQuery,
MessageRecord,
PageResult,
} from "../message-capture/types.js";
import type { AIAnalysisUpdate } from "./messages.analysis.js";
import { MessagesAnalysis } from "./messages.analysis.js";
import { MessagesCleanup } from "./messages.cleanup.js";
import { MessagesCrud } from "./messages.crud.js";
import { MessagesPagination } from "./messages.pagination.js";
import { MessagesSearch } from "./messages.search.js";
// ─── Helpers ────────────────────────────────────────────────────────────────
// Re-export AIAnalysisUpdate for consumers (messageStore.ts imports it)
export type { AIAnalysisUpdate } from "./messages.analysis.js";
function channelOrThreadCondition(channelId: string): SQL {
return or(
eq(messagesTable.channel_id, channelId),
eq(messagesTable.thread_id, channelId),
) as SQL;
}
function buildListMessageConditions(query: MessageQuery): SQL[] {
const conditions: SQL[] = [];
if (query.guildId) {
conditions.push(eq(messagesTable.guild_id, query.guildId));
}
if (query.channelId) {
conditions.push(channelOrThreadCondition(query.channelId));
}
if (query.threadId) {
conditions.push(eq(messagesTable.thread_id, query.threadId));
}
if (query.userId) {
conditions.push(eq(messagesTable.user_id, query.userId));
}
if (query.status && query.status.length > 0) {
conditions.push(sql`${messagesTable.ai_status} in ${query.status}`);
}
if (query.q) {
const pattern = `%${query.q.toLowerCase()}%`;
conditions.push(sql`lower(${messagesTable.content}) like ${pattern}`);
}
const cursorData = decodeCursor(query.cursor);
if (cursorData) {
conditions.push(
sql`(${messagesTable.created_at} < ${cursorData.created_at} or (${messagesTable.created_at} = ${cursorData.created_at} and ${messagesTable.id} < ${cursorData.id}))`,
);
}
return conditions;
}
function pageRows<T extends { created_at: number; id: string }>(
rows: unknown[],
limit: number,
): PageResult<T> {
const hasMore = rows.length > limit;
const data = rows.slice(0, limit) as T[];
const lastItem = data[data.length - 1];
const nextCursor =
hasMore && lastItem
? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id })
: null;
return { data, nextCursor };
}
function pageMessages(
rows: unknown[],
limit: number,
): PageResult<MessageRecord> {
return pageRows<MessageRecord>(rows, limit);
}
function stringifyAIList(
value: string[] | string | null | undefined,
): string | null {
if (value == null) return null;
return Array.isArray(value) ? JSON.stringify(value) : value;
}
// ─── AIAnalysisUpdate interface ────────────────────────────────────────────
export interface AIAnalysisUpdate {
status: "pending" | "processing" | "clean" | "warn" | "flagged" | "error";
flags?: string | null;
score?: number | null;
analysis?: string | null;
categories?: string[] | string | null;
severity?: MessageRecord["ai_severity"] | null;
confidence?: number | null;
recommendedAction?: MessageRecord["ai_recommended_action"] | null;
analyzedAt?: number | null;
error?: string | null;
}
// ─── MessagesDb Class ──────────────────────────────────────────────────────
// ─── MessagesDb Facade ────────────────────────────────────────────────────────
// Thin facade that delegates to domain-specific sub-modules.
export class MessagesDb {
private logger: Logger;
private crud: MessagesCrud;
private analysis: MessagesAnalysis;
private search: MessagesSearch;
private pagination: MessagesPagination;
private cleanup: MessagesCleanup;
constructor(
private db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-db");
constructor(db: NodePgDatabase<typeof schema>, _parentLogger?: Logger) {
const logger = _parentLogger ?? createChildLogger("messages-db");
this.crud = new MessagesCrud(db, logger);
this.analysis = new MessagesAnalysis(db, logger);
this.search = new MessagesSearch(db, logger);
this.pagination = new MessagesPagination(db, logger);
this.cleanup = new MessagesCleanup(db, logger);
}
// ── CRUD ──────────────────────────────────────────────────────────────
// ── CRUD ────────────────────────────────────────────────────────────────
async insertMessage(message: MessageRecord): Promise<void> {
this.logger.debug({ messageId: message.id }, "insertMessage entry");
try {
await this.db
.insert(messagesTable)
.values(message as any)
.onConflictDoNothing();
} catch (error) {
this.logger.error(
{
messageId: message.id,
error: error instanceof Error ? error.message : String(error),
},
"Failed to insert message",
);
throw error;
}
insertMessage(message: MessageRecord): Promise<void> {
return this.crud.insertMessage(message);
}
async upsertMessageForCapture(message: MessageRecord): Promise<boolean> {
this.logger.debug(
{ messageId: message.id },
"upsertMessageForCapture entry",
);
try {
const messageWithAIStatus = {
...message,
ai_status: "pending" as const,
};
const rows = await this.db
.insert(messagesTable)
.values(messageWithAIStatus as any)
.onConflictDoNothing()
.returning({ id: messagesTable.id });
return rows.length > 0;
} catch (error) {
this.logger.error(
{
messageId: message.id,
error: error instanceof Error ? error.message : String(error),
},
"Failed to upsert message for capture",
);
throw error;
}
upsertMessageForCapture(message: MessageRecord): Promise<boolean> {
return this.crud.upsertMessageForCapture(message);
}
async updateMessageAsEdited(
updateMessageAsEdited(
messageId: string,
editedContent: string,
editedAt: number,
): Promise<void> {
this.logger.debug({ messageId }, "updateMessageAsEdited entry");
try {
await this.db
.update(messagesTable)
.set({
edited_content: editedContent,
edited_at: editedAt,
type: "edited",
ai_status: "pending",
ai_moderation_flags: null,
ai_moderation_score: null,
ai_analysis: null,
ai_categories: null,
ai_severity: null,
ai_confidence: null,
ai_recommended_action: null,
ai_analyzed_at: null,
ai_error: null,
})
.where(eq(messagesTable.id, messageId));
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message as edited",
);
throw error;
}
return this.crud.updateMessageAsEdited(messageId, editedContent, editedAt);
}
async updateMessageAsDeleted(
messageId: string,
deletedAt: number,
): Promise<void> {
this.logger.debug({ messageId }, "updateMessageAsDeleted entry");
try {
await this.db
.update(messagesTable)
.set({
deleted_at: deletedAt,
type: "deleted",
})
.where(eq(messagesTable.id, messageId));
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message as deleted",
);
throw error;
}
updateMessageAsDeleted(messageId: string, deletedAt: number): Promise<void> {
return this.crud.updateMessageAsDeleted(messageId, deletedAt);
}
async getMessagesByChannel(
getMessagesByChannel(
channelId: string,
limit: number = 50,
offset: number = 0,
limit?: number,
offset?: number,
guildId?: string,
): Promise<MessageRecord[]> {
this.logger.debug(
{ channelId, limit, offset, guildId },
"getMessagesByChannel entry",
);
try {
const conditions: SQL[] = [
or(
eq(messagesTable.channel_id, channelId),
eq(messagesTable.thread_id, channelId),
) as SQL,
];
if (guildId) {
conditions.push(eq(messagesTable.guild_id, guildId));
}
const rows = await this.db
.select()
.from(messagesTable)
.where(and(...conditions))
.orderBy(desc(messagesTable.created_at), desc(messagesTable.id))
.limit(limit)
.offset(offset);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
channelId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get messages by channel",
);
throw error;
}
return this.crud.getMessagesByChannel(channelId, limit, offset, guildId);
}
async getMessageById(messageId: string): Promise<MessageRecord | null> {
this.logger.debug({ messageId }, "getMessageById entry");
try {
const rows = await this.db
.select()
.from(messagesTable)
.where(eq(messagesTable.id, messageId));
return (rows[0] as MessageRecord) ?? null;
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get message by id",
);
throw error;
}
getMessageById(messageId: string): Promise<MessageRecord | null> {
return this.crud.getMessageById(messageId);
}
// ── AI Analysis ───────────────────────────────────────────────────────
// ── AI Analysis ─────────────────────────────────────────────────────────
async updateMessageAIAnalysis(
updateMessageAIAnalysis(
messageId: string,
result: AIAnalysisUpdate,
): Promise<MessageRecord | null> {
this.logger.debug({ messageId }, "updateMessageAIAnalysis entry");
try {
await this.db
.update(messagesTable)
.set({
ai_status: result.status,
ai_moderation_flags: result.flags ?? null,
ai_moderation_score: result.score ?? null,
ai_analysis: result.analysis ?? null,
ai_categories: stringifyAIList(result.categories),
ai_severity: result.severity ?? null,
ai_confidence: result.confidence ?? result.score ?? null,
ai_recommended_action: result.recommendedAction ?? null,
ai_analyzed_at: result.analyzedAt ?? Date.now(),
ai_error: result.error ?? null,
})
.where(eq(messagesTable.id, messageId));
const rows = await this.db
.select()
.from(messagesTable)
.where(eq(messagesTable.id, messageId));
return (rows[0] as MessageRecord) ?? null;
} catch (error) {
this.logger.error(
{
messageId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to update message AI analysis",
);
throw error;
}
return this.analysis.updateMessageAIAnalysis(messageId, result);
}
async updateMessagesAIAnalysisBulk(
updateMessagesAIAnalysisBulk(
updates: Array<{ messageId: string; result: AIAnalysisUpdate }>,
): Promise<MessageRecord[]> {
this.logger.debug(
{ count: updates.length },
"updateMessagesAIAnalysisBulk entry",
);
if (updates.length === 0) return [];
try {
const now = Date.now();
await this.db.transaction(async (tx) => {
for (const { messageId, result } of updates) {
await tx
.update(messagesTable)
.set({
ai_status: result.status,
ai_moderation_flags: result.flags ?? null,
ai_moderation_score: result.score ?? null,
ai_analysis: result.analysis ?? null,
ai_categories: stringifyAIList(result.categories),
ai_severity: result.severity ?? null,
ai_confidence: result.confidence ?? result.score ?? null,
ai_recommended_action: result.recommendedAction ?? null,
ai_analyzed_at: result.analyzedAt ?? now,
ai_error: result.error ?? null,
})
.where(eq(messagesTable.id, messageId));
}
});
const ids = updates.map(({ messageId }) => messageId);
const rows = await this.db
.select()
.from(messagesTable)
.where(inArray(messagesTable.id, ids));
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
error: error instanceof Error ? error.message : String(error),
},
"Failed to bulk update messages AI analysis",
);
throw error;
}
return this.analysis.updateMessagesAIAnalysisBulk(updates);
}
async getPendingAIAnalysisMessages(
limit: number = 25,
): Promise<MessageRecord[]> {
this.logger.debug({ limit }, "getPendingAIAnalysisMessages entry");
try {
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get pending AI analysis messages",
);
throw error;
}
getPendingAIAnalysisMessages(limit?: number): Promise<MessageRecord[]> {
return this.analysis.getPendingAIAnalysisMessages(limit);
}
// ── Listing / Pagination ──────────────────────────────────────────────
async listMessages(query: MessageQuery): Promise<PageResult<MessageRecord>> {
this.logger.debug({ query }, "listMessages entry");
try {
const conditions = buildListMessageConditions(query);
const rows = await this.db
.select()
.from(messagesTable)
.where(conditions.length > 0 ? and(...conditions) : undefined)
.orderBy(desc(messagesTable.created_at), desc(messagesTable.id))
.limit(query.limit + 1);
return pageMessages(rows, query.limit);
} catch (error) {
this.logger.error(
{
query,
error: error instanceof Error ? error.message : String(error),
},
"Failed to list messages",
);
throw error;
}
}
async listReviewMessages(
query: Omit<MessageQuery, "status">,
): Promise<PageResult<MessageRecord>> {
return this.listMessages({
...query,
status: ["warn", "flagged", "error"],
});
}
// ── Conversation Context ──────────────────────────────────────────────
async getConversationContextBefore(input: {
getConversationContextBefore(input: {
channelId: string;
threadId: string | null;
beforeCreatedAt: number;
limit: number;
}): Promise<MessageRecord[]> {
this.logger.debug(
{ channelId: input.channelId, threadId: input.threadId },
"getConversationContextBefore entry",
);
try {
const { channelId, threadId, beforeCreatedAt, limit } = input;
const locationCondition = threadId
? eq(messagesTable.thread_id, threadId)
: eq(messagesTable.channel_id, channelId);
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
locationCondition,
sql`${messagesTable.created_at} < ${beforeCreatedAt}`,
isNull(messagesTable.deleted_at),
),
)
.orderBy(desc(messagesTable.created_at))
.limit(limit);
return (rows as MessageRecord[]).reverse();
} catch (error) {
this.logger.error(
{
channelId: input.channelId,
threadId: input.threadId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get conversation context before",
);
throw error;
}
return this.analysis.getConversationContextBefore(input);
}
async getPendingMessagesByConversation(
getPendingMessagesByConversation(
conversationKey: string,
limit: number = 200,
limit?: number,
): Promise<MessageRecord[]> {
this.logger.debug(
{ conversationKey, limit },
"getPendingMessagesByConversation entry",
return this.analysis.getPendingMessagesByConversation(
conversationKey,
limit,
);
try {
const rows = await this.db.transaction(async (tx) => {
const pendingIdsQuery = tx
.select({ id: messagesTable.id })
.from(messagesTable)
.where(
and(
or(
eq(messagesTable.thread_id, conversationKey),
eq(messagesTable.channel_id, conversationKey),
),
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit)
.for("update", { skipLocked: true });
const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>;
if (pendingIds.length === 0) return [];
return await tx
.update(messagesTable)
.set({ ai_status: "processing", ai_analyzed_at: Date.now() })
.where(
inArray(
messagesTable.id,
pendingIds.map((r) => r.id),
),
)
.returning();
});
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
conversationKey,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get pending messages by conversation",
);
throw error;
}
}
// ── Conversation Keys ─────────────────────────────────────────────────
async getPendingConversationKeys(limit: number = 500): Promise<string[]> {
this.logger.debug({ limit }, "getPendingConversationKeys entry");
try {
const rows = (await this.db
.selectDistinct({
thread_id: messagesTable.thread_id,
channel_id: messagesTable.channel_id,
})
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "pending"),
isNull(messagesTable.deleted_at),
),
)
.limit(limit)) as Array<{
thread_id: string | null;
channel_id: string;
}>;
const keys: string[] = [];
for (const row of rows) {
const key = row.thread_id || row.channel_id;
if (key && !keys.includes(key)) {
keys.push(key);
}
}
return keys;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get pending conversation keys",
);
throw error;
}
getPendingConversationKeys(limit?: number): Promise<string[]> {
return this.analysis.getPendingConversationKeys(limit);
}
async getConversationKeysWithIncompleteAnalysis(
limit: number = 200,
): Promise<string[]> {
this.logger.debug(
{ limit },
"getConversationKeysWithIncompleteAnalysis entry",
);
try {
const rows = (await this.db
.selectDistinct({
thread_id: messagesTable.thread_id,
channel_id: messagesTable.channel_id,
})
.from(messagesTable)
.where(
and(
eq(messagesTable.ai_status, "error"),
sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`,
sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`,
isNull(messagesTable.deleted_at),
),
)
.limit(limit)) as Array<{
thread_id: string | null;
channel_id: string;
}>;
const keys: string[] = [];
for (const row of rows) {
const key = row.thread_id || row.channel_id;
if (key && !keys.includes(key)) {
keys.push(key);
}
}
return keys;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get conversation keys with incomplete analysis",
);
throw error;
}
getConversationKeysWithIncompleteAnalysis(limit?: number): Promise<string[]> {
return this.analysis.getConversationKeysWithIncompleteAnalysis(limit);
}
async getIncompleteMessagesByConversation(
getIncompleteMessagesByConversation(
conversationKey: string,
limit: number = 500,
limit?: number,
): Promise<MessageRecord[]> {
this.logger.debug(
{ conversationKey, limit },
"getIncompleteMessagesByConversation entry",
return this.analysis.getIncompleteMessagesByConversation(
conversationKey,
limit,
);
try {
const rows = await this.db.transaction(async (tx) => {
const pendingIdsQuery = tx
.select({ id: messagesTable.id })
.from(messagesTable)
.where(
and(
or(
eq(messagesTable.thread_id, conversationKey),
eq(messagesTable.channel_id, conversationKey),
),
eq(messagesTable.ai_status, "error"),
sql`${messagesTable.ai_moderation_flags} LIKE ${"%analysis_incomplete%"}`,
sql`(${messagesTable.ai_moderation_flags} IS NULL OR ${messagesTable.ai_moderation_flags} NOT LIKE ${"%individual_analysis_exhausted%"})`,
isNull(messagesTable.deleted_at),
),
)
.orderBy(asc(messagesTable.created_at))
.limit(limit)
.for("update", { skipLocked: true });
const pendingIds = (await pendingIdsQuery) as Array<{ id: string }>;
if (pendingIds.length === 0) return [];
return await tx
.update(messagesTable)
.set({ ai_status: "processing", ai_analyzed_at: Date.now() })
.where(
inArray(
messagesTable.id,
pendingIds.map((r) => r.id),
),
)
.returning();
});
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
conversationKey,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get incomplete messages by conversation",
);
throw error;
}
}
// ── Search ────────────────────────────────────────────────────────────
// ── Search ──────────────────────────────────────────────────────────────
async searchMessages(input: {
searchMessages(input: {
query: string;
channelId?: string;
guildId?: string;
limit?: number;
}): Promise<MessageRecord[]> {
this.logger.debug({ query: input.query }, "searchMessages entry");
try {
const { query, channelId, guildId, limit = 20 } = input;
const searchPattern = `%${query}%`;
const conditions: (SQL | undefined)[] = [
isNull(messagesTable.deleted_at),
];
if (guildId) {
conditions.push(eq(messagesTable.guild_id, guildId));
}
if (channelId) {
conditions.push(channelOrThreadCondition(channelId));
}
conditions.push(
or(
sql`${messagesTable.content} LIKE ${searchPattern}`,
sql`${messagesTable.edited_content} LIKE ${searchPattern}`,
),
);
const validConditions = conditions.filter(
(c): c is SQL => c !== undefined,
);
const rows = await this.db
.select()
.from(messagesTable)
.where(and(...validConditions))
.orderBy(desc(messagesTable.created_at))
.limit(limit);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
query: input.query,
channelId: input.channelId,
guildId: input.guildId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to search messages",
);
throw error;
}
return this.search.searchMessages(input);
}
// ── Retention / Recovery ──────────────────────────────────────────────
// ── Pagination ──────────────────────────────────────────────────────────
async getExpiredMessages(retentionDays: number): Promise<MessageRecord[]> {
this.logger.debug({ retentionDays }, "getExpiredMessages entry");
try {
const cutoffTime = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
const rows = await this.db
.select()
.from(messagesTable)
.where(
and(
sql`${messagesTable.created_at} < ${cutoffTime}`,
isNull(messagesTable.deleted_at),
),
)
.limit(1000);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
retentionDays,
error: error instanceof Error ? error.message : String(error),
},
"Failed to get expired messages",
);
throw error;
}
listMessages(query: MessageQuery): Promise<PageResult<MessageRecord>> {
return this.pagination.listMessages(query);
}
async revertStuckProcessingMessages(
timeoutMs: number = 300000,
): Promise<number> {
this.logger.debug({ timeoutMs }, "revertStuckProcessingMessages entry");
try {
const cutoffTime = Date.now() - timeoutMs;
listReviewMessages(
query: Omit<MessageQuery, "status">,
): Promise<PageResult<MessageRecord>> {
return this.pagination.listReviewMessages(query);
}
const rows = await this.db
.update(messagesTable)
.set({ ai_status: "pending", ai_analyzed_at: null })
.where(
and(
eq(messagesTable.ai_status, "processing"),
sql`${messagesTable.ai_analyzed_at} < ${cutoffTime}`,
),
)
.returning({ id: messagesTable.id });
// ── Cleanup ─────────────────────────────────────────────────────────────
if (Array.isArray(rows) && rows.length > 0) {
this.logger.info(
{
count: rows.length,
messageIds: rows.map((r: { id: string }) => r.id),
},
"Reverted stuck processing messages back to pending",
);
}
getExpiredMessages(retentionDays: number): Promise<MessageRecord[]> {
return this.cleanup.getExpiredMessages(retentionDays);
}
return Array.isArray(rows) ? rows.length : 0;
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to revert stuck processing messages",
);
return 0;
}
revertStuckProcessingMessages(timeoutMs?: number): Promise<number> {
return this.cleanup.revertStuckProcessingMessages(timeoutMs);
}
}
@@ -0,0 +1,100 @@
import { decodeCursor, encodeCursor, pageResult } from "@bete/shared";
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, type SQL, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type {
MessageQuery,
MessageRecord,
PageResult,
} from "../message-capture/types.js";
import { channelOrThreadCondition } from "./messages.crud.js";
// ─── Helpers ──────────────────────────────────────────────────────────────────
export function buildListMessageConditions(query: MessageQuery): SQL[] {
const conditions: SQL[] = [];
if (query.guildId) {
conditions.push(eq(messagesTable.guild_id, query.guildId));
}
if (query.channelId) {
conditions.push(channelOrThreadCondition(query.channelId));
}
if (query.threadId) {
conditions.push(eq(messagesTable.thread_id, query.threadId));
}
if (query.userId) {
conditions.push(eq(messagesTable.user_id, query.userId));
}
if (query.status && query.status.length > 0) {
conditions.push(sql`${messagesTable.ai_status} in ${query.status}`);
}
if (query.q) {
const pattern = `%${query.q.toLowerCase()}%`;
conditions.push(sql`lower(${messagesTable.content}) like ${pattern}`);
}
const cursorData = decodeCursor(query.cursor);
if (cursorData) {
conditions.push(
sql`(${messagesTable.created_at} < ${cursorData.created_at} or (${messagesTable.created_at} = ${cursorData.created_at} and ${messagesTable.id} < ${cursorData.id}))`,
);
}
return conditions;
}
const pageRows = pageResult;
// ─── MessagesPagination Class ────────────────────────────────────────────────
export class MessagesPagination {
private logger: Logger;
constructor(
private db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-pagination");
}
async listMessages(query: MessageQuery): Promise<PageResult<MessageRecord>> {
this.logger.debug({ query }, "listMessages entry");
try {
const conditions = buildListMessageConditions(query);
const rows = await this.db
.select()
.from(messagesTable)
.where(conditions.length > 0 ? and(...conditions) : undefined)
.orderBy(desc(messagesTable.created_at), desc(messagesTable.id))
.limit(query.limit + 1);
return pageRows<MessageRecord>(rows, query.limit);
} catch (error) {
this.logger.error(
{
query,
error: error instanceof Error ? error.message : String(error),
},
"Failed to list messages",
);
throw error;
}
}
async listReviewMessages(
query: Omit<MessageQuery, "status">,
): Promise<PageResult<MessageRecord>> {
return this.listMessages({
...query,
status: ["warn", "flagged", "error"],
});
}
}
@@ -0,0 +1,76 @@
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, isNull, or, type SQL, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messagesTable } from "../../shared/database/schema.js";
import type { MessageRecord } from "../message-capture/types.js";
import { channelOrThreadCondition } from "./messages.crud.js";
// ─── MessagesSearch Class ─────────────────────────────────────────────────────
export class MessagesSearch {
private logger: Logger;
constructor(
private db: NodePgDatabase<typeof schema>,
_parentLogger?: Logger,
) {
this.logger = createChildLogger("messages-search");
}
async searchMessages(input: {
query: string;
channelId?: string;
guildId?: string;
limit?: number;
}): Promise<MessageRecord[]> {
this.logger.debug({ query: input.query }, "searchMessages entry");
try {
const { query, channelId, guildId, limit = 20 } = input;
const searchPattern = `%${query}%`;
const conditions: (SQL | undefined)[] = [
isNull(messagesTable.deleted_at),
];
if (guildId) {
conditions.push(eq(messagesTable.guild_id, guildId));
}
if (channelId) {
conditions.push(channelOrThreadCondition(channelId));
}
conditions.push(
or(
sql`${messagesTable.content} LIKE ${searchPattern}`,
sql`${messagesTable.edited_content} LIKE ${searchPattern}`,
),
);
const validConditions = conditions.filter(
(c): c is SQL => c !== undefined,
);
const rows = await this.db
.select()
.from(messagesTable)
.where(and(...validConditions))
.orderBy(desc(messagesTable.created_at))
.limit(limit);
return rows as MessageRecord[];
} catch (error) {
this.logger.error(
{
query: input.query,
channelId: input.channelId,
guildId: input.guildId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to search messages",
);
throw error;
}
}
}
@@ -1,28 +1,11 @@
import { decodeCursor, encodeCursor, pageResult } from "@bete/shared";
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, type SQL, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { moderationActionsTable } from "../../shared/database/schema.js";
import { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
import type { ModerationAction, PageResult } from "../message-capture/types.js";
// ─── Helpers ────────────────────────────────────────────────────────────────
function pageRows<T extends { created_at: number; id: string }>(
rows: unknown[],
limit: number,
): PageResult<T> {
const hasMore = rows.length > limit;
const data = rows.slice(0, limit) as T[];
const lastItem = data[data.length - 1];
const nextCursor =
hasMore && lastItem
? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id })
: null;
return { data, nextCursor };
}
// ─── ModerationActionsDb Class ──────────────────────────────────────────────
export class ModerationActionsDb {
@@ -126,7 +109,7 @@ export class ModerationActionsDb {
)
.limit(limit + 1);
return pageRows<ModerationAction>(rows, limit);
return pageResult<ModerationAction>(rows, limit);
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
@@ -1,21 +1,36 @@
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("pagination");
export interface CursorData {
created_at: number;
id: string;
}
export function encodeCursor(data: CursorData): string {
return Buffer.from(JSON.stringify(data)).toString("base64");
const encoded = Buffer.from(JSON.stringify(data)).toString("base64");
logger.debug({ id: data.id, createdAt: data.created_at }, "Encoded cursor");
return encoded;
}
export function decodeCursor(cursor?: string): CursorData | null {
if (!cursor) return null;
if (!cursor) {
logger.debug("No cursor provided to decode");
return null;
}
try {
const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8"));
if (typeof data.created_at === "number" && typeof data.id === "string") {
logger.debug(
{ id: data.id, createdAt: data.created_at },
"Decoded cursor",
);
return data;
}
logger.warn({ cursor }, "Decoded cursor has invalid shape");
return null;
} catch {
} catch (err) {
logger.warn({ cursor, error: String(err) }, "Failed to decode cursor");
return null;
}
}
@@ -1,28 +1,11 @@
import { decodeCursor, encodeCursor, pageResult } from "@bete/shared";
import { createChildLogger, type Logger } from "@bete/shared/logger";
import { and, desc, eq, type SQL, sql } from "drizzle-orm";
import type { NodePgDatabase } from "drizzle-orm/node-postgres";
import type * as schema from "../../shared/database/schema.js";
import { messageReviewsTable } from "../../shared/database/schema.js";
import { decodeCursor, encodeCursor } from "../message-capture/pagination.js";
import type { MessageReview, PageResult } from "../message-capture/types.js";
// ─── Helpers ────────────────────────────────────────────────────────────────
function pageRows<T extends { created_at: number; id: string }>(
rows: unknown[],
limit: number,
): PageResult<T> {
const hasMore = rows.length > limit;
const data = rows.slice(0, limit) as T[];
const lastItem = data[data.length - 1];
const nextCursor =
hasMore && lastItem
? encodeCursor({ created_at: lastItem.created_at, id: lastItem.id })
: null;
return { data, nextCursor };
}
// ─── ReviewsDb Class ────────────────────────────────────────────────────────
export class ReviewsDb {
@@ -128,7 +111,7 @@ export class ReviewsDb {
)
.limit(limit + 1);
return pageRows<MessageReview>(rows, limit);
return pageResult<MessageReview>(rows, limit);
} catch (error) {
this.logger.error(
{ error: error instanceof Error ? error.message : String(error) },
@@ -1,4 +1,7 @@
import { spawn } from "node:child_process";
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("ffmpeg-process");
export interface MuxFfmpegArgsOptions {
inputs: string[];
@@ -42,19 +45,24 @@ export function buildMuxFfmpegArgs(options: MuxFfmpegArgsOptions): string[] {
*/
export function runFfmpeg(args: string[]): Promise<void> {
return new Promise((resolve, reject) => {
logger.debug({ args }, "Starting ffmpeg");
const proc = spawn("ffmpeg", args, {
stdio: ["ignore", "inherit", "inherit"],
});
proc.on("close", (code) => {
if (code === 0) {
logger.debug("ffmpeg completed successfully");
resolve();
} else {
logger.warn({ exitCode: code }, "ffmpeg exited with non-zero code");
reject(new Error(`ffmpeg exited with code ${code}`));
}
});
proc.on("error", (err) => {
logger.error({ error: err.message }, "ffmpeg process error");
reject(err);
});
});
@@ -1,4 +1,7 @@
import { Transform, TransformCallback } from "node:stream";
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("packet-filter");
/**
* Transform stream to filter out audio packets that are too small.
@@ -31,6 +34,11 @@ export class PacketFilter extends Transform {
callback();
}
/** Returns the number of packets filtered out and processed. */
getStats(): { filtered: number; total: number } {
return { filtered: this.filteredCount, total: this.totalCount };
}
_flush(callback: TransformCallback): void {
callback();
}
@@ -1,6 +1,9 @@
import { createChildLogger } from "@bete/shared/logger";
import { EndBehaviorType, type VoiceReceiver } from "@discordjs/voice";
import { config } from "../../../shared/config/config.js";
const logger = createChildLogger("audio-stream");
export interface AudioStreamHandlers {
onPacket: (chunk: Buffer) => void;
onEnd: () => void;
@@ -12,6 +15,8 @@ export function subscribeToAudioStream(
userId: string,
handlers: AudioStreamHandlers,
): NodeJS.ReadableStream {
logger.debug({ userId }, "Subscribing to audio stream");
const audioStream = receiver.subscribe(userId, {
end: {
behavior: EndBehaviorType.AfterSilence,
@@ -20,8 +25,14 @@ export function subscribeToAudioStream(
});
audioStream.on("data", handlers.onPacket);
audioStream.on("end", handlers.onEnd);
audioStream.on("error", handlers.onError);
audioStream.on("end", () => {
logger.debug({ userId }, "Audio stream ended");
handlers.onEnd();
});
audioStream.on("error", (error: Error) => {
logger.warn({ userId, error: error.message }, "Audio stream error");
handlers.onError(error);
});
return audioStream;
}
@@ -1,6 +1,10 @@
import path from "node:path";
import { createChildLogger } from "@bete/shared/logger";
import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
import { config } from "../../../shared/config/config.js";
const logger = createChildLogger("voice-metadata");
import type {
SegmentMetadata,
SegmentState,
@@ -12,12 +16,20 @@ export async function collectUserMetadata(
userId: string,
channel: VoiceChannel,
): Promise<UserMetadata> {
logger.debug({ userId }, "Collecting user metadata");
const user =
client.users.cache.get(userId) ||
(await client.users.fetch(userId).catch(() => null));
(await client.users.fetch(userId).catch(() => {
logger.warn({ userId }, "Failed to fetch user");
return null;
}));
const member =
channel.guild.members.cache.get(userId) ||
(await channel.guild.members.fetch(userId).catch(() => null));
(await channel.guild.members.fetch(userId).catch(() => {
logger.warn({ userId }, "Failed to fetch guild member");
return null;
}));
const username = user?.username ?? "Unknown User";
const roles =
member?.roles.cache
@@ -1,8 +1,11 @@
import fs from "node:fs";
import path from "node:path";
import { createChildLogger } from "@bete/shared/logger";
import * as prism from "prism-media";
import type { SegmentState } from "../../message-capture/types.js";
const logger = createChildLogger("segment");
export function buildSegmentPaths(
userDir: string,
startTime: number,
@@ -54,6 +57,11 @@ export class SegmentManager {
oggStream,
out,
};
logger.debug(
{ index, startTime, filename, userDir: this.userDir },
"Segment opened",
);
return this.currentSegment;
}
@@ -64,6 +72,25 @@ export class SegmentManager {
oggPacketStream.unpipe(segment.oggStream);
segment.oggStream.end();
this.currentSegment = null;
// Get file size after closing
let fileSize = 0;
try {
const stat = fs.statSync(segment.filename);
fileSize = stat.size;
} catch {
// File might not exist yet
}
logger.debug(
{
index: segment.index,
filename: segment.filename,
fileSize,
durationMs: (segment.endTime ?? 0) - segment.startTime,
},
"Segment closed",
);
return segment;
}
@@ -77,6 +104,15 @@ export class SegmentManager {
)
)
return null;
logger.debug(
{
index: this.currentSegment.index,
filename: this.currentSegment.filename,
durationMs: Date.now() - this.currentSegment.startTime,
},
"Segment rotating",
);
this.close(oggPacketStream);
return this.open(oggPacketStream);
}
@@ -1,11 +1,14 @@
import fs, { promises as fsPromises } from "node:fs";
import path from "node:path";
import { createChildLogger } from "@bete/shared/logger";
import type { UserMetadata } from "../../message-capture/types.js";
import {
buildMuxFfmpegArgs,
runFfmpeg as defaultRunFfmpeg,
} from "../ffmpegProcess.js";
const logger = createChildLogger("recording-session");
export type SessionRecordingStatus =
| "pending"
| "completed"
@@ -86,6 +89,16 @@ export function createRecordingSession(
const participants = new Map<string, SessionParticipant>();
const segments: SessionSegmentRef[] = [];
logger.info(
{
sessionId,
guildId: options.guildId,
channelId: options.channelId,
channelName: options.channelName,
},
"Recording session created",
);
return {
sessionId,
recordingsDir: options.recordingsDir,
@@ -108,6 +121,10 @@ export function createRecordingSession(
durationMs: input.endTime - input.startTime,
offsetMs: input.startTime - options.startTime,
});
logger.debug(
{ sessionId, userId: input.user.userId, segmentCount: segments.length },
"Segment registered in session",
);
},
snapshot(endTime: number): SessionRecordingMetadata {
@@ -132,6 +149,11 @@ export function buildSessionMuxFilter(
segments: Array<{ startTime: number }>,
sessionStartTime: number,
): string {
if (segments.length === 0) {
logger.debug("Building mux filter with no segments");
return "";
}
const filters = segments.map((segment, index) => {
const delayMs = Math.max(0, segment.startTime - sessionStartTime);
return `[${index}:a]adelay=${delayMs}|${delayMs}[pad${index}]`;
@@ -140,6 +162,11 @@ export function buildSessionMuxFilter(
filters.push(
`${inputs}amix=inputs=${segments.length}:dropout_transition=0[out]`,
);
logger.debug(
{ segmentCount: segments.length, filter: filters.join(";") },
"Built mux filter",
);
return filters.join(";");
}
@@ -166,26 +193,66 @@ export async function finalizeRecordingSession(
await mkdir(sessionDir);
const metadata = session.snapshot(endTime);
logger.info(
{
sessionId: session.sessionId,
segmentCount: metadata.segments.length,
outputFile,
},
"Finalizing recording session",
);
if (metadata.segments.length === 0) {
await writeJson(metadataFile, { ...metadata, status: "empty" });
logger.info(
{ sessionId: session.sessionId },
"Recording session finalized with no segments",
);
return;
}
try {
await runFfmpeg(
buildMuxFfmpegArgs({
inputs: metadata.segments.map((segment) => segment.oggPath),
filter: buildSessionMuxFilter(metadata.segments, metadata.startTime),
output: outputFile,
codec: "libopus",
}),
const ffmpegArgs = buildMuxFfmpegArgs({
inputs: metadata.segments.map((segment) => segment.oggPath),
filter: buildSessionMuxFilter(metadata.segments, metadata.startTime),
output: outputFile,
codec: "libopus",
});
logger.debug(
{ sessionId: session.sessionId, ffmpegArgs },
"Running FFmpeg mux for session",
);
await runFfmpeg(ffmpegArgs);
// Get output file size
let outputSize = 0;
try {
const outStat = await fsPromises.stat(outputFile);
outputSize = outStat.size;
} catch {
// File might not exist yet, ignore
}
await writeJson(metadataFile, {
...metadata,
status: "completed",
outputFile,
});
logger.info(
{ sessionId: session.sessionId, outputFile, outputSize },
"Recording session finalized successfully",
);
} catch (error) {
logger.error(
{
sessionId: session.sessionId,
error: error instanceof Error ? error.message : String(error),
},
"Failed to finalize recording session via FFmpeg",
);
await writeJson(metadataFile, {
...metadata,
status: "failed",
@@ -1,5 +1,8 @@
import { createChildLogger } from "@bete/shared/logger";
import { retryWithBackoff } from "@bete/shared/utils";
const logger = createChildLogger("tele-upload");
export interface TeleUploadResponse {
download_url: string;
public_id?: string;
@@ -40,6 +43,8 @@ export async function uploadToTele(input: {
const { buffer, filename, contentType, uploadUrl, timeoutMs, retries } =
input;
logger.debug({ filename, uploadUrl }, "Starting tele upload");
const response = await retryWithBackoff(
async () => {
const fileBlob = new Blob([new Uint8Array(buffer)], {
@@ -1,3 +1,4 @@
import { pgAttachmentsTable, pgMessagesTable } from "@bete/shared";
import {
bigint as pgBigint,
boolean as pgBoolean,
@@ -5,7 +6,6 @@ import {
index as pgIndex,
integer as pgInteger,
jsonb as pgJsonb,
real as pgReal,
pgTable,
text as pgText,
timestamp as pgTimestamp,
@@ -41,137 +41,7 @@ export const pgMuxerJobsTable = pgTable(
}),
);
/**
* Messages Table (PostgreSQL)
* Stores text messages with AI moderation analysis
*/
export const pgMessagesTable = pgTable(
"messages",
{
id: pgText("id").primaryKey(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
username: pgText("username").notNull(),
avatar_url: pgText("avatar_url"),
content: pgText("content").notNull(),
edited_content: pgText("edited_content"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
edited_at: pgBigint("edited_at", { mode: "number" }),
deleted_at: pgBigint("deleted_at", { mode: "number" }),
type: pgText("type", { enum: ["text", "edited", "deleted"] })
.notNull()
.default("text"),
metadata: pgText("metadata"),
ai_status: pgText("ai_status", {
enum: ["pending", "processing", "clean", "warn", "flagged", "error"],
})
.notNull()
.default("pending"),
ai_moderation_flags: pgText("ai_moderation_flags"),
ai_moderation_score: pgReal("ai_moderation_score"),
ai_analysis: pgText("ai_analysis"),
ai_categories: pgText("ai_categories"),
ai_severity: pgText("ai_severity", {
enum: ["none", "low", "medium", "high", "critical"],
}),
ai_confidence: pgReal("ai_confidence"),
ai_recommended_action: pgText("ai_recommended_action", {
enum: ["none", "monitor", "warn", "review", "delete", "escalate"],
}),
ai_analyzed_at: pgBigint("ai_analyzed_at", { mode: "number" }),
ai_error: pgText("ai_error"),
},
(table) => ({
channelIdx: pgIndex("idx_messages_channel").on(table.channel_id),
userIdx: pgIndex("idx_messages_user").on(table.user_id),
createdIdx: pgIndex("idx_messages_created").on(table.created_at),
threadIdx: pgIndex("idx_messages_thread").on(table.thread_id),
channelCreatedIdx: pgIndex("idx_messages_channel_created").on(
table.channel_id,
table.created_at,
table.id,
),
threadCreatedIdx: pgIndex("idx_messages_thread_created").on(
table.thread_id,
table.created_at,
table.id,
),
aiStatusCreatedIdx: pgIndex("idx_messages_ai_status_created").on(
table.ai_status,
table.created_at,
table.id,
),
guildAiStatusCreatedIdx: pgIndex("idx_messages_guild_ai_status_created").on(
table.guild_id,
table.ai_status,
table.created_at,
table.id,
),
guildCreatedDeletedIdx: pgIndex("idx_messages_guild_created_deleted").on(
table.guild_id,
table.created_at,
table.deleted_at,
table.id,
),
channelAiStatusCreatedIdx: pgIndex(
"idx_messages_channel_ai_status_created",
).on(table.channel_id, table.ai_status, table.created_at, table.id),
threadAiStatusCreatedIdx: pgIndex(
"idx_messages_thread_ai_status_created",
).on(table.thread_id, table.ai_status, table.created_at, table.id),
}),
);
/**
* Attachments Table (PostgreSQL)
* Stores attachment metadata with upload status tracking
*/
export const pgAttachmentsTable = pgTable(
"attachments",
{
id: pgText("id").primaryKey(),
message_id: pgText("message_id").notNull(),
guild_id: pgText("guild_id").notNull(),
channel_id: pgText("channel_id").notNull(),
thread_id: pgText("thread_id"),
user_id: pgText("user_id").notNull(),
filename: pgText("filename").notNull(),
size: pgInteger("size").notNull(),
type: pgText("type").notNull(),
discord_url: pgText("discord_url").notNull(),
uploaded_url: pgText("uploaded_url"),
upload_status: pgText("upload_status", {
enum: ["pending", "uploaded", "failed"],
})
.notNull()
.default("pending"),
upload_error: pgText("upload_error"),
created_at: pgBigint("created_at", { mode: "number" }).notNull(),
uploaded_at: pgBigint("uploaded_at", { mode: "number" }),
},
(table) => ({
channelIdx: pgIndex("idx_attachments_channel").on(table.channel_id),
messageIdx: pgIndex("idx_attachments_message").on(table.message_id),
statusIdx: pgIndex("idx_attachments_status").on(table.upload_status),
channelCreatedIdx: pgIndex("idx_attachments_channel_created").on(
table.channel_id,
table.created_at,
table.id,
),
threadCreatedIdx: pgIndex("idx_attachments_thread_created").on(
table.thread_id,
table.created_at,
table.id,
),
messageFk: pgForeignKey({
columns: [table.message_id],
foreignColumns: [pgMessagesTable.id],
name: "fk_attachments_message_id",
}).onDelete("cascade"),
}),
);
// (pgMessagesTable and pgAttachmentsTable are imported from @bete/shared)
/**
* UI State Table (PostgreSQL)
@@ -7,6 +7,10 @@ import {
skipMedia,
stopMedia,
} from "../../../shared/api/client";
import { useAsyncAction } from "../../../shared/hooks/useAsyncAction.js";
import { createLogger } from "../../../shared/lib/logger.js";
const logger = createLogger("use-media-control");
const emptyMediaState: MediaState = {
playing: false,
@@ -17,8 +21,7 @@ const emptyMediaState: MediaState = {
export function useMediaControl() {
const [mediaState, setMediaState] = useState<MediaState>(emptyMediaState);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const { loading, error, execute, clearError } = useAsyncAction();
const refreshMedia = useCallback(async () => {
const state = await getMediaStatus();
@@ -28,71 +31,62 @@ export function useMediaControl() {
const enqueue = useCallback(
async (source: string, mode: "music" | "screen") => {
setLoading(true);
setError(null);
try {
const state = await queueMedia(source, mode);
setMediaState(state);
return state;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
const result = await execute(() => queueMedia(source, mode));
if (result) {
setMediaState(result);
logger.info("Media queued", { source, mode });
} else {
logger.error("Failed to queue media", { source, mode });
}
return result;
},
[],
[execute],
);
const skip = useCallback(async () => {
setLoading(true);
setError(null);
try {
const state = await skipMedia();
setMediaState(state);
return state;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
const result = await execute(() => skipMedia());
if (result) {
setMediaState(result);
logger.info("Media skipped");
} else {
logger.error("Failed to skip media");
}
}, []);
return result;
}, [execute]);
const stop = useCallback(async () => {
setLoading(true);
setError(null);
try {
const state = await stopMedia();
setMediaState(state);
return state;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
const result = await execute(() => stopMedia());
if (result) {
setMediaState(result);
logger.info("Media stopped");
} else {
logger.error("Failed to stop media");
}
}, []);
return result;
}, [execute]);
const setVolume = useCallback(async (volume: number) => {
setError(null);
try {
const state = await setMediaVolume(volume);
setMediaState(state);
return state;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
}
}, []);
const setVolume = useCallback(
async (volume: number) => {
clearError();
try {
const state = await setMediaVolume(volume);
setMediaState(state);
logger.info("Volume set", { volume });
return state;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to set volume", { volume, error: message });
throw err;
}
},
[clearError],
);
useEffect(() => {
refreshMedia().catch((err) =>
setError(err instanceof Error ? err.message : String(err)),
logger.error("Failed to refresh media state on mount", {
error: String(err),
}),
);
}, [refreshMedia]);
@@ -8,6 +8,10 @@ import {
getVoiceChannels,
getVoiceStatus,
} from "../../../shared/api/client";
import { useAsyncAction } from "../../../shared/hooks/useAsyncAction.js";
import { createLogger } from "../../../shared/lib/logger.js";
const logger = createLogger("use-voice-control");
export function useVoiceControl() {
const [guilds, setGuilds] = useState<Guild[]>([]);
@@ -19,15 +23,14 @@ export function useVoiceControl() {
activeChannelId: null,
activeChannelName: null,
});
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const { loading, error, execute, clearError } = useAsyncAction();
const refreshGuilds = useCallback(async () => {
setError(null);
clearError();
const nextGuilds = await getGuilds();
setGuilds(nextGuilds);
return nextGuilds;
}, []);
}, [clearError]);
const refreshVoiceStatus = useCallback(async () => {
const status = await getVoiceStatus();
@@ -55,44 +58,39 @@ export function useVoiceControl() {
return channels;
}, []);
const joinVoice = useCallback(async (guildId: string, channelId: string) => {
setLoading(true);
setError(null);
try {
const status = await connectVoice(guildId, channelId);
setVoiceStatus(status);
return status;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
}
}, []);
const joinVoice = useCallback(
async (guildId: string, channelId: string) => {
const result = await execute(() => connectVoice(guildId, channelId));
if (result) {
setVoiceStatus(result);
logger.info("Connected to voice", { guildId, channelId });
} else {
logger.error("Failed to connect to voice", { guildId, channelId });
}
return result;
},
[execute],
);
const leaveVoice = useCallback(async () => {
setLoading(true);
setError(null);
try {
const status = await disconnectVoice();
setVoiceStatus(status);
return status;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
throw err;
} finally {
setLoading(false);
const result = await execute(() => disconnectVoice());
if (result) {
setVoiceStatus(result);
logger.info("Disconnected from voice");
} else {
logger.error("Failed to disconnect from voice");
}
}, []);
return result;
}, [execute]);
useEffect(() => {
refreshGuilds().catch((err) =>
setError(err instanceof Error ? err.message : String(err)),
logger.error("Failed to refresh guilds on mount", { error: String(err) }),
);
refreshVoiceStatus().catch((err) =>
setError(err instanceof Error ? err.message : String(err)),
logger.error("Failed to refresh voice status on mount", {
error: String(err),
}),
);
}, [refreshGuilds, refreshVoiceStatus]);
@@ -5,6 +5,9 @@ import {
reanalyzeErrorBatch,
reanalyzeMessage,
} from "../../../shared/api/client";
import { createLogger } from "../../../shared/lib/logger.js";
const logger = createLogger("use-messages");
const PAGE_SIZE = 100;
@@ -54,6 +57,7 @@ export function useMessages() {
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setError(message);
logger.error("Failed to fetch messages", { guildId, error: message });
throw err;
} finally {
setLoading(false);
@@ -72,6 +76,9 @@ export function useMessages() {
setMessages((prev) => [...prev, ...result.data]);
setCursor(result.nextCursor);
setHasMore(!!result.nextCursor);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to load more messages", { error: message });
} finally {
setLoadingMore(false);
}
@@ -106,6 +113,8 @@ export function useMessages() {
prev.map((message) => (message.id === id ? snapshot : message)),
);
}
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to reanalyze message", { id, error: message });
throw err;
}
}, []);
@@ -124,10 +133,17 @@ export function useMessages() {
: message,
),
);
const { count } = await reanalyzeErrorBatch({
guildId: currentGuild.current ?? undefined,
});
return count;
try {
const { count } = await reanalyzeErrorBatch({
guildId: currentGuild.current ?? undefined,
});
logger.info("Reanalyze all errors complete", { count });
return count;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to reanalyze error batch", { error: message });
throw err;
}
}, []);
return {
+9 -1
View File
@@ -1,6 +1,9 @@
// ─── Shared HTTP client — all API endpoints in one file ──────────────────────
import type { MessageRecord, PageResult } from "@bete/shared";
import { createLogger } from "../lib/logger.js";
const logger = createLogger("api");
const BE_API_URL = import.meta.env.VITE_BE_API_URL || "http://localhost:3001";
const BE_WS_URL = import.meta.env.VITE_BE_WS_URL || "ws://localhost:3001";
@@ -20,6 +23,8 @@ class ApiError extends Error {
export async function request<T>(path: string, init?: RequestInit): Promise<T> {
const password = localStorage.getItem("admin-password");
const url = path.startsWith("http") ? path : `${BE_API_URL}${path}`;
logger.debug("Request", { method: init?.method ?? "GET", url });
const res = await fetch(url, {
headers: {
"Content-Type": "application/json",
@@ -38,10 +43,13 @@ export async function request<T>(path: string, init?: RequestInit): Promise<T> {
} catch {
// ignore parse errors
}
logger.error("Request failed", { url, status: res.status, code, message });
throw new ApiError(code, message, res.status);
}
return res.json() as Promise<T>;
const result = (await res.json()) as T;
logger.debug("Response", { url, status: res.status });
return result;
}
export function getWebSocketURL(): string {
@@ -0,0 +1,37 @@
// ─── Generic async action state hook ──────────────────────────────────────
import { useCallback, useState } from "react";
interface AsyncActionState {
loading: boolean;
error: string | null;
}
export function useAsyncAction() {
const [state, setState] = useState<AsyncActionState>({
loading: false,
error: null,
});
const execute = useCallback(
async <T>(fn: () => Promise<T>): Promise<T | null> => {
setState({ loading: true, error: null });
try {
const result = await fn();
setState({ loading: false, error: null });
return result;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
setState({ loading: false, error: message });
return null;
}
},
[],
);
const clearError = useCallback(() => {
setState((prev) => ({ ...prev, error: null }));
}, []);
return { ...state, execute, clearError };
}
@@ -1,5 +1,8 @@
// ─── Audio playback hook — receives PCM from WebSocket and plays through Web Audio API ──
import { useCallback, useRef, useState } from "react";
import { createLogger } from "../lib/logger.js";
const logger = createLogger("use-audio-playback");
const SAMPLE_RATE = 24000;
const CHANNELS = 1;
@@ -15,56 +18,64 @@ export function useAudioPlayback() {
const handleIncomingPcm = useCallback(
(data: { userId: string; pcm: string }) => {
// Decode base64 PCM data
const binaryString = atob(data.pcm);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
const int16Array = new Int16Array(bytes.buffer);
try {
const binaryString = atob(data.pcm);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
const int16Array = new Int16Array(bytes.buffer);
// Calculate audio levels for visualization
let sum = 0;
for (const sample of int16Array) sum += Math.abs(sample / 32768);
const average = int16Array.length ? sum / int16Array.length : 0;
setLevels((prev) =>
prev.map((_, index) =>
Math.max(
0.04,
average *
(0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) *
5,
// Calculate audio levels for visualization
let sum = 0;
for (const sample of int16Array) sum += Math.abs(sample / 32768);
const average = int16Array.length ? sum / int16Array.length : 0;
setLevels((prev) =>
prev.map((_, index) =>
Math.max(
0.04,
average *
(0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) *
5,
),
),
),
);
);
const audioContext = audioContextRef.current;
if (!isListening || !audioContext) return;
const audioContext = audioContextRef.current;
if (!isListening || !audioContext) return;
// Convert to float32 for Web Audio API
const float32Array = new Float32Array(int16Array.length);
for (let i = 0; i < int16Array.length; i++)
float32Array[i] = int16Array[i] / 32768;
// Convert to float32 for Web Audio API
const float32Array = new Float32Array(int16Array.length);
for (let i = 0; i < int16Array.length; i++)
float32Array[i] = int16Array[i] / 32768;
const audioBuffer = audioContext.createBuffer(
CHANNELS,
float32Array.length,
SAMPLE_RATE,
);
audioBuffer.getChannelData(0).set(float32Array);
const audioBuffer = audioContext.createBuffer(
CHANNELS,
float32Array.length,
SAMPLE_RATE,
);
audioBuffer.getChannelData(0).set(float32Array);
const source = audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(audioContext.destination);
const source = audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(audioContext.destination);
// Schedule playback per user to avoid overlaps
const currentTime = audioContext.currentTime;
let nextStart = userTimelinesRef.current.get(data.userId) || 0;
if (nextStart < currentTime) nextStart = currentTime + 0.05;
source.start(nextStart);
userTimelinesRef.current.set(
data.userId,
nextStart + audioBuffer.duration,
);
// Schedule playback per user to avoid overlaps
const currentTime = audioContext.currentTime;
let nextStart = userTimelinesRef.current.get(data.userId) || 0;
if (nextStart < currentTime) nextStart = currentTime + 0.05;
source.start(nextStart);
userTimelinesRef.current.set(
data.userId,
nextStart + audioBuffer.duration,
);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
logger.error("Failed to decode PCM audio", {
userId: data.userId,
error: message,
});
}
},
[isListening],
);
@@ -74,6 +85,7 @@ export function useAudioPlayback() {
await audioContextRef.current?.suspend();
userTimelinesRef.current.clear();
setIsListening(false);
logger.info("Audio playback paused");
return;
}
const AudioContextCtor =
@@ -85,6 +97,7 @@ export function useAudioPlayback() {
});
await audioContextRef.current.resume();
setIsListening(true);
logger.info("Audio playback started");
}, [isListening]);
return {
@@ -0,0 +1,22 @@
// ─── Client-side structured logger ────────────────────────────────────────
const LOG_PREFIX = "[Bete]";
export function createLogger(context: string) {
const prefix = `${LOG_PREFIX} [${context}]`;
return {
debug: (msg: string, data?: Record<string, unknown>) => {
if (import.meta.env.DEV) console.debug(prefix, msg, data ?? "");
},
info: (msg: string, data?: Record<string, unknown>) => {
console.info(prefix, msg, data ?? "");
},
warn: (msg: string, data?: Record<string, unknown>) => {
console.warn(prefix, msg, data ?? "");
},
error: (msg: string, data?: Record<string, unknown>) => {
console.error(prefix, msg, data ?? "");
},
};
}
+21 -4
View File
@@ -1,5 +1,8 @@
// ─── WebSocket singleton with reconnect, typed events, and observable status ─
import { useCallback, useEffect, useRef, useState } from "react";
import { createLogger } from "../lib/logger.js";
const logger = createLogger("socket");
export type WsStatus = "connecting" | "connected" | "disconnected" | "error";
@@ -26,6 +29,7 @@ export interface WsHandlers {
let _wsInstance: WebSocket | null = null;
let _reconnectTimer: ReturnType<typeof setTimeout> | null = null;
let _closed = false;
let _reconnectAttempts = 0;
const _listeners = new Set<WsHandlers>();
const _statusCallbacks = new Set<(s: WsStatus) => void>();
@@ -41,12 +45,23 @@ function doConnect(): WebSocket {
const ws = new WebSocket(url);
ws.binaryType = "arraybuffer";
dispatchStatus("connecting");
logger.info("Connecting", { url });
ws.addEventListener("open", () => dispatchStatus("connected"));
ws.addEventListener("error", () => dispatchStatus("error"));
ws.addEventListener("close", () => {
ws.addEventListener("open", () => {
_reconnectAttempts = 0;
dispatchStatus("connected");
logger.info("Connected");
});
ws.addEventListener("error", () => {
dispatchStatus("error");
logger.error("WebSocket error");
});
ws.addEventListener("close", (event) => {
dispatchStatus("disconnected");
logger.info("Disconnected", { code: event.code, reason: event.reason });
if (!_closed && _listeners.size > 0) {
_reconnectAttempts++;
logger.warn("Reconnecting", { attempt: _reconnectAttempts });
_reconnectTimer = setTimeout(() => doReconnect(), 2500);
}
});
@@ -108,7 +123,9 @@ function doConnect(): WebSocket {
}
}
} catch {
// ignore malformed messages
logger.error("Failed to parse message", {
raw: event.data.slice(0, 200),
});
}
});