refactor: comprehensive codebase cleanup and architecture hardening

- Sprint 1 (Quick Wins): Remove dead analytics modules, fix 4 unresolved
  imports, replace 3 console.warn with logger, remove mock-crc import
- Sprint 2 (Architecture): Create MascotChatRepository, AnalysisRepository,
  3 Zod schemas (mascot-chat, analysis, voice), deduplicate error classes,
  move 3 SQL queries from routes to repository
- Sprint 3 (Complexity): Replace 7 any types with proper interfaces,
  extract 6 helpers from prepareMediaMessage (CC 85 -> ~15)
- Sprint 4 (Config): Remove 22 dead env vars from .env, add 30 missing
  vars to .env.example, standardize naming

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-06-09 10:16:04 +07:00
co-authored by Claude Opus 4.8
parent d0d9e1669e
commit 4becf0d6f1
89 changed files with 1260 additions and 5499 deletions
+51 -1
View File
@@ -1,5 +1,8 @@
# Discord Bot Configuration # Discord Bot Configuration
DISCORD_TOKEN=your_bot_token_here DISCORD_TOKEN=your_bot_token_here
MONITOR_GUILD_ID=your_guild_id_here
TEXT_GUILD_ID=optional_text_guild_id
TEXT_CHANNEL_ID=optional_text_channel_id
# Recording Configuration # Recording Configuration
RECORDINGS_DIR=./recordings RECORDINGS_DIR=./recordings
@@ -50,6 +53,39 @@ AI_LLM_BASE_URL=https://9router.asepharyana.tech/v1
AI_LLM_MODEL=free AI_LLM_MODEL=free
# Vision model for image/video moderation (falls back to AI_LLM_MODEL if unset) # Vision model for image/video moderation (falls back to AI_LLM_MODEL if unset)
AI_LLM_VISION_MODEL=multimodal AI_LLM_VISION_MODEL=multimodal
# Max concurrent LLM API calls (default: 5)
AI_LLM_MAX_CONCURRENT=5
# Maximum image dimension in pixels before resize for vision API (default: 1024)
AI_LLM_IMAGE_MAX_DIMENSION=1024
# Maximum messages per text-only moderation batch (default: 20)
AI_LLM_TEXT_BATCH_SIZE=20
# Timeout in ms for individual media analysis calls (default: 60000)
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS=60000
# AI Moderation Analysis Tuning (advanced)
AI_ANALYSIS_DEBOUNCE_MS=500
AI_ANALYSIS_RECOVERY_INTERVAL_MS=15000
AI_ANALYSIS_ERROR_COOLDOWN_MS=30000
# Max messages fetched per conversation batch (default: 200)
AI_ANALYSIS_MAX_BATCH_SIZE=200
AI_ANALYSIS_MAX_CONTEXT_TOKENS=8000
# Token budget for target messages specifically (default: 4000)
AI_ANALYSIS_MAX_TARGET_TOKENS=4000
AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT=20
# How long a conversation is locked while being processed (default: 120000ms)
AI_ANALYSIS_PROCESSING_TIMEOUT_MS=120000
# Max concurrent individual-fallback jobs (default: 50)
AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT=50
# Consecutive errors before individual circuit breaker trips (default: 50)
AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD=50
# OpenAI Moderation (optional separate provider)
# OPENAI_MODERATION_API_KEY=your_key_here
# OPENAI_MODERATION_BASE_URL=https://api.openai.com/v1
# OPENAI_MODERATION_MODEL=omni-moderation-latest
# Admin
ADMIN_PASSWORD=admin123
# Database Configuration (PostgreSQL) # Database Configuration (PostgreSQL)
# Option 1: Use DATABASE_URL for connection string # Option 1: Use DATABASE_URL for connection string
@@ -72,22 +108,36 @@ AI_LLM_VISION_MODEL=multimodal
# Auto-Delete Configuration # Auto-Delete Configuration
AUTO_DELETE_FLAGGED_ENABLED=true AUTO_DELETE_FLAGGED_ENABLED=true
AUTO_DELETE_FLAGGED_DRY_RUN=true AUTO_DELETE_FLAGGED_DRY_RUN=true
AUTO_DELETE_FLAGGED_DELAY_MS=0
AUTO_DELETE_MIN_CONFIDENCE=0.50 AUTO_DELETE_MIN_CONFIDENCE=0.50
AUTO_DELETE_ALLOWED_SEVERITIES=critical,high,medium AUTO_DELETE_ALLOWED_SEVERITIES=critical,high,medium
AUTO_DELETE_NOTIFY_USER=false
# Optional: comma-separated channel/user IDs to exclude # Optional: comma-separated channel/user IDs to exclude
# AUTO_DELETE_EXCLUDED_CHANNEL_IDS= # AUTO_DELETE_EXCLUDED_CHANNEL_IDS=
# AUTO_DELETE_EXCLUDED_USER_IDS= # AUTO_DELETE_EXCLUDED_USER_IDS=
# Optional: comma-separated category filter (empty = all categories) # Optional: comma-separated category filter (empty = all categories)
# AUTO_DELETE_ALLOWED_CATEGORIES= # AUTO_DELETE_ALLOWED_CATEGORIES=
# Optional: log channel ID for auto-delete actions
# AUTO_DELETE_LOG_CHANNEL_ID=
# Retention Configuration (0 = disabled)
RETENTION_MESSAGES_DAYS=0
RETENTION_ATTACHMENTS_DAYS=0
RETENTION_VOICE_DAYS=0
# Cleanup interval in ms (default: 24h)
RETENTION_CLEANUP_INTERVAL_MS=86400000
RETENTION_DRY_RUN=true
# Database Migration Configuration # Database Migration Configuration
# Safe default: run migrations on startup before the app accepts traffic. # Safe default: run migrations on startup before the app accepts traffic.
AUTO_MIGRATE_ON_STARTUP=true AUTO_MIGRATE_ON_STARTUP=true
# Worker Pool Configuration
# PISCINA_MAX_THREADS=4
# Cache Model Versioning # Cache Model Versioning
# Bump this version when the vision/LLM model prompt changes significantly. # Bump this version when the vision/LLM model prompt changes significantly.
# Old cache entries with mismatched versions are automatically ignored, forcing fresh analysis. # Old cache entries with mismatched versions are automatically ignored, forcing fresh analysis.
# Format: "v<N>" or "v<N>-<date>-<description>" # Format: "v<N>" or "v<N>-<date>-<description>"
# Example progression: v1 → v2-2026-06-02-terminal-fix → v3-2026-06-15-new-model # Example progression: v1 → v2-2026-06-02-terminal-fix → v3-2026-06-15-new-model
# CACHE_MODEL_VERSION=v2-2026-06-02 # CACHE_MODEL_VERSION=v2-2026-06-02
+1 -5
View File
@@ -21,11 +21,7 @@ export class ValidationError extends AppError {
export class NotFoundError extends AppError { export class NotFoundError extends AppError {
constructor(resource: string, id?: string) { constructor(resource: string, id?: string) {
super( super(`${resource} not found${id ? `: ${id}` : ""}`, "NOT_FOUND", 404);
`${resource} not found${id ? `: ${id}` : ""}`,
"NOT_FOUND",
404,
);
this.name = "NotFoundError"; this.name = "NotFoundError";
} }
} }
+9 -8
View File
@@ -1,7 +1,7 @@
import path from "node:path"; import path from "node:path";
import Database from "better-sqlite3"; import Database from "better-sqlite3";
import { createChildLogger } from "../src/logger"; import { createChildLogger } from "@bete/shared/logger";
import * as postgres from "../src/database/postgres"; import { getPool, closeDatabase, initializeDatabase } from "../services/backend/src/shared/database/index.js";
const logger = createChildLogger("migrate-data"); const logger = createChildLogger("migrate-data");
@@ -76,7 +76,8 @@ async function migrateData(): Promise<void> {
logger.info({ dbPath }, "SQLite database opened"); logger.info({ dbPath }, "SQLite database opened");
// Initialize PostgreSQL pool // Initialize PostgreSQL pool
const pool = postgres.getPool(); await initializeDatabase();
const pool = getPool();
logger.info("PostgreSQL connection pool initialized"); logger.info("PostgreSQL connection pool initialized");
// Migrate muxer_jobs table // Migrate muxer_jobs table
@@ -85,7 +86,7 @@ async function migrateData(): Promise<void> {
const muxerJobs = muxerJobsStmt.all() as MuxerJob[]; const muxerJobs = muxerJobsStmt.all() as MuxerJob[];
for (const job of muxerJobs) { for (const job of muxerJobs) {
await postgres.query( await pool.query(
`INSERT INTO muxer_jobs (id, data, status, attempts, maxAttempts, createdAt, updatedAt, error) `INSERT INTO muxer_jobs (id, data, status, attempts, maxAttempts, createdAt, updatedAt, error)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (id) DO NOTHING`, ON CONFLICT (id) DO NOTHING`,
@@ -109,7 +110,7 @@ async function migrateData(): Promise<void> {
const messages = messagesStmt.all() as Message[]; const messages = messagesStmt.all() as Message[];
for (const msg of messages) { for (const msg of messages) {
await postgres.query( await pool.query(
`INSERT INTO messages ( `INSERT INTO messages (
id, guild_id, channel_id, thread_id, user_id, username, avatar_url, id, guild_id, channel_id, thread_id, user_id, username, avatar_url,
content, edited_content, created_at, edited_at, deleted_at, type, content, edited_content, created_at, edited_at, deleted_at, type,
@@ -153,7 +154,7 @@ async function migrateData(): Promise<void> {
const attachments = attachmentsStmt.all() as Attachment[]; const attachments = attachmentsStmt.all() as Attachment[];
for (const att of attachments) { for (const att of attachments) {
await postgres.query( await pool.query(
`INSERT INTO attachments ( `INSERT INTO attachments (
id, message_id, guild_id, channel_id, thread_id, user_id, filename, id, message_id, guild_id, channel_id, thread_id, user_id, filename,
size, type, discord_url, uploaded_url, upload_status, upload_error, size, type, discord_url, uploaded_url, upload_status, upload_error,
@@ -189,7 +190,7 @@ async function migrateData(): Promise<void> {
const uiStates = uiStateStmt.all() as UiState[]; const uiStates = uiStateStmt.all() as UiState[];
for (const state of uiStates) { for (const state of uiStates) {
await postgres.query( await pool.query(
`INSERT INTO ui_state (key, value, updated_at) `INSERT INTO ui_state (key, value, updated_at)
VALUES ($1, $2, $3) VALUES ($1, $2, $3)
ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`, ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`,
@@ -224,7 +225,7 @@ async function migrateData(): Promise<void> {
} }
// Close PostgreSQL pool // Close PostgreSQL pool
await postgres.closePool(); await closeDatabase();
logger.info("PostgreSQL connection pool closed"); logger.info("PostgreSQL connection pool closed");
} }
} }
+2 -4
View File
@@ -1,3 +1,4 @@
import { createChildLogger } from "@bete/shared/logger";
import express, { import express, {
type Express, type Express,
type NextFunction, type NextFunction,
@@ -6,7 +7,6 @@ import express, {
} from "express"; } from "express";
import helmet from "helmet"; import helmet from "helmet";
import { createAnalysisRouter } from "../modules/analysis/analysis.routes.js"; import { createAnalysisRouter } from "../modules/analysis/analysis.routes.js";
import { createAnalyticsRouter } from "../modules/analytics/analytics.routes.js";
import { createAuthRouter } from "../modules/auth/auth.routes.js"; import { createAuthRouter } from "../modules/auth/auth.routes.js";
import { createConfigRouter } from "../modules/config/config.routes.js"; import { createConfigRouter } from "../modules/config/config.routes.js";
import { createHealthRouter } from "../modules/health/health.routes.js"; import { createHealthRouter } from "../modules/health/health.routes.js";
@@ -15,9 +15,8 @@ import { createMediaRouter } from "../modules/media/media.routes.js";
import { createMessagesRouter } from "../modules/messages/messages.routes.js"; import { createMessagesRouter } from "../modules/messages/messages.routes.js";
import { createRecordingsRouter } from "../modules/recordings/recordings.routes.js"; import { createRecordingsRouter } from "../modules/recordings/recordings.routes.js";
import { createUiStateRouter } from "../modules/ui-state/ui-state.routes.js"; import { createUiStateRouter } from "../modules/ui-state/ui-state.routes.js";
import { createVoiceRouter } from "../modules/voice/voice.routes.js";
import { createGuildsRouter } from "../modules/voice/guilds.routes.js"; import { createGuildsRouter } from "../modules/voice/guilds.routes.js";
import { createChildLogger } from "@bete/shared/logger"; import { createVoiceRouter } from "../modules/voice/voice.routes.js";
import { errorHandler } from "../shared/middlewares/index.js"; import { errorHandler } from "../shared/middlewares/index.js";
const logger = createChildLogger("http.app"); const logger = createChildLogger("http.app");
@@ -66,7 +65,6 @@ export function createHttpApp(): Express {
app.use("/api", createConfigRouter()); app.use("/api", createConfigRouter());
app.use("/api", createMessagesRouter()); app.use("/api", createMessagesRouter());
app.use("/api", createAnalysisRouter()); app.use("/api", createAnalysisRouter());
app.use("/api", createAnalyticsRouter());
app.use("/api", createMascotChatRouter()); app.use("/api", createMascotChatRouter());
app.use("/api", createMediaRouter()); app.use("/api", createMediaRouter());
app.use("/api", createVoiceRouter()); app.use("/api", createVoiceRouter());
+3 -3
View File
@@ -1,10 +1,10 @@
import { createServer, type Server } from "node:http"; import { createServer, type Server } from "node:http";
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../shared/config/index.js"; import { config } from "../shared/config/index.js";
import { initializeDatabase } from "../shared/database/index.js"; import { initializeDatabase } from "../shared/database/index.js";
import { createChildLogger } from "@bete/shared/logger";
import { createHttpApp } from "./app.js";
import { createWebSocketServer } from "../ws/server.js";
import { startRedisBridge } from "../ws/redis-bridge.js"; import { startRedisBridge } from "../ws/redis-bridge.js";
import { createWebSocketServer } from "../ws/server.js";
import { createHttpApp } from "./app.js";
const logger = createChildLogger("http.server"); const logger = createChildLogger("http.server");
+2 -2
View File
@@ -1,6 +1,6 @@
import { startHttpServer } from "./http/server.js";
import { createChildLogger } from "@bete/shared/logger";
import type { Server } from "node:http"; import type { Server } from "node:http";
import { createChildLogger } from "@bete/shared/logger";
import { startHttpServer } from "./http/server.js";
const logger = createChildLogger("backend"); const logger = createChildLogger("backend");
@@ -0,0 +1,114 @@
import { createChildLogger } from "@bete/shared/logger";
import { getPool } from "../../shared/database/index.js";
const logger = createChildLogger("analysis.repository");
export interface AnalysisSearchQuery {
q?: string;
channelId?: string;
guildId?: string;
limit?: number;
}
export interface AnalysisSearchResult {
id: string;
guild_id: string;
channel_id: string;
thread_id: string | null;
user_id: string;
username: string;
avatar_url: string | null;
content: string;
edited_content: string | null;
created_at: number;
edited_at: number | null;
deleted_at: number | null;
type: string;
metadata: string | null;
ai_status: string | null;
ai_moderation_flags: string | null;
ai_moderation_score: number | null;
ai_analysis: string | null;
ai_categories: string | null;
ai_severity: string | null;
ai_confidence: number | null;
ai_recommended_action: string | null;
ai_analyzed_at: number | null;
ai_error: string | null;
}
function mapSearchResult(row: Record<string, unknown>): AnalysisSearchResult {
return {
id: String(row.id ?? ""),
guild_id: String(row.guild_id ?? ""),
channel_id: String(row.channel_id ?? ""),
thread_id: (row.thread_id as string | null) ?? null,
user_id: String(row.user_id ?? ""),
username: String(row.username ?? ""),
avatar_url: (row.avatar_url as string | null) ?? null,
content: String(row.content ?? ""),
edited_content: (row.edited_content as string | null) ?? null,
created_at: Number(row.created_at ?? 0),
edited_at: (row.edited_at as number | null) ?? null,
deleted_at: (row.deleted_at as number | null) ?? null,
type: String(row.type ?? "text"),
metadata: (row.metadata as string | null) ?? null,
ai_status: (row.ai_status as string | null) ?? null,
ai_moderation_flags: (row.ai_moderation_flags as string | null) ?? null,
ai_moderation_score: (row.ai_moderation_score as number | null) ?? null,
ai_analysis: (row.ai_analysis as string | null) ?? null,
ai_categories: (row.ai_categories as string | null) ?? null,
ai_severity: (row.ai_severity as string | null) ?? null,
ai_confidence: (row.ai_confidence as number | null) ?? null,
ai_recommended_action: (row.ai_recommended_action as string | null) ?? null,
ai_analyzed_at: (row.ai_analyzed_at as number | null) ?? null,
ai_error: (row.ai_error as string | null) ?? null,
};
}
export class AnalysisRepository {
async search(query: AnalysisSearchQuery): Promise<AnalysisSearchResult[]> {
const pool = getPool();
const { q = "", channelId, guildId, limit = 20 } = query;
logger.debug({ q, channelId, guildId, limit }, "Searching analysis");
const searchPattern = `%${q}%`;
const clauses: string[] = ["content ILIKE $1"];
const params: (string | number)[] = [searchPattern];
let p = 2;
if (guildId) {
clauses.push(`guild_id = $${p}`);
params.push(guildId);
p++;
}
if (channelId) {
clauses.push(`channel_id = $${p}`);
params.push(channelId);
p++;
}
const where = clauses.join(" AND ");
const { rows } = await pool.query(
`SELECT
id, guild_id, channel_id, thread_id,
user_id, username, avatar_url,
content, edited_content, created_at, edited_at, deleted_at,
type, metadata,
ai_status, ai_moderation_flags, ai_moderation_score,
ai_analysis, ai_categories, ai_severity, ai_confidence,
ai_recommended_action, ai_analyzed_at, ai_error
FROM messages
WHERE ${where}
ORDER BY created_at DESC
LIMIT $${p}`,
[...params, limit],
);
return rows.map((r) => mapSearchResult(r as Record<string, unknown>));
}
}
export const analysisRepository = new AnalysisRepository();
@@ -1,6 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response, Router } from "express"; import type { Request, Response, Router } from "express";
import express from "express"; import express from "express";
import { createChildLogger } from "@bete/shared/logger";
import { asyncHandler } from "../../shared/middlewares/index.js"; import { asyncHandler } from "../../shared/middlewares/index.js";
import { analysisService } from "./analysis.service.js"; import { analysisService } from "./analysis.service.js";
@@ -0,0 +1,10 @@
import { z } from "zod";
export const searchQuerySchema = z.object({
q: z.string().default(""),
channelId: z.string().optional(),
guildId: z.string().optional(),
limit: z.coerce.number().int().positive().max(100).default(20),
});
export type SearchQuery = z.infer<typeof searchQuerySchema>;
@@ -1,78 +1,26 @@
import { sql } from "drizzle-orm";
import { config } from "../../shared/config/index.js";
import { getDatabase } from "../../shared/database/index.js";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/index.js";
import type { AnalysisSearchQuery } from "./analysis.repository.js";
import { analysisRepository } from "./analysis.repository.js";
const logger = createChildLogger("analysis.service"); const logger = createChildLogger("analysis.service");
export interface AnalysisSearchQuery { export type { AnalysisSearchQuery };
q?: string;
channelId?: string;
limit?: number;
}
/** Full message columns for search results — matches MessageRecord from client.ts */
const FULL_COLUMNS = sql.raw(`
id, guild_id, channel_id, thread_id,
user_id, username, avatar_url,
content, edited_content, created_at, edited_at, deleted_at,
type, metadata,
ai_status, ai_moderation_flags, ai_moderation_score,
ai_analysis, ai_categories, ai_severity, ai_confidence,
ai_recommended_action, ai_analyzed_at, ai_error
`);
export class AnalysisService { export class AnalysisService {
async search(query: AnalysisSearchQuery) { async search(query: AnalysisSearchQuery) {
const db = getDatabase();
const { q = "", channelId, limit = 20 } = query; const { q = "", channelId, limit = 20 } = query;
const guildId = config.MONITOR_GUILD_ID; const guildId = config.MONITOR_GUILD_ID;
logger.debug({ q, channelId, limit, guildId }, "Searching analysis"); logger.debug({ q, channelId, limit, guildId }, "Searching analysis");
const searchPattern = `%${q}%`; const rows = await analysisRepository.search({
const limitVal = limit; q,
channelId,
guildId,
limit,
});
let sqlQuery;
if (channelId && guildId) {
sqlQuery = sql`
SELECT ${FULL_COLUMNS}
FROM messages
WHERE guild_id = ${guildId}
AND channel_id = ${channelId}
AND content ILIKE ${searchPattern}
ORDER BY created_at DESC
LIMIT ${limitVal}
`;
} else if (channelId) {
sqlQuery = sql`
SELECT ${FULL_COLUMNS}
FROM messages
WHERE channel_id = ${channelId}
AND content ILIKE ${searchPattern}
ORDER BY created_at DESC
LIMIT ${limitVal}
`;
} else if (guildId) {
sqlQuery = sql`
SELECT ${FULL_COLUMNS}
FROM messages
WHERE guild_id = ${guildId}
AND content ILIKE ${searchPattern}
ORDER BY created_at DESC
LIMIT ${limitVal}
`;
} else {
sqlQuery = sql`
SELECT ${FULL_COLUMNS}
FROM messages
WHERE content ILIKE ${searchPattern}
ORDER BY created_at DESC
LIMIT ${limitVal}
`;
}
const { rows } = await db.execute(sqlQuery);
return { results: rows }; return { results: rows };
} }
} }
@@ -1,197 +0,0 @@
import { createChildLogger } from "@bete/shared/logger";
import type { NextFunction, Request, Response } from "express";
import { asyncHandler, requireParam } from "../../shared/middlewares/index.js";
import { analyticsQuerySchema } from "./analytics.schema.js";
import { analyticsService } from "./analytics.service.js";
const logger = createChildLogger("analytics.controller");
export function handleGetOverview(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
logger.debug({ query }, "Handling get overview");
const result = await analyticsService.getOverview(query);
res.json(result);
})(req, res, next);
}
export function handleGetDailyTrend(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const guildId = requireParam(
req.query.guildId,
"query parameter",
"guildId",
);
const hours = req.query.hours ? Number(req.query.hours) : 24;
logger.debug({ guildId, hours }, "Handling get daily trend");
const result = await analyticsService.getDailyTrend(guildId, hours);
res.json(result);
})(req, res, next);
}
export function handleGetHourlyStats(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
logger.debug({ query }, "Handling get hourly stats");
const result = await analyticsService.getHourlyStats(
query.guildId,
query.channelId,
query.hours,
);
res.json(result);
})(req, res, next);
}
export function handleGetTopViolators(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
const limit = req.query.limit ? Number(req.query.limit) : 10;
logger.debug({ query, limit }, "Handling get top violators");
const result = await analyticsService.getTopViolators(
query.guildId,
query.channelId,
query.hours,
limit,
);
res.json(result);
})(req, res, next);
}
export function handleGetUserLeaderboard(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
const limit = req.query.limit ? Number(req.query.limit) : 10;
logger.debug({ query, limit }, "Handling get user leaderboard");
const result = await analyticsService.getUserLeaderboard(
query.guildId,
query.channelId,
query.hours,
limit,
);
res.json(result);
})(req, res, next);
}
export function handleGetModerationStats(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
logger.debug({ query }, "Handling get moderation stats");
const result = await analyticsService.getModerationStats(
query.guildId,
query.channelId,
query.hours,
);
res.json(result);
})(req, res, next);
}
export function handleGetHeatmap(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
logger.debug({ query }, "Handling get heatmap");
const result = await analyticsService.getHeatmap(
query.guildId,
query.channelId,
query.hours,
);
res.json(result);
})(req, res, next);
}
export function handleGetTopics(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
logger.debug({ query }, "Handling get topics");
const result = await analyticsService.getTopics(
query.guildId,
query.channelId,
query.hours,
);
res.json(result);
})(req, res, next);
}
export function handleGetModerationActions(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
const limit = req.query.limit ? Number(req.query.limit) : 20;
logger.debug({ query, limit }, "Handling get moderation actions");
const result = await analyticsService.getModerationActions(
query.guildId,
query.channelId,
query.hours,
limit,
);
res.json(result);
})(req, res, next);
}
export function handleGetAIStats(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
logger.debug({ query }, "Handling get AI stats");
const result = await analyticsService.getAIStats(
query.guildId,
query.channelId,
query.hours,
);
res.json(result);
})(req, res, next);
}
export function handleGetAttachmentStats(
req: Request,
res: Response,
next: NextFunction,
) {
return asyncHandler(async (req: Request, res: Response) => {
const query = analyticsQuerySchema.parse(req.query);
logger.debug({ query }, "Handling get attachment stats");
const result = await analyticsService.getAttachmentStats(
query.guildId,
query.channelId,
query.hours,
);
res.json(result);
})(req, res, next);
}
@@ -1,551 +0,0 @@
import { createChildLogger } from "@bete/shared/logger";
import { getPool } from "../../shared/database/index.js";
const logger = createChildLogger("analytics.repository");
interface TimeFilter {
where: string;
params: Array<string | number>;
paramOffset: number;
}
function buildTimeFilter(
guildId: string,
channelId: string | undefined,
hours: number,
offset = 1,
): TimeFilter {
const clauses: string[] = ["guild_id = $" + offset];
const params: Array<string | number> = [guildId];
let p = offset + 1;
if (channelId) {
clauses.push("channel_id = $" + p);
params.push(channelId);
p++;
}
clauses.push("created_at > (EXTRACT(EPOCH FROM NOW()) * 1000 - $" + p + ")");
params.push(hours * 3_600_000);
return { where: "WHERE " + clauses.join(" AND "), params, paramOffset: p };
}
export class AnalyticsRepository {
async getOverview(guildId: string, channelId?: string, hours = 24) {
logger.debug({ guildId, channelId, hours }, "Getting analytics overview");
const pool = getPool();
const now = Date.now();
const start = now - hours * 3_600_000;
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
COUNT(*)::int AS total_messages,
COUNT(DISTINCT user_id)::int AS active_users_count,
COUNT(DISTINCT channel_id)::int AS total_channels,
COUNT(*) FILTER (WHERE ai_status = 'clean')::int AS clean,
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned,
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged,
COUNT(*) FILTER (WHERE ai_status = 'error')::int AS error,
COUNT(*) FILTER (WHERE ai_status = 'pending')::int AS pending,
COALESCE(AVG(ai_moderation_score), 0)::real AS average_score
FROM messages
${filter.where}
`,
filter.params,
);
const row = rows[0] as Record<string, unknown> | undefined;
// Fetch hourly stats, topics, and top violators to include in overview
const hourly = await this.getHourlyStats(guildId, channelId, hours);
const topics = await this.getTopics(guildId, channelId, hours);
const topUsers = await this.getTopViolators(guildId, channelId, hours, 5);
return {
period: { start, end: now },
messages: {
total: Number(row?.total_messages ?? 0),
clean: Number(row?.clean ?? 0),
warned: Number(row?.warned ?? 0),
flagged: Number(row?.flagged ?? 0),
error: Number(row?.error ?? 0),
pending: Number(row?.pending ?? 0),
average_score: Number(row?.average_score ?? 0),
},
hourly,
topics,
top_users: topUsers,
active_users_count: Number(row?.active_users_count ?? 0),
total_channels: Number(row?.total_channels ?? 0),
};
}
async getDailyTrend(guildId: string, hours = 24) {
logger.debug({ guildId, hours }, "Getting daily trend");
const pool = getPool();
const filter = buildTimeFilter(guildId, undefined, hours);
const { rows } = await pool.query(
`
SELECT
TO_CHAR(to_timestamp(created_at / 1000), 'YYYY-MM-DD') AS date,
COUNT(*)::int AS count,
COUNT(*) FILTER (WHERE ai_status = 'clean')::int AS clean,
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned,
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged,
COUNT(*) FILTER (WHERE ai_status = 'error')::int AS error
FROM messages
${filter.where}
GROUP BY date
ORDER BY date ASC
`,
filter.params,
);
return rows.map((r) => ({
date: r.date as string,
count: Number(r.count ?? 0),
clean: Number(r.clean ?? 0),
warned: Number(r.warned ?? 0),
flagged: Number(r.flagged ?? 0),
error: Number(r.error ?? 0),
}));
}
async getHourlyStats(guildId: string, channelId?: string, hours = 24) {
logger.debug({ guildId, channelId, hours }, "Getting hourly stats");
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
TO_CHAR(to_timestamp(created_at / 1000), 'HH24') AS hour,
COUNT(*)::int AS count,
COUNT(*) FILTER (WHERE ai_status = 'clean')::int AS clean,
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned,
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged,
COUNT(*) FILTER (WHERE ai_status = 'error')::int AS error
FROM messages
${filter.where}
GROUP BY hour
ORDER BY hour ASC
`,
filter.params,
);
return rows.map((r) => ({
hour: r.hour as string,
count: Number(r.count ?? 0),
clean: Number(r.clean ?? 0),
warned: Number(r.warned ?? 0),
flagged: Number(r.flagged ?? 0),
error: Number(r.error ?? 0),
}));
}
async getTopViolators(
guildId: string,
channelId?: string,
hours = 24,
limit = 10,
) {
logger.debug({ guildId, channelId, hours, limit }, "Getting top violators");
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
user_id,
MAX(username) AS username,
MAX(avatar_url) AS avatar_url,
COUNT(*)::int AS total_messages,
COUNT(*) FILTER (WHERE ai_status IN ('warn', 'flagged', 'error'))::int AS flagged_count,
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned_count,
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS hard_flagged_count,
COUNT(*) FILTER (WHERE ai_status = 'error')::int AS error_count,
COALESCE(AVG(ai_moderation_score), 0)::real AS violation_score,
MAX(ai_moderation_flags) AS worst_flags,
MAX(created_at) AS last_violation
FROM messages
${filter.where}
AND ai_status IN ('warn', 'flagged', 'error')
GROUP BY user_id
ORDER BY flagged_count DESC
LIMIT $${filter.params.length + 1}
`,
[...filter.params, limit],
);
return rows.map((r) => ({
user_id: r.user_id as string,
username: (r.username as string) ?? "",
avatar_url: (r.avatar_url as string | null) ?? null,
total_messages: Number(r.total_messages ?? 0),
flagged_count: Number(r.flagged_count ?? 0),
warned_count: Number(r.warned_count ?? 0),
violation_score: Number(r.violation_score ?? 0),
worst_flags: (r.worst_flags as string | null)
? (r.worst_flags as string)
.split(",")
.map((s) => s.trim())
.filter(Boolean)
: [],
last_violation: Number(r.last_violation ?? 0),
}));
}
async getUserLeaderboard(
guildId: string,
channelId?: string,
hours = 24,
limit = 10,
) {
logger.debug(
{ guildId, channelId, hours, limit },
"Getting user leaderboard",
);
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
user_id,
MAX(username) AS username,
MAX(avatar_url) AS avatar_url,
COUNT(*)::int AS message_count,
COUNT(*) FILTER (WHERE type = 'edited')::int AS edited_count,
COUNT(*) FILTER (WHERE type = 'deleted')::int AS deleted_count,
COUNT(*) FILTER (WHERE ai_status IN ('warn', 'flagged', 'error'))::int AS flagged_count,
MAX(created_at) AS last_active
FROM messages
${filter.where}
GROUP BY user_id
ORDER BY message_count DESC
LIMIT $${filter.params.length + 1}
`,
[...filter.params, limit],
);
return rows.map((r) => ({
user_id: r.user_id as string,
username: (r.username as string) ?? "",
avatar_url: (r.avatar_url as string | null) ?? null,
message_count: Number(r.message_count ?? 0),
edited_count: Number(r.edited_count ?? 0),
deleted_count: Number(r.deleted_count ?? 0),
flagged_count: Number(r.flagged_count ?? 0),
last_active: Number(r.last_active ?? 0),
}));
}
async getModerationStats(guildId: string, channelId?: string, hours = 24) {
logger.debug({ guildId, channelId, hours }, "Getting moderation stats");
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE ai_status = 'clean')::int AS clean,
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned,
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged,
COUNT(*) FILTER (WHERE ai_status = 'error')::int AS error,
COUNT(*) FILTER (WHERE ai_status = 'pending')::int AS pending,
COALESCE(AVG(ai_moderation_score), 0)::real AS average_score
FROM messages
${filter.where}
`,
filter.params,
);
const row = rows[0] as Record<string, unknown> | undefined;
return {
total: Number(row?.total ?? 0),
clean: Number(row?.clean ?? 0),
warned: Number(row?.warned ?? 0),
flagged: Number(row?.flagged ?? 0),
error: Number(row?.error ?? 0),
pending: Number(row?.pending ?? 0),
average_score: Number(row?.average_score ?? 0),
};
}
async getHeatmap(guildId: string, channelId?: string, hours = 24) {
logger.debug({ guildId, channelId, hours }, "Getting heatmap data");
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
EXTRACT(DOW FROM to_timestamp(created_at / 1000))::int AS day_of_week,
EXTRACT(HOUR FROM to_timestamp(created_at / 1000))::int AS hour,
COUNT(*)::int AS count,
COUNT(*) FILTER (WHERE ai_status = 'clean')::int AS clean,
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned,
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged
FROM messages
${filter.where}
GROUP BY day_of_week, hour
ORDER BY day_of_week, hour
`,
filter.params,
);
return rows.map((r) => ({
dayOfWeek: Number(r.day_of_week ?? 0),
hour: Number(r.hour ?? 0),
count: Number(r.count ?? 0),
clean: Number(r.clean ?? 0),
warned: Number(r.warned ?? 0),
flagged: Number(r.flagged ?? 0),
}));
}
async getTopics(guildId: string, channelId?: string, hours = 24) {
logger.debug({ guildId, channelId, hours }, "Getting topics");
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
try {
const { rows } = await pool.query(
`
WITH word_list AS (
SELECT
LOWER(TRIM(BOTH '.,!?;:\"()[]{}' FROM word)) AS word,
ai_moderation_score
FROM messages,
LATERAL UNNEST(STRING_TO_ARRAY(content, ' ')) AS word
${filter.where}
AND content IS NOT NULL AND content != ''
AND LENGTH(TRIM(BOTH '.,!?;:\"()[]{}' FROM word)) >= 4
AND word !~ '^<.+:\d+>$'
AND word !~ '^\['
AND word !~ '^https?://'
AND word !~ '^discord\.'
AND word !~ '^cdn\.'
)
SELECT
word AS topic,
COUNT(*)::int AS count,
COALESCE(AVG(ai_moderation_score), 0)::real AS score
FROM word_list
WHERE word NOT IN (
'yang','dan','di','ke','dari','dengan','untuk','pada','ini','itu',
'ada','akan','telah','sudah','bisa','dapat','tidak','nggak','enggak',
'gak','gk','ga','aku','saya','kamu','dia','kami','kita','mereka',
'iya','ya','yah','oh','ah','eh','lah','pun','juga','masih',
'saja','hanya','sama','atau','tapi','namun','sedang','sangat',
'begitu','karena','sebab','kalau','jika','maka','lalu','setelah',
'seperti','antara','oleh','sebagai','secara','melalui','dalam',
'the','and','for','are','but','not','you','all','can','has',
'was','were','been','like','just','that','this','with','your',
'from','they','have','what','when','where','which','their',
'about','would','could','should','very','also','than','then',
'mau','lagi','jadi','aja','nya','apa','orang',
'lihat','kak','bro','bang','mas','pack','sih','dong','kok',
'nih','deh','kali','loh','lho','doang',
'gue','lo','lu','gua','elo','ane','wkwk','wkwkwk',
'wkwkwkwk','wkwkwkwkwk','haha','hahaha','hehe','wk','wkwk',
'kalo','buat','udah','jir','kan','tuh','pake','dulu',
'banget','kayak','kya','kyk','klo','karna','soalnya',
'bikin','bilang','makan','minum','tidur','main','pergi','pulang',
'mana','cuma','kah','udh','gitu','gini','gtu','gni',
'mending','wkakak','wakak'
)
AND word NOT LIKE '%.jpg'
AND word NOT LIKE '%.jpeg'
AND word NOT LIKE '%.png'
AND word NOT LIKE '%.gif'
AND word NOT LIKE '%.webp'
AND word NOT LIKE '%.mp4'
AND word NOT LIKE '%.mp3'
AND word NOT LIKE '%.pdf'
AND word NOT LIKE '%.zip'
GROUP BY word
ORDER BY COUNT(*) DESC
LIMIT 10
`,
filter.params,
);
return rows.map((r) => ({
topic: (r.topic as string) ?? "",
count: Number(r.count ?? 0),
score: Number(r.score ?? 0),
}));
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error), guildId },
"getTopics query failed — returning empty",
);
return [];
}
}
// ── New endpoints ─────────────────────────────────────────────────────────
async getModerationActions(
guildId: string,
channelId?: string,
hours = 24,
limit = 20,
) {
logger.debug(
{ guildId, channelId, hours, limit },
"Getting moderation actions",
);
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
ma.id,
ma.message_id,
ma.user_id,
ma.guild_id,
ma.action_type,
ma.reason,
ma.executed_by,
ma.status,
ma.error,
ma.created_at,
ma.executed_at,
m.username,
m.content
FROM moderation_actions ma
LEFT JOIN messages m ON m.id = ma.message_id
${filter.where.replace("guild_id", "ma.guild_id").replace("created_at", "ma.created_at")}
ORDER BY ma.created_at DESC
LIMIT $${filter.params.length + 1}
`,
[...filter.params, limit],
);
return rows.map((r) => ({
id: r.id as string,
message_id: (r.message_id as string) ?? null,
user_id: r.user_id as string,
guild_id: r.guild_id as string,
action_type: r.action_type as string,
reason: (r.reason as string) ?? null,
executed_by: (r.executed_by as string) ?? null,
status: r.status as string,
error: (r.error as string) ?? null,
created_at: Number(r.created_at ?? 0),
executed_at: r.executed_at ? Number(r.executed_at) : null,
username: (r.username as string) ?? "Unknown",
content: (r.content as string) ?? null,
}));
}
async getAIStats(guildId: string, channelId?: string, hours = 24) {
logger.debug({ guildId, channelId, hours }, "Getting AI stats");
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
COUNT(*)::int AS total_analyzed,
COUNT(*) FILTER (WHERE ai_severity = 'none')::int AS severity_none,
COUNT(*) FILTER (WHERE ai_severity = 'low')::int AS severity_low,
COUNT(*) FILTER (WHERE ai_severity = 'medium')::int AS severity_medium,
COUNT(*) FILTER (WHERE ai_severity = 'high')::int AS severity_high,
COUNT(*) FILTER (WHERE ai_severity = 'critical')::int AS severity_critical,
COUNT(*) FILTER (WHERE ai_recommended_action = 'none')::int AS action_none,
COUNT(*) FILTER (WHERE ai_recommended_action = 'monitor')::int AS action_monitor,
COUNT(*) FILTER (WHERE ai_recommended_action = 'warn')::int AS action_warn,
COUNT(*) FILTER (WHERE ai_recommended_action = 'review')::int AS action_review,
COUNT(*) FILTER (WHERE ai_recommended_action = 'delete')::int AS action_delete,
COUNT(*) FILTER (WHERE ai_recommended_action = 'escalate')::int AS action_escalate,
COUNT(*) FILTER (WHERE ai_status = 'error')::int AS analysis_errors,
COUNT(*) FILTER (WHERE ai_status = 'pending')::int AS analysis_pending,
COALESCE(AVG(ai_confidence), 0)::real AS avg_confidence,
COALESCE(AVG(ai_moderation_score), 0)::real AS avg_score
FROM messages
${filter.where}
`,
filter.params,
);
const row = rows[0] as Record<string, unknown> | undefined;
return {
total_analyzed: Number(row?.total_analyzed ?? 0),
severity: {
none: Number(row?.severity_none ?? 0),
low: Number(row?.severity_low ?? 0),
medium: Number(row?.severity_medium ?? 0),
high: Number(row?.severity_high ?? 0),
critical: Number(row?.severity_critical ?? 0),
},
recommended_actions: {
none: Number(row?.action_none ?? 0),
monitor: Number(row?.action_monitor ?? 0),
warn: Number(row?.action_warn ?? 0),
review: Number(row?.action_review ?? 0),
delete: Number(row?.action_delete ?? 0),
escalate: Number(row?.action_escalate ?? 0),
},
analysis_errors: Number(row?.analysis_errors ?? 0),
analysis_pending: Number(row?.analysis_pending ?? 0),
avg_confidence: Number(row?.avg_confidence ?? 0),
avg_score: Number(row?.avg_score ?? 0),
};
}
async getAttachmentStats(guildId: string, channelId?: string, hours = 24) {
logger.debug({ guildId, channelId, hours }, "Getting attachment stats");
const pool = getPool();
const filter = buildTimeFilter(guildId, channelId, hours);
const { rows } = await pool.query(
`
SELECT
COUNT(*)::int AS total_attachments,
COUNT(*) FILTER (WHERE a.upload_status = 'uploaded')::int AS uploaded,
COUNT(*) FILTER (WHERE a.upload_status = 'pending')::int AS pending,
COUNT(*) FILTER (WHERE a.upload_status = 'failed')::int AS failed,
COALESCE(SUM(a.size), 0)::bigint AS total_size_bytes,
COUNT(DISTINCT a.user_id)::int AS unique_uploaders,
(SELECT a2.type FROM attachments a2
WHERE a2.guild_id = $1
${channelId ? "AND a2.channel_id = $" + (filter.paramOffset - 1) : ""}
AND a2.created_at > (EXTRACT(EPOCH FROM NOW()) * 1000 - $${filter.paramOffset})
GROUP BY a2.type ORDER BY COUNT(*) DESC LIMIT 1
) AS top_mime_type
FROM attachments a
WHERE a.guild_id = $1
${channelId ? "AND a.channel_id = $" + (filter.paramOffset - 1) : ""}
AND a.created_at > (EXTRACT(EPOCH FROM NOW()) * 1000 - $${filter.paramOffset})
`,
channelId
? [guildId, channelId, hours * 3_600_000]
: [guildId, hours * 3_600_000],
);
const row = rows[0] as Record<string, unknown> | undefined;
return {
total_attachments: Number(row?.total_attachments ?? 0),
uploaded: Number(row?.uploaded ?? 0),
pending: Number(row?.pending ?? 0),
failed: Number(row?.failed ?? 0),
total_size_bytes: Number(row?.total_size_bytes ?? 0),
unique_uploaders: Number(row?.unique_uploaders ?? 0),
top_mime_type: (row?.top_mime_type as string) ?? null,
};
}
}
export const analyticsRepository = new AnalyticsRepository();
@@ -1,33 +0,0 @@
import type { Router } from "express";
import express from "express";
import {
handleGetAIStats,
handleGetAttachmentStats,
handleGetDailyTrend,
handleGetHeatmap,
handleGetHourlyStats,
handleGetModerationActions,
handleGetModerationStats,
handleGetOverview,
handleGetTopics,
handleGetTopViolators,
handleGetUserLeaderboard,
} from "./analytics.controller.js";
export function createAnalyticsRouter(): Router {
const router = express.Router();
router.get("/analytics/overview", handleGetOverview);
router.get("/analytics/trend", handleGetDailyTrend);
router.get("/analytics/hourly", handleGetHourlyStats);
router.get("/analytics/violators", handleGetTopViolators);
router.get("/analytics/leaderboard", handleGetUserLeaderboard);
router.get("/analytics/stats", handleGetModerationStats);
router.get("/analytics/heatmap", handleGetHeatmap);
router.get("/analytics/topics", handleGetTopics);
router.get("/analytics/moderation-actions", handleGetModerationActions);
router.get("/analytics/ai-stats", handleGetAIStats);
router.get("/analytics/attachment-stats", handleGetAttachmentStats);
return router;
}
@@ -1,9 +0,0 @@
import { z } from "zod";
export const analyticsQuerySchema = z.object({
guildId: z.string(),
channelId: z.string().optional(),
hours: z.coerce.number().int().positive().default(24),
});
export type AnalyticsQuery = z.infer<typeof analyticsQuerySchema>;
@@ -1,127 +0,0 @@
import { ForbiddenError, ValidationError } from "@bete/shared/errors";
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/index.js";
import { analyticsRepository } from "./analytics.repository.js";
import type { AnalyticsQuery } from "./analytics.schema.js";
const logger = createChildLogger("analytics.service");
export class AnalyticsService {
private assertMonitorGuild(guildId: string) {
if (!config.MONITOR_GUILD_ID) {
throw new ValidationError("MONITOR_GUILD_ID is not configured");
}
if (guildId !== config.MONITOR_GUILD_ID) {
throw new ForbiddenError("Analytics are restricted to the monitor guild");
}
}
async getOverview(query: AnalyticsQuery) {
this.assertMonitorGuild(query.guildId);
logger.debug({ query }, "Getting analytics overview");
return analyticsRepository.getOverview(
query.guildId,
query.channelId,
query.hours,
);
}
async getDailyTrend(guildId: string, hours = 24) {
this.assertMonitorGuild(guildId);
logger.debug({ guildId, hours }, "Getting daily trend");
return analyticsRepository.getDailyTrend(guildId, hours);
}
async getHourlyStats(guildId: string, channelId?: string, hours = 24) {
this.assertMonitorGuild(guildId);
logger.debug({ guildId, channelId, hours }, "Getting hourly stats");
return analyticsRepository.getHourlyStats(guildId, channelId, hours);
}
async getTopViolators(
guildId: string,
channelId?: string,
hours = 24,
limit = 10,
) {
this.assertMonitorGuild(guildId);
logger.debug({ guildId, channelId, hours, limit }, "Getting top violators");
return analyticsRepository.getTopViolators(
guildId,
channelId,
hours,
limit,
);
}
async getUserLeaderboard(
guildId: string,
channelId?: string,
hours = 24,
limit = 10,
) {
this.assertMonitorGuild(guildId);
logger.debug(
{ guildId, channelId, hours, limit },
"Getting user leaderboard",
);
return analyticsRepository.getUserLeaderboard(
guildId,
channelId,
hours,
limit,
);
}
async getModerationStats(guildId: string, channelId?: string, hours = 24) {
this.assertMonitorGuild(guildId);
logger.debug({ guildId, channelId, hours }, "Getting moderation stats");
return analyticsRepository.getModerationStats(guildId, channelId, hours);
}
async getHeatmap(guildId: string, channelId?: string, hours = 24) {
this.assertMonitorGuild(guildId);
logger.debug({ guildId, channelId, hours }, "Getting heatmap");
return analyticsRepository.getHeatmap(guildId, channelId, hours);
}
async getTopics(guildId: string, channelId?: string, hours = 24) {
this.assertMonitorGuild(guildId);
logger.debug({ guildId, channelId, hours }, "Getting topics");
return analyticsRepository.getTopics(guildId, channelId, hours);
}
async getModerationActions(
guildId: string,
channelId?: string,
hours = 24,
limit = 20,
) {
this.assertMonitorGuild(guildId);
logger.debug(
{ guildId, channelId, hours, limit },
"Getting moderation actions",
);
return analyticsRepository.getModerationActions(
guildId,
channelId,
hours,
limit,
);
}
async getAIStats(guildId: string, channelId?: string, hours = 24) {
this.assertMonitorGuild(guildId);
logger.debug({ guildId, channelId, hours }, "Getting AI stats");
return analyticsRepository.getAIStats(guildId, channelId, hours);
}
async getAttachmentStats(guildId: string, channelId?: string, hours = 24) {
this.assertMonitorGuild(guildId);
logger.debug({ guildId, channelId, hours }, "Getting attachment stats");
return analyticsRepository.getAttachmentStats(guildId, channelId, hours);
}
}
export const analyticsService = new AnalyticsService();
@@ -1,8 +1,8 @@
import { UnauthorizedError } from "@bete/shared/errors";
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response, Router } from "express"; import type { Request, Response, Router } from "express";
import express from "express"; import express from "express";
import { config } from "../../shared/config/index.js"; import { config } from "../../shared/config/index.js";
import { UnauthorizedError } from "@bete/shared/errors";
import { createChildLogger } from "@bete/shared/logger";
import { asyncHandler } from "../../shared/middlewares/index.js"; import { asyncHandler } from "../../shared/middlewares/index.js";
const logger = createChildLogger("auth.routes"); const logger = createChildLogger("auth.routes");
@@ -1,5 +1,5 @@
import { getPool } from "../../shared/database/index.js";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { getPool } from "../../shared/database/index.js";
const logger = createChildLogger("health.repository"); const logger = createChildLogger("health.repository");
@@ -1,5 +1,5 @@
import type { Request, Response } from "express";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response } from "express";
import { mascotChatService } from "./mascot-chat.service.js"; import { mascotChatService } from "./mascot-chat.service.js";
const logger = createChildLogger("mascot-chat.controller"); const logger = createChildLogger("mascot-chat.controller");
@@ -20,11 +20,15 @@ export async function handleMascotChat(req: Request, res: Response) {
logger.debug( logger.debug(
{ userId, messageLength: message.length, context }, { userId, messageLength: message.length, context },
"Received mascot chat message" "Received mascot chat message",
); );
// Process message & generate response // Process message & generate response
const response = await mascotChatService.processMessage(message, context, userId); const response = await mascotChatService.processMessage(
message,
context,
userId,
);
// Save conversation to database // Save conversation to database
await mascotChatService.saveConversation({ await mascotChatService.saveConversation({
@@ -0,0 +1,180 @@
import { createChildLogger } from "@bete/shared/logger";
import { getPool } from "../../shared/database/index.js";
const logger = createChildLogger("mascot-chat.repository");
export interface MascotChatContext {
messageCount?: number;
activeParticipants?: number;
lastActivity?: string;
topicsDiscussed?: string[];
guildId?: string;
channelId?: string;
}
export interface SaveConversationInput {
userId: string;
userMessage: string;
mascotResponse: string;
context?: MascotChatContext;
timestamp: Date;
}
export interface MascotChatHistoryRow {
id: string;
user_id: string;
user_message: string;
mascot_response: string;
context: MascotChatContext | null;
created_at: string;
}
export interface ServerInsights {
total_messages: number;
active_users: number;
flagged: number;
warned: number;
}
export class MascotChatRepository {
private initialized = false;
async ensureSchema(): Promise<void> {
if (this.initialized) return;
const pool = getPool();
await pool.query(`
CREATE TABLE IF NOT EXISTS mascot_chat_messages (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id TEXT NOT NULL,
user_message TEXT NOT NULL,
mascot_response TEXT NOT NULL,
context JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
`);
await pool.query(`
CREATE INDEX IF NOT EXISTS idx_mascot_chat_messages_user_created
ON mascot_chat_messages (user_id, created_at DESC)
`);
this.initialized = true;
logger.info("Mascot chat schema ready");
}
async saveConversation(input: SaveConversationInput): Promise<void> {
await this.ensureSchema();
const pool = getPool();
await pool.query(
`
INSERT INTO mascot_chat_messages
(user_id, user_message, mascot_response, context, created_at)
VALUES ($1, $2, $3, $4::jsonb, $5)
`,
[
input.userId,
input.userMessage,
input.mascotResponse,
JSON.stringify(input.context ?? {}),
input.timestamp.toISOString(),
],
);
logger.debug({ userId: input.userId }, "Conversation saved");
}
async getChatHistory(
userId: string,
limit: number,
): Promise<MascotChatHistoryRow[]> {
await this.ensureSchema();
const pool = getPool();
const { rows } = await pool.query<MascotChatHistoryRow>(
`
SELECT id, user_id, user_message, mascot_response, context, created_at
FROM mascot_chat_messages
WHERE user_id = $1
ORDER BY created_at DESC
LIMIT $2
`,
[userId, limit],
);
logger.debug({ userId, count: rows.length }, "Chat history fetched");
return rows.reverse();
}
async clearChatHistory(userId: string): Promise<void> {
await this.ensureSchema();
const pool = getPool();
const { rowCount } = await pool.query(
`DELETE FROM mascot_chat_messages WHERE user_id = $1`,
[userId],
);
logger.info({ userId, deletedRows: rowCount ?? 0 }, "Chat history cleared");
}
async getServerInsights(
guildId?: string,
channelId?: string,
): Promise<ServerInsights> {
await this.ensureSchema();
const pool = getPool();
try {
const params: string[] = [];
const clauses: string[] = [];
if (guildId) {
params.push(guildId);
clauses.push(`guild_id = $${params.length}`);
}
if (channelId) {
params.push(channelId);
clauses.push(`channel_id = $${params.length}`);
}
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
const { rows } = await pool.query<ServerInsights>(
`
SELECT
COUNT(*)::int AS total_messages,
COUNT(DISTINCT user_id)::int AS active_users,
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged,
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned
FROM messages
${where}
`,
params,
);
const insights = rows[0] ?? {
total_messages: 0,
active_users: 0,
flagged: 0,
warned: 0,
};
logger.debug({ guildId, channelId, insights }, "Server insights fetched");
return insights;
} catch (error) {
logger.warn(
{ error, guildId, channelId },
"Failed to load server insights",
);
return {
total_messages: 0,
active_users: 0,
flagged: 0,
warned: 0,
};
}
}
}
export const mascotChatRepository = new MascotChatRepository();
@@ -0,0 +1,29 @@
import { z } from "zod";
export const contextSchema = z.object({
messageCount: z.number().int().nonnegative().optional(),
activeParticipants: z.number().int().nonnegative().optional(),
lastActivity: z.string().datetime().optional(),
topicsDiscussed: z.array(z.string()).optional(),
guildId: z.string().optional(),
channelId: z.string().optional(),
});
export const chatRequestSchema = z.object({
message: z.string().min(1, "Message is required"),
context: contextSchema.optional(),
});
export const chatResponseSchema = z.object({
response: z.string(),
timestamp: z.string(),
});
export const chatHistoryQuerySchema = z.object({
limit: z.coerce.number().int().positive().max(100).default(50),
});
export type ChatRequest = z.infer<typeof chatRequestSchema>;
export type ChatResponse = z.infer<typeof chatResponseSchema>;
export type ChatContext = z.infer<typeof contextSchema>;
export type ChatHistoryQuery = z.infer<typeof chatHistoryQuerySchema>;
@@ -1,47 +1,25 @@
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/index.js"; import { config } from "../../shared/config/index.js";
import { getPool } from "../../shared/database/index.js"; import type {
MascotChatContext,
MascotChatHistoryRow,
SaveConversationInput,
} from "./mascot-chat.repository.js";
import { mascotChatRepository } from "./mascot-chat.repository.js";
const logger = createChildLogger("mascot-chat.service"); const logger = createChildLogger("mascot-chat.service");
export interface MascotChatContext {
messageCount?: number;
activeParticipants?: number;
lastActivity?: string;
topicsDiscussed?: string[];
guildId?: string;
channelId?: string;
}
export interface SaveConversationInput {
userId: string;
userMessage: string;
mascotResponse: string;
context?: MascotChatContext;
timestamp: Date;
}
export interface MascotChatHistoryRow {
id: string;
user_id: string;
user_message: string;
mascot_response: string;
context: MascotChatContext | null;
created_at: string;
}
class MascotChatService { class MascotChatService {
private initialized = false;
async processMessage( async processMessage(
message: string, message: string,
context: MascotChatContext | undefined, context: MascotChatContext | undefined,
userId: string, userId: string,
): Promise<string> { ): Promise<string> {
await this.ensureSchema();
const recentContext = await this.getRecentConversationContext(userId); const recentContext = await this.getRecentConversationContext(userId);
const serverInsights = await this.getServerInsights(context); const serverInsights = await mascotChatRepository.getServerInsights(
context?.guildId,
context?.channelId,
);
// Build LLM messages // Build LLM messages
const systemPrompt = this.buildSystemPrompt(serverInsights); const systemPrompt = this.buildSystemPrompt(serverInsights);
@@ -56,142 +34,30 @@ class MascotChatService {
} }
async saveConversation(input: SaveConversationInput): Promise<void> { async saveConversation(input: SaveConversationInput): Promise<void> {
await this.ensureSchema(); await mascotChatRepository.saveConversation(input);
const pool = getPool();
await pool.query(
`
INSERT INTO mascot_chat_messages
(user_id, user_message, mascot_response, context, created_at)
VALUES ($1, $2, $3, $4::jsonb, $5)
`,
[
input.userId,
input.userMessage,
input.mascotResponse,
JSON.stringify(input.context ?? {}),
input.timestamp.toISOString(),
],
);
} }
async getChatHistory( async getChatHistory(
userId: string, userId: string,
limit: number, limit: number,
): Promise<MascotChatHistoryRow[]> { ): Promise<MascotChatHistoryRow[]> {
await this.ensureSchema(); return mascotChatRepository.getChatHistory(userId, limit);
const pool = getPool();
const { rows } = await pool.query<MascotChatHistoryRow>(
`
SELECT id, user_id, user_message, mascot_response, context, created_at
FROM mascot_chat_messages
WHERE user_id = $1
ORDER BY created_at DESC
LIMIT $2
`,
[userId, limit],
);
return rows.reverse();
} }
async clearChatHistory(userId: string): Promise<void> { async clearChatHistory(userId: string): Promise<void> {
await this.ensureSchema(); await mascotChatRepository.clearChatHistory(userId);
const pool = getPool();
await pool.query(`DELETE FROM mascot_chat_messages WHERE user_id = $1`, [
userId,
]);
}
private async ensureSchema(): Promise<void> {
if (this.initialized) return;
const pool = getPool();
await pool.query(`
CREATE TABLE IF NOT EXISTS mascot_chat_messages (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id TEXT NOT NULL,
user_message TEXT NOT NULL,
mascot_response TEXT NOT NULL,
context JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
`);
await pool.query(`
CREATE INDEX IF NOT EXISTS idx_mascot_chat_messages_user_created
ON mascot_chat_messages (user_id, created_at DESC)
`);
this.initialized = true;
logger.info("Mascot chat schema ready");
} }
private async getRecentConversationContext( private async getRecentConversationContext(
userId: string, userId: string,
): Promise<string[]> { ): Promise<string[]> {
const history = await this.getChatHistory(userId, 3); const history = await mascotChatRepository.getChatHistory(userId, 3);
return history.flatMap((row) => [ return history.flatMap((row) => [
`User: ${row.user_message}`, `User: ${row.user_message}`,
`Mascot: ${row.mascot_response}`, `Mascot: ${row.mascot_response}`,
]); ]);
} }
private async getServerInsights(context?: MascotChatContext) {
const pool = getPool();
const guildId = context?.guildId;
const channelId = context?.channelId;
try {
const params: string[] = [];
const clauses: string[] = [];
if (guildId) {
params.push(guildId);
clauses.push(`guild_id = $${params.length}`);
}
if (channelId) {
params.push(channelId);
clauses.push(`channel_id = $${params.length}`);
}
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
const { rows } = await pool.query<{
total_messages: number;
active_users: number;
flagged: number;
warned: number;
}>(
`
SELECT
COUNT(*)::int AS total_messages,
COUNT(DISTINCT user_id)::int AS active_users,
COUNT(*) FILTER (WHERE ai_status = 'flagged')::int AS flagged,
COUNT(*) FILTER (WHERE ai_status = 'warn')::int AS warned
FROM messages
${where}
`,
params,
);
return (
rows[0] ?? {
total_messages: 0,
active_users: 0,
flagged: 0,
warned: 0,
}
);
} catch (error) {
logger.warn({ error }, "Failed to load mascot server insights");
return {
total_messages: context?.messageCount ?? 0,
active_users: context?.activeParticipants ?? 0,
flagged: 0,
warned: 0,
};
}
}
private buildSystemPrompt(insights: { private buildSystemPrompt(insights: {
total_messages: number; total_messages: number;
active_users: number; active_users: number;
@@ -1,8 +1,8 @@
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response, Router } from "express"; import type { Request, Response, Router } from "express";
import express from "express"; import express from "express";
import { createChildLogger } from "@bete/shared/logger";
import { asyncHandler } from "../../shared/middlewares/index.js"; import { asyncHandler } from "../../shared/middlewares/index.js";
import { queue, skip, stop, setVolume, getStatus } from "./media.service.js"; import { getStatus, queue, setVolume, skip, stop } from "./media.service.js";
const logger = createChildLogger("media.routes"); const logger = createChildLogger("media.routes");
@@ -1,9 +1,6 @@
import type { NextFunction, Request, Response } from "express";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { import type { NextFunction, Request, Response } from "express";
asyncHandler, import { asyncHandler, requireParam } from "../../shared/middlewares/index.js";
requireParam,
} from "../../shared/middlewares/index.js";
import { messageQuerySchema } from "./messages.schema.js"; import { messageQuerySchema } from "./messages.schema.js";
import { messagesService } from "./messages.service.js"; import { messagesService } from "./messages.service.js";
@@ -28,7 +25,11 @@ export function handleGetMessagesByChannel(
next: NextFunction, next: NextFunction,
) { ) {
return asyncHandler(async (req: Request, res: Response) => { return asyncHandler(async (req: Request, res: Response) => {
const channelId = requireParam(req.params.channelId, "route parameter", "channelId"); const channelId = requireParam(
req.params.channelId,
"route parameter",
"channelId",
);
const query = messageQuerySchema.parse(req.query); const query = messageQuerySchema.parse(req.query);
logger.debug({ channelId, query }, "Handling get messages by channel"); logger.debug({ channelId, query }, "Handling get messages by channel");
const result = await messagesService.getMessagesByChannel(channelId, query); const result = await messagesService.getMessagesByChannel(channelId, query);
@@ -55,7 +56,11 @@ export function handleGetAttachmentsByChannel(
next: NextFunction, next: NextFunction,
) { ) {
return asyncHandler(async (req: Request, res: Response) => { return asyncHandler(async (req: Request, res: Response) => {
const channelId = requireParam(req.params.channelId, "route parameter", "channelId"); const channelId = requireParam(
req.params.channelId,
"route parameter",
"channelId",
);
const query = messageQuerySchema.parse(req.query); const query = messageQuerySchema.parse(req.query);
logger.debug({ channelId, query }, "Handling get attachments by channel"); logger.debug({ channelId, query }, "Handling get attachments by channel");
const result = await messagesService.getAttachmentsByChannel( const result = await messagesService.getAttachmentsByChannel(
@@ -1,5 +1,5 @@
import { getPool } from "../../shared/database/index.js";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { getPool } from "../../shared/database/index.js";
import type { import type {
MessageCreate, MessageCreate,
MessageQuery, MessageQuery,
@@ -61,7 +61,9 @@ function mapMessageRow(row: Record<string, unknown>) {
} }
export class MessagesRepository { export class MessagesRepository {
async findMany(query: MessageQuery): Promise<PageResult<ReturnType<typeof mapMessageRow>>> { async findMany(
query: MessageQuery,
): Promise<PageResult<ReturnType<typeof mapMessageRow>>> {
const pool = getPool(); const pool = getPool();
const limit = query.limit ?? 50; const limit = query.limit ?? 50;
const clauses: string[] = []; const clauses: string[] = [];
@@ -101,7 +103,8 @@ export class MessagesRepository {
); );
const data = rows.slice(0, limit).map(mapMessageRow); const data = rows.slice(0, limit).map(mapMessageRow);
const nextCursor = rows.length > limit ? String(rows[limit].created_at) : null; const nextCursor =
rows.length > limit ? String(rows[limit].created_at) : null;
logger.debug({ count: data.length, nextCursor }, "Found messages"); logger.debug({ count: data.length, nextCursor }, "Found messages");
return { data, nextCursor }; return { data, nextCursor };
@@ -109,10 +112,9 @@ export class MessagesRepository {
async findById(id: string) { async findById(id: string) {
const pool = getPool(); const pool = getPool();
const { rows } = await pool.query( const { rows } = await pool.query(`SELECT * FROM messages WHERE id = $1`, [
`SELECT * FROM messages WHERE id = $1`, id,
[id], ]);
);
if (rows.length === 0) return null; if (rows.length === 0) return null;
return mapMessageRow(rows[0] as Record<string, unknown>); return mapMessageRow(rows[0] as Record<string, unknown>);
@@ -140,7 +142,8 @@ export class MessagesRepository {
); );
const data = rows.slice(0, limit).map(mapMessageRow); const data = rows.slice(0, limit).map(mapMessageRow);
const nextCursor = rows.length > limit ? String(rows[limit].created_at) : null; const nextCursor =
rows.length > limit ? String(rows[limit].created_at) : null;
return { data, nextCursor }; return { data, nextCursor };
} }
@@ -260,6 +263,58 @@ export class MessagesRepository {
return rowCount ?? 0; return rowCount ?? 0;
} }
/**
* Mark a single message for re-analysis by resetting ai_status to 'pending'.
* Skips messages already in 'pending' state to avoid write amplification.
*/
async markForReanalysis(id: string): Promise<void> {
const pool = getPool();
await pool.query(
`UPDATE messages SET ai_status = 'pending'
WHERE id = $1 AND ai_status != 'pending'`,
[id],
);
logger.debug({ id }, "Message marked for re-analysis");
}
/**
* Retrieve messages flagged for review (ai_status IN ('warn', 'flagged')).
* Optionally filtered by channelId, with configurable limit.
*/
async getReviewMessages(
channelId?: string,
limit: number = 20,
): Promise<Record<string, unknown>[]> {
const pool = getPool();
if (channelId) {
const { rows } = await pool.query(
`SELECT id, guild_id, channel_id, user_id, username, avatar_url,
content, type, created_at, ai_status, ai_severity,
ai_confidence, ai_analysis
FROM messages
WHERE ai_status IN ('warn', 'flagged')
AND channel_id = $1
ORDER BY created_at DESC
LIMIT $2`,
[channelId, limit],
);
return rows;
}
const { rows } = await pool.query(
`SELECT id, guild_id, channel_id, user_id, username, avatar_url,
content, type, created_at, ai_status, ai_severity,
ai_confidence, ai_analysis
FROM messages
WHERE ai_status IN ('warn', 'flagged')
ORDER BY created_at DESC
LIMIT $1`,
[limit],
);
return rows;
}
async delete(id: string): Promise<boolean> { async delete(id: string): Promise<boolean> {
const pool = getPool(); const pool = getPool();
const { rowCount } = await pool.query( const { rowCount } = await pool.query(
@@ -308,7 +363,8 @@ export class MessagesRepository {
uploaded_at: (r.uploaded_at as number | null) ?? null, uploaded_at: (r.uploaded_at as number | null) ?? null,
})); }));
const nextCursor = data.length > limit ? String(data[limit].created_at) : null; const nextCursor =
data.length > limit ? String(data[limit].created_at) : null;
const trimmed = data.slice(0, limit); const trimmed = data.slice(0, limit);
return { data: trimmed, nextCursor }; return { data: trimmed, nextCursor };
@@ -1,7 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response, Router } from "express"; import type { Request, Response, Router } from "express";
import express from "express"; import express from "express";
import { getPool } from "../../shared/database/index.js";
import { createChildLogger } from "@bete/shared/logger";
import { asyncHandler } from "../../shared/middlewares/index.js"; import { asyncHandler } from "../../shared/middlewares/index.js";
import { import {
handleGetAttachmentsByChannel, handleGetAttachmentsByChannel,
@@ -85,7 +84,6 @@ export function createMessagesRouter(): Router {
}), }),
); );
// POST /api/messages/:id/reanalyze - Mark single message for re-analysis // POST /api/messages/:id/reanalyze - Mark single message for re-analysis
router.post( router.post(
"/messages/:id/reanalyze", "/messages/:id/reanalyze",
@@ -104,20 +102,11 @@ export function createMessagesRouter(): Router {
reanalyzeInFlight.add(id); reanalyzeInFlight.add(id);
try { try {
const pool = getPool(); await messagesService.markForReanalysis(id);
await pool.query(
// Only revert to pending if the message is not currently being
// processed (pending) already — prevents write amplification when
// the recovery worker already picked it up between UI clicks.
`UPDATE messages SET ai_status = 'pending'
WHERE id = $1 AND ai_status != 'pending'`,
[id],
);
} finally { } finally {
reanalyzeInFlight.delete(id); reanalyzeInFlight.delete(id);
} }
logger.debug({ id }, "Message marked for re-analysis");
res.status(200).json({ ok: true }); res.status(200).json({ ok: true });
}), }),
); );
@@ -129,36 +118,7 @@ export function createMessagesRouter(): Router {
const limit = Number(req.query.limit) || 20; const limit = Number(req.query.limit) || 20;
const channelId = (req.query.channelId as string) || undefined; const channelId = (req.query.channelId as string) || undefined;
const pool = getPool(); const rows = await messagesService.getReviewMessages(channelId, limit);
let sqlQuery: string;
let params: (string | number)[];
if (channelId) {
sqlQuery = `
SELECT id, guild_id, channel_id, user_id, username, avatar_url,
content, type, created_at, ai_status, ai_severity,
ai_confidence, ai_analysis
FROM messages
WHERE ai_status IN ('warn', 'flagged')
AND channel_id = $1
ORDER BY created_at DESC
LIMIT $2
`;
params = [channelId, limit];
} else {
sqlQuery = `
SELECT id, guild_id, channel_id, user_id, username, avatar_url,
content, type, created_at, ai_status, ai_severity,
ai_confidence, ai_analysis
FROM messages
WHERE ai_status IN ('warn', 'flagged')
ORDER BY created_at DESC
LIMIT $1
`;
params = [limit];
}
const { rows } = await pool.query(sqlQuery, params);
logger.debug({ limit, channelId }, "Review query executed"); logger.debug({ limit, channelId }, "Review query executed");
res.json({ results: rows, limit, cursor: null }); res.json({ results: rows, limit, cursor: null });
}), }),
@@ -168,7 +128,7 @@ export function createMessagesRouter(): Router {
router.post( router.post(
"/messages/:id/moderate", "/messages/:id/moderate",
asyncHandler(async (req: Request, res: Response) => { asyncHandler(async (req: Request, res: Response) => {
const id = req.params.id; const id = String(req.params.id ?? "");
if (!id) { if (!id) {
res.status(400).json({ error: "MISSING_ID" }); res.status(400).json({ error: "MISSING_ID" });
return; return;
@@ -196,20 +156,12 @@ export function createMessagesRouter(): Router {
} }
// Fetch the message to get guild/user context // Fetch the message to get guild/user context
const pool = getPool(); const msg = await messagesService.getMessageById(id).catch(() => null);
const { rows } = await pool.query( if (!msg) {
`SELECT id, guild_id, channel_id, thread_id, user_id, content
FROM messages WHERE id = $1`,
[id],
);
if (rows.length === 0) {
res.status(404).json({ error: "MESSAGE_NOT_FOUND" }); res.status(404).json({ error: "MESSAGE_NOT_FOUND" });
return; return;
} }
const msg = rows[0] as Record<string, unknown>;
// Publish command to DG via Redis // Publish command to DG via Redis
const { publishCommand } = await import("../../ws/redis-bridge.js"); const { publishCommand } = await import("../../ws/redis-bridge.js");
await publishCommand({ await publishCommand({
@@ -217,9 +169,9 @@ export function createMessagesRouter(): Router {
type: "moderation:action", type: "moderation:action",
payload: { payload: {
messageId: id, messageId: id,
guildId: String(msg.guild_id ?? ""), guildId: msg.guild_id,
channelId: (msg.thread_id as string) || String(msg.channel_id ?? ""), channelId: msg.thread_id || msg.channel_id,
userId: String(msg.user_id ?? ""), userId: msg.user_id,
actionType, actionType,
reason: reason ?? "Manual moderation from dashboard", reason: reason ?? "Manual moderation from dashboard",
requestedAt: Date.now(), requestedAt: Date.now(),
@@ -46,12 +46,33 @@ export class MessagesService {
return messagesRepository.getAttachmentsByChannel(channelId, query); return messagesRepository.getAttachmentsByChannel(channelId, query);
} }
async markForReanalysis(id: string): Promise<void> {
if (!id) {
throw new ValidationError("message ID is required");
}
logger.debug({ id }, "Marking message for re-analysis");
await messagesRepository.markForReanalysis(id);
}
async getReviewMessages(
channelId?: string,
limit?: number,
): Promise<Record<string, unknown>[]> {
logger.debug({ channelId, limit }, "Getting review messages");
return messagesRepository.getReviewMessages(channelId, limit);
}
async reanalyzeErrorBatch(opts: { async reanalyzeErrorBatch(opts: {
guildId?: string; guildId?: string;
channelId?: string; channelId?: string;
messageIds?: string[]; messageIds?: string[];
}) { }) {
if (!opts.guildId && !opts.channelId && (!opts.messageIds || opts.messageIds.length === 0)) { if (
!opts.guildId &&
!opts.channelId &&
(!opts.messageIds || opts.messageIds.length === 0)
) {
throw new ValidationError( throw new ValidationError(
"At least one of guildId, channelId, or messageIds[] is required", "At least one of guildId, channelId, or messageIds[] is required",
); );
@@ -1,6 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response, Router } from "express"; import type { Request, Response, Router } from "express";
import express from "express"; import express from "express";
import { createChildLogger } from "@bete/shared/logger";
import { asyncHandler } from "../../shared/middlewares/index.js"; import { asyncHandler } from "../../shared/middlewares/index.js";
import { recordingsService } from "./recordings.service.js"; import { recordingsService } from "./recordings.service.js";
@@ -1,6 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import { sql } from "drizzle-orm"; import { sql } from "drizzle-orm";
import { getDatabase } from "../../shared/database/index.js"; import { getDatabase } from "../../shared/database/index.js";
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("recordings.service"); const logger = createChildLogger("recordings.service");
@@ -1,6 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import type { Request, Response, Router } from "express"; import type { Request, Response, Router } from "express";
import express from "express"; import express from "express";
import { createChildLogger } from "@bete/shared/logger";
import { asyncHandler } from "../../shared/middlewares/index.js"; import { asyncHandler } from "../../shared/middlewares/index.js";
import { uiStateService } from "./ui-state.service.js"; import { uiStateService } from "./ui-state.service.js";
@@ -1,6 +1,6 @@
import { createChildLogger } from "@bete/shared/logger";
import { sql } from "drizzle-orm"; import { sql } from "drizzle-orm";
import { getDatabase } from "../../shared/database/index.js"; import { getDatabase } from "../../shared/database/index.js";
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("ui-state.service"); const logger = createChildLogger("ui-state.service");
@@ -0,0 +1,39 @@
import { z } from "zod";
export const voiceCommandSchema = z.object({
command: z.string().min(1, "command is required"),
});
export const connectVoiceSchema = z.object({
guildId: z.string().min(1, "guildId is required"),
channelId: z.string().min(1, "channelId is required"),
});
export const guildIdParamSchema = z.object({
guildId: z.string().min(1),
});
export const guildSchema = z.object({
id: z.string(),
name: z.string(),
icon: z.string().nullable(),
});
export const channelSchema = z.object({
id: z.string(),
name: z.string(),
type: z.enum(["voice", "text"]),
});
export const voiceStatusSchema = z.object({
connected: z.boolean(),
activeGuildId: z.string().nullable(),
activeChannelId: z.string().nullable(),
activeChannelName: z.string().nullable(),
});
export type VoiceCommand = z.infer<typeof voiceCommandSchema>;
export type ConnectVoice = z.infer<typeof connectVoiceSchema>;
export type Guild = z.infer<typeof guildSchema>;
export type Channel = z.infer<typeof channelSchema>;
export type VoiceStatus = z.infer<typeof voiceStatusSchema>;
@@ -1,9 +1,6 @@
import { getPool } from "../../shared/database/index.js";
import {
publishCommand,
readRedisStatus,
} from "../../shared/redis/index.js";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { getPool } from "../../shared/database/index.js";
import { publishCommand, readRedisStatus } from "../../shared/redis/index.js";
const logger = createChildLogger("voice.service"); const logger = createChildLogger("voice.service");
@@ -32,8 +29,7 @@ export interface VoiceStatus {
*/ */
export async function getGuilds(): Promise<Guild[]> { export async function getGuilds(): Promise<Guild[]> {
const reply = await publishCommand<Guild[]>("guilds:list", {}); const reply = await publishCommand<Guild[]>("guilds:list", {});
if (reply?.success && reply.data && reply.data.length > 0) if (reply?.success && reply.data && reply.data.length > 0) return reply.data;
return reply.data;
// Fallback: Postgres with synthetic names // Fallback: Postgres with synthetic names
logger.warn( logger.warn(
@@ -59,8 +55,7 @@ export async function getTextChannels(guildId: string): Promise<Channel[]> {
const reply = await publishCommand<Channel[]>("guilds:text-channels", { const reply = await publishCommand<Channel[]>("guilds:text-channels", {
guildId, guildId,
}); });
if (reply?.success && reply.data && reply.data.length > 0) if (reply?.success && reply.data && reply.data.length > 0) return reply.data;
return reply.data;
// Fallback: Postgres with synthetic names // Fallback: Postgres with synthetic names
logger.warn( logger.warn(
@@ -117,12 +112,14 @@ export async function connectVoice(
// Fallback: read from Redis status key // Fallback: read from Redis status key
const cached = await readRedisStatus("voice:status"); const cached = await readRedisStatus("voice:status");
return (cached as unknown as VoiceStatus) ?? { return (
(cached as unknown as VoiceStatus) ?? {
connected: false, connected: false,
activeGuildId: null, activeGuildId: null,
activeChannelId: null, activeChannelId: null,
activeChannelName: null, activeChannelName: null,
}; }
);
} }
/** /**
@@ -133,10 +130,12 @@ export async function disconnectVoice(): Promise<VoiceStatus> {
if (reply?.success && reply.data) return reply.data; if (reply?.success && reply.data) return reply.data;
const cached = await readRedisStatus("voice:status"); const cached = await readRedisStatus("voice:status");
return (cached as unknown as VoiceStatus) ?? { return (
(cached as unknown as VoiceStatus) ?? {
connected: false, connected: false,
activeGuildId: null, activeGuildId: null,
activeChannelId: null, activeChannelId: null,
activeChannelName: null, activeChannelName: null,
}; }
);
} }
@@ -1,7 +1,7 @@
import { createChildLogger } from "@bete/shared/logger";
import { drizzle } from "drizzle-orm/node-postgres"; import { drizzle } from "drizzle-orm/node-postgres";
import { Pool } from "pg"; import { Pool } from "pg";
import { config } from "../config/index.js"; import { config } from "../config/index.js";
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("database"); const logger = createChildLogger("database");
@@ -1,10 +1,10 @@
import type { NextFunction, Request, Response } from "express";
import { import {
AppError, AppError,
UnauthorizedError, UnauthorizedError,
ValidationError, ValidationError,
} from "@bete/shared/errors"; } from "@bete/shared/errors";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import type { NextFunction, Request, Response } from "express";
const logger = createChildLogger("middleware"); const logger = createChildLogger("middleware");
@@ -54,7 +54,11 @@ export function asyncHandler(
* Validate that a value is a non-empty string, or throw a descriptive error. * Validate that a value is a non-empty string, or throw a descriptive error.
* Use for both route params and query string values. * Use for both route params and query string values.
*/ */
export function requireParam(value: unknown, kind: string, name: string): string { export function requireParam(
value: unknown,
kind: string,
name: string,
): string {
if (typeof value !== "string" || value.length === 0) { if (typeof value !== "string" || value.length === 0) {
throw new Error(`Missing ${kind}: ${name}`); throw new Error(`Missing ${kind}: ${name}`);
} }
+36 -11
View File
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto"; import { randomUUID } from "node:crypto";
import { createChildLogger } from "@bete/shared/logger";
import Redis from "ioredis"; import Redis from "ioredis";
import { config } from "../config/index.js"; import { config } from "../config/index.js";
import { createChildLogger } from "@bete/shared/logger";
const logger = createChildLogger("redis.command-channel"); const logger = createChildLogger("redis.command-channel");
@@ -73,13 +73,21 @@ export async function publishCommand<T = unknown>(
timeoutMs = 5000, timeoutMs = 5000,
): Promise<CommandReply<T> | null> { ): Promise<CommandReply<T> | null> {
if (!ensureRedisConfig()) { if (!ensureRedisConfig()) {
logger.warn({ commandType }, "Redis not configured, skipping command publish"); logger.warn(
{ commandType },
"Redis not configured, skipping command publish",
);
return null; return null;
} }
const id = randomUUID(); const id = randomUUID();
const replyChannel = `backend:command:reply:${id}`; const replyChannel = `backend:command:reply:${id}`;
const command: CommandMessage = { id, type: commandType, payload, replyChannel }; const command: CommandMessage = {
id,
type: commandType,
payload,
replyChannel,
};
return new Promise<CommandReply<T> | null>((resolve) => { return new Promise<CommandReply<T> | null>((resolve) => {
const pub = getPublisher(); const pub = getPublisher();
@@ -88,7 +96,9 @@ export async function publishCommand<T = unknown>(
const timer = setTimeout(() => { const timer = setTimeout(() => {
if (settled) return; if (settled) return;
settled = true; settled = true;
sub.unsubscribe(replyChannel).catch(() => {/* ignore */}); sub.unsubscribe(replyChannel).catch(() => {
/* ignore */
});
logger.warn({ id, commandType }, "Command timed out waiting for reply"); logger.warn({ id, commandType }, "Command timed out waiting for reply");
resolve(null); resolve(null);
}, timeoutMs); }, timeoutMs);
@@ -99,11 +109,16 @@ export async function publishCommand<T = unknown>(
if (channel !== replyChannel || settled) return; if (channel !== replyChannel || settled) return;
settled = true; settled = true;
clearTimeout(timer); clearTimeout(timer);
sub.unsubscribe(replyChannel).catch(() => {/* ignore */}); sub.unsubscribe(replyChannel).catch(() => {
/* ignore */
});
try { try {
const reply: CommandReply<T> = JSON.parse(message); const reply: CommandReply<T> = JSON.parse(message);
logger.debug({ id, commandType, success: reply.success }, "Command reply received"); logger.debug(
{ id, commandType, success: reply.success },
"Command reply received",
);
resolve(reply); resolve(reply);
} catch (err) { } catch (err) {
logger.error({ id, err }, "Failed to parse command reply"); logger.error({ id, err }, "Failed to parse command reply");
@@ -113,7 +128,9 @@ export async function publishCommand<T = unknown>(
sub.on("message", onMessage); sub.on("message", onMessage);
sub.subscribe(replyChannel).then(() => { sub
.subscribe(replyChannel)
.then(() => {
pub pub
.publish("backend:command", JSON.stringify(command)) .publish("backend:command", JSON.stringify(command))
.then(() => { .then(() => {
@@ -123,12 +140,15 @@ export async function publishCommand<T = unknown>(
if (!settled) { if (!settled) {
settled = true; settled = true;
clearTimeout(timer); clearTimeout(timer);
sub.unsubscribe(replyChannel).catch(() => {/* ignore */}); sub.unsubscribe(replyChannel).catch(() => {
/* ignore */
});
logger.error({ err }, "Failed to publish command"); logger.error({ err }, "Failed to publish command");
resolve(null); resolve(null);
} }
}); });
}).catch((err: Error) => { })
.catch((err: Error) => {
if (!settled) { if (!settled) {
settled = true; settled = true;
clearTimeout(timer); clearTimeout(timer);
@@ -147,7 +167,10 @@ export async function publishCommandNoReply(
payload: Record<string, unknown> = {}, payload: Record<string, unknown> = {},
): Promise<void> { ): Promise<void> {
if (!ensureRedisConfig()) { if (!ensureRedisConfig()) {
logger.warn({ commandType }, "Redis not configured, skipping command publish"); logger.warn(
{ commandType },
"Redis not configured, skipping command publish",
);
return; return;
} }
@@ -213,7 +236,9 @@ export function subscribe(
// Status helpers — read keys set by discord-gateway // Status helpers — read keys set by discord-gateway
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
export async function readRedisStatus(key: string): Promise<Record<string, unknown> | null> { export async function readRedisStatus(
key: string,
): Promise<Record<string, unknown> | null> {
if (!ensureRedisConfig()) { if (!ensureRedisConfig()) {
return null; return null;
} }
+1 -1
View File
@@ -1,7 +1,7 @@
import { createChildLogger } from "@bete/shared/logger";
import Redis from "ioredis"; import Redis from "ioredis";
import { config } from "../shared/config/index.js"; import { config } from "../shared/config/index.js";
import { getCommandPublisher } from "../shared/redis/index.js"; import { getCommandPublisher } from "../shared/redis/index.js";
import { createChildLogger } from "@bete/shared/logger";
import { broadcastRaw } from "./broadcast.js"; import { broadcastRaw } from "./broadcast.js";
const logger = createChildLogger("ws.redis-bridge"); const logger = createChildLogger("ws.redis-bridge");
+40 -17
View File
@@ -66,38 +66,61 @@ export function createWebSocketServer(server: Server): WebSocketServer {
ws.on("message", (data: Buffer) => { ws.on("message", (data: Buffer) => {
// Handle JSON messages from browser // Handle JSON messages from browser
if (typeof data === 'string' || (Buffer.isBuffer(data) && data.length > 0 && data[0] === 0x7B)) { if (
typeof data === "string" ||
(Buffer.isBuffer(data) && data.length > 0 && data[0] === 0x7b)
) {
try { try {
const message = JSON.parse(data.toString()); const message = JSON.parse(data.toString());
if (message.type === 'voice_transmit' && message.buffer) { if (message.type === "voice_transmit" && message.buffer) {
// Forward PCM data to Redis for discord-gateway // Forward PCM data to Redis for discord-gateway
import('../shared/redis/index.js').then(({ getCommandPublisher }) => { import("../shared/redis/index.js").then(
({ getCommandPublisher }) => {
const publisher = getCommandPublisher(); const publisher = getCommandPublisher();
publisher.publish('backend:voice:transmit', JSON.stringify({ publisher
type: 'pcm', .publish(
buffer: message.buffer "backend:voice:transmit",
})).catch((err: Error) => { JSON.stringify({
logger.error({ err }, 'Failed to publish voice transmit to Redis'); type: "pcm",
buffer: message.buffer,
}),
)
.catch((err: Error) => {
logger.error(
{ err },
"Failed to publish voice transmit to Redis",
);
}); });
}); },
} else if (message.type === 'voice_command' && message.command) { );
} else if (message.type === "voice_command" && message.command) {
// Forward voice commands to discord-gateway // Forward voice commands to discord-gateway
import('../shared/redis/index.js').then(({ getCommandPublisher }) => { import("../shared/redis/index.js").then(
({ getCommandPublisher }) => {
const publisher = getCommandPublisher(); const publisher = getCommandPublisher();
const commandId = `cmd-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`; const commandId = `cmd-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
publisher.publish('backend:command', JSON.stringify({ publisher
.publish(
"backend:command",
JSON.stringify({
id: commandId, id: commandId,
type: message.command, type: message.command,
payload: {}, payload: {},
replyChannel: `reply:${commandId}` replyChannel: `reply:${commandId}`,
})).catch((err: Error) => { }),
logger.error({ err }, 'Failed to publish voice command to Redis'); )
}); .catch((err: Error) => {
logger.error(
{ err },
"Failed to publish voice command to Redis",
);
}); });
},
);
} }
} catch (err) { } catch (err) {
logger.debug({ err }, 'Failed to parse WebSocket message as JSON'); logger.debug({ err }, "Failed to parse WebSocket message as JSON");
} }
} }
}); });
+1 -1
View File
@@ -260,7 +260,7 @@ On SIGINT/SIGTERM/uncaughtException/unhandledRejection:
### Shared Infrastructure (9 files) ### Shared Infrastructure (9 files)
- `src/shared/config/config.ts` - `src/shared/config/config.ts`
- `src/shared/database/` (5 files) - `src/shared/database/` (5 files)
- `src/shared/errors/errors.ts` - `@bete/shared/errors` (shared package)
- `src/shared/logger/logger.ts` - `src/shared/logger/logger.ts`
- `src/shared/logger/serialization.ts` - `src/shared/logger/serialization.ts`
- `src/shared/utils/retry.ts` - `src/shared/utils/retry.ts`
@@ -1,3 +1,4 @@
import { ConfigError, DatabaseError } from "@bete/shared/errors";
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { Client } from "discord.js-selfbot-v13"; import { Client } from "discord.js-selfbot-v13";
import { startPendingAIAnalysisWorker } from "../modules/ai-moderation/aiAnalyzer.js"; import { startPendingAIAnalysisWorker } from "../modules/ai-moderation/aiAnalyzer.js";
@@ -19,7 +20,6 @@ import {
} from "../shared/database/drizzle.js"; } from "../shared/database/drizzle.js";
import { runMigrations } from "../shared/database/migrate.js"; import { runMigrations } from "../shared/database/migrate.js";
import { createDiscordClientOptions } from "../shared/discord/clientOptions.js"; import { createDiscordClientOptions } from "../shared/discord/clientOptions.js";
import { ConfigError, DatabaseError } from "../shared/errors/errors.js";
import { createGracefulShutdown } from "./shutdown.js"; import { createGracefulShutdown } from "./shutdown.js";
const logger = createChildLogger("discord-gateway"); const logger = createChildLogger("discord-gateway");
-1
View File
@@ -1,4 +1,3 @@
import "./mock-crc.js";
import "libsodium-wrappers"; import "libsodium-wrappers";
import "@snazzah/davey"; import "@snazzah/davey";
import "dotenv/config"; import "dotenv/config";
@@ -7,7 +7,6 @@ import { LRUCache } from "lru-cache";
import { Piscina } from "piscina"; import { Piscina } from "piscina";
import { config } from "../../shared/config/config.js"; import { config } from "../../shared/config/config.js";
import type { EventBroadcaster } from "../event-broadcaster/index.js"; import type { EventBroadcaster } from "../event-broadcaster/index.js";
import { invalidateAnalyticsCache } from "../message-capture/analyticsStore.js";
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js"; import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
import { import {
getConversationKeysWithIncompleteAnalysis, getConversationKeysWithIncompleteAnalysis,
@@ -500,7 +499,6 @@ async function processIndividualFallback(
const rows = await updateMessagesAIAnalysisBulk(updates); const rows = await updateMessagesAIAnalysisBulk(updates);
for (const row of rows) { for (const row of rows) {
broadcastAnalysisCompleted(row); broadcastAnalysisCompleted(row);
invalidateAnalyticsCache(row.guild_id);
scheduleAutoDelete(row); scheduleAutoDelete(row);
// Update reputation autonomously (Belajar & Kebijaksanaan) // Update reputation autonomously (Belajar & Kebijaksanaan)
@@ -1,8 +1,8 @@
import { eq } from "drizzle-orm"; import { eq } from "drizzle-orm";
import { getDatabase } from "../../shared/database/drizzle.js"; import { getDatabase } from "../../shared/database/drizzle.js";
import { import {
channelCulturesTable,
ChannelCulture, ChannelCulture,
channelCulturesTable,
} from "../../shared/database/schema.js"; } from "../../shared/database/schema.js";
/** /**
@@ -1,13 +1,13 @@
import { eq, desc, sql, and } from "drizzle-orm"; import { createChildLogger } from "@bete/shared/logger";
import { and, desc, eq, sql } from "drizzle-orm";
import { config } from "../../shared/config/config.js";
import { getDatabase } from "../../shared/database/drizzle.js"; import { getDatabase } from "../../shared/database/drizzle.js";
import { import {
messagesTable,
channelCulturesTable, channelCulturesTable,
messagesTable,
} from "../../shared/database/schema.js"; } from "../../shared/database/schema.js";
import { config } from "../../shared/config/config.js";
import { createChildLogger } from "@bete/shared/logger";
import { llmChat } from "./llmClient.js";
import { updateChannelCulture } from "./channelCultureStore.js"; import { updateChannelCulture } from "./channelCultureStore.js";
import { llmChat } from "./llmClient.js";
const CULTURE_LEARNING_INTERVAL = 1000 * 60 * 60 * 12; // 12 hours const CULTURE_LEARNING_INTERVAL = 1000 * 60 * 60 * 12; // 12 hours
const log = createChildLogger("cultureLearner"); const log = createChildLogger("cultureLearner");
@@ -84,14 +84,14 @@ export async function llmChat(
signal, signal,
} = opts; } = opts;
const params: any = { const params = {
model, model,
messages, messages,
}; ...(stream !== undefined ? { stream } : {}),
} as OpenAI.Chat.Completions.ChatCompletionCreateParams;
// Attach optional parameters only if explicitly provided to maintain // Attach optional parameters only if explicitly provided to maintain
// maximum compatibility with various LLM providers and local APIs. // maximum compatibility with various LLM providers and local APIs.
if (stream !== undefined) params.stream = stream;
if (temperature !== undefined) params.temperature = temperature; if (temperature !== undefined) params.temperature = temperature;
if (top_p !== undefined) params.top_p = top_p; if (top_p !== undefined) params.top_p = top_p;
if (max_tokens !== undefined) params.max_tokens = max_tokens; if (max_tokens !== undefined) params.max_tokens = max_tokens;
@@ -103,7 +103,9 @@ export async function llmChat(
return retryWithBackoff( return retryWithBackoff(
async () => { async () => {
return withLlmConcurrency(async () => { return withLlmConcurrency(async () => {
const execute = async (currentParams: any) => { const execute = async (
currentParams: OpenAI.Chat.Completions.ChatCompletionCreateParams,
) => {
const response = await client.chat.completions.create(currentParams, { const response = await client.chat.completions.create(currentParams, {
signal, signal,
}); });
@@ -161,7 +163,9 @@ export async function llmChat(
{ model }, { model },
"Provider rejected non-streaming request. Fallback to stream: true initiated.", "Provider rejected non-streaming request. Fallback to stream: true initiated.",
); );
params.stream = true; (
params as unknown as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming
).stream = true;
return await execute(params); return await execute(params);
} }
@@ -11,11 +11,9 @@ import type {
AttachmentRecord, AttachmentRecord,
MessageRecord, MessageRecord,
} from "../message-capture/types.js"; } from "../message-capture/types.js";
import { getChannelCulture } from "./channelCultureStore.js";
import { llmChat, llmVision } from "./llmClient.js"; import { llmChat, llmVision } from "./llmClient.js";
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js"; import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
import { initializeUserReputation } from "./userReputationStore.js";
import { getChannelCulture } from "./channelCultureStore.js";
import { logModerationAnalysis, logModerationError } from "./responseLogger.js"; import { logModerationAnalysis, logModerationError } from "./responseLogger.js";
import { import {
getStickerFromCache, getStickerFromCache,
@@ -46,6 +44,7 @@ import {
upsertCachedMediaByPhash, upsertCachedMediaByPhash,
} from "./textCacheStore.js"; } from "./textCacheStore.js";
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js"; import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
import { initializeUserReputation } from "./userReputationStore.js";
const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]); const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]);
const RecommendedActionSchema = z.enum([ const RecommendedActionSchema = z.enum([
@@ -170,7 +169,7 @@ function deriveRecommendedAction(
/** /**
* Helper to extract JSON from a potentially conversational or markdown-wrapped string. * Helper to extract JSON from a potentially conversational or markdown-wrapped string.
*/ */
export function extractJson(content: string): any { export function extractJson(content: string): unknown {
const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g; const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g;
const matches = content.matchAll(codeBlockRegex); const matches = content.matchAll(codeBlockRegex);
for (const match of matches) { for (const match of matches) {
@@ -1046,7 +1045,7 @@ async function runTextOnlyBatch(
if (rawContent.length > 0 && rawContent.length < 20) { if (rawContent.length > 0 && rawContent.length < 20) {
const groupKey = rawContent.toLowerCase(); const groupKey = rawContent.toLowerCase();
if (shortContentGroups.has(groupKey)) { if (shortContentGroups.has(groupKey)) {
shortContentGroups.get(groupKey)!.push(msg); shortContentGroups.get(groupKey)?.push(msg);
} else { } else {
shortContentGroups.set(groupKey, [msg]); shortContentGroups.set(groupKey, [msg]);
deduplicatedTargets.push(msg); // first occurrence = representative deduplicatedTargets.push(msg); // first occurrence = representative
@@ -1278,9 +1277,6 @@ async function prepareMediaMessage(
const webTextMap = new Map<string, string[]>(); const webTextMap = new Map<string, string[]>();
const mediaAnalysisMap = new Map<string, string[]>(); const mediaAnalysisMap = new Map<string, string[]>();
const getAttachmentImageUrl = (att: AttachmentRecord): string | null =>
att.uploaded_url ?? att.discord_url ?? null;
const maxDimension = config.AI_LLM_IMAGE_MAX_DIMENSION ?? 1024; const maxDimension = config.AI_LLM_IMAGE_MAX_DIMENSION ?? 1024;
const content = getAnalysisContent(target); const content = getAnalysisContent(target);
@@ -1292,93 +1288,14 @@ async function prepareMediaMessage(
.filter( .filter(
(att) => (att) =>
att.message_id === targetId && att.message_id === targetId &&
getAttachmentImageUrl(att) && (att.uploaded_url ?? att.discord_url ?? null) &&
att.type.startsWith("image/"), att.type.startsWith("image/"),
) )
.slice(0, 8); .slice(0, 8);
for (const att of msgAttachments) { for (const att of msgAttachments) {
downloadPromises.push( downloadPromises.push(
(async () => { downloadSingleAttachment(att, targetId, maxDimension, imageMap),
const urlToUse = getAttachmentImageUrl(att);
if (!urlToUse) {
log.warn(
{ attachmentId: att.id, messageId: att.message_id },
"Skipping attachment: no uploaded URL available",
);
return;
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 15000);
try {
const res = await fetch(urlToUse, { signal: controller.signal });
if (!res.ok || !res.body) {
log.warn(
{ attachmentId: att.id, url: urlToUse, status: res.status },
"Failed to download attachment: HTTP error or no body",
);
return;
}
let totalBytes = 0;
const chunks: Uint8Array[] = [];
const reader = res.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) {
totalBytes += value.length;
if (totalBytes > 10 * 1024 * 1024) {
log.warn(
{ attachmentId: att.id, totalBytes },
"Attachment too large (>10MB) — skipping",
);
reader.cancel();
return;
}
chunks.push(value);
}
}
const imageBytes = Buffer.concat(chunks);
const sniffedMime = sniffImageMimeType(imageBytes);
if (!sniffedMime) {
log.warn(
{ attachmentId: att.id },
"Skipping attachment: not a recognised image format",
);
return;
}
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(imageBytes, maxDimension);
const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`;
const part: MessageImagePart = {
type: "image_url",
image_url: { url: dataUrl },
sourceLabel: `[gambar di atas adalah attachment ${att.filename} dari pesan id=${att.message_id}]`,
};
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
} catch (err) {
log.warn(
{
attachmentId: att.id,
error: err instanceof Error ? err.message : String(err),
},
"Error downloading attachment",
);
} finally {
clearTimeout(timeoutId);
}
})(),
); );
} }
@@ -1388,186 +1305,23 @@ async function prepareMediaMessage(
for (const url of urls) { for (const url of urls) {
downloadPromises.push( downloadPromises.push(
(async () => { fetchUrlInline(url, targetId, maxDimension, imageMap, urlWebTexts),
const result = await fetchUrlSafely(url);
if (result.type === "image" && result.data && result.mimeType) {
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(result.data, maxDimension);
const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`;
const part: MessageImagePart = {
type: "image_url",
image_url: { url: dataUrl },
sourceLabel: `[gambar di atas berasal dari link ${url} pada pesan id=${targetId}]`,
};
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
} else if (result.type === "text" && result.textContent) {
urlWebTexts.push(`[Isi Web dari ${url}]: ${result.textContent}`);
}
})(),
); );
} }
// ── Sticker / embed / custom emoji download promises ── // ── Sticker / embed / custom emoji download promises ──
const mediaEvidence = extractMessageMediaEvidence(target.metadata); const mediaEvidence = extractMessageMediaEvidence(target.metadata);
const mediaCandidates: Array<{ const mediaCandidates = buildMediaCandidates(targetId, mediaEvidence);
messageId: string;
url: string;
label: string;
stickerName?: string;
customEmojiId?: string;
customEmojiName?: string;
}> = [
...mediaEvidence.stickers
.filter((s) => s.url)
.map((s) => ({
messageId: targetId,
url: s.url,
label: `[gambar di atas adalah sticker "${s.name}" dari pesan id=${targetId}]`,
stickerName: s.name,
})),
...mediaEvidence.embeds.flatMap((embed) =>
[
embed.image
? {
messageId: targetId,
url: embed.image,
label: `[gambar di atas berasal dari embed image pada pesan id=${targetId}]`,
}
: null,
embed.thumbnail
? {
messageId: targetId,
url: embed.thumbnail,
label: `[gambar di atas berasal dari embed thumbnail pada pesan id=${targetId}]`,
}
: null,
].filter(
(
c,
): c is {
messageId: string;
url: string;
label: string;
stickerName?: string;
customEmojiId?: string;
customEmojiName?: string;
} => c !== null,
),
),
...mediaEvidence.customEmojis.map((emoji) => ({
messageId: targetId,
url: emoji.url,
label: `[gambar di atas adalah custom emoji "${emoji.name}" dari pesan id=${targetId}]`,
customEmojiId: emoji.id,
customEmojiName: emoji.name,
})),
];
for (const candidate of mediaCandidates) { for (const candidate of mediaCandidates) {
downloadPromises.push( downloadPromises.push(
(async () => { downloadMediaCandidate(
if ((imageMap.get(targetId)?.length ?? 0) >= 8) return; candidate,
targetId,
if (candidate.customEmojiId || candidate.stickerName) { maxDimension,
const visionCacheKey = candidate.customEmojiId imageMap,
? makeCustomEmojiCacheKey(candidate.customEmojiId) mediaAnalysisMap,
: makeStickerCacheKey(candidate.stickerName!); ),
const cachedVision = await getCachedMediaAnalysis(visionCacheKey);
if (cachedVision) {
log.debug(
{ cacheKey: visionCacheKey },
"Vision cache HIT for media candidate — skipped download",
);
const analysisText = `[Media analysis for message ${candidate.messageId}] ${candidate.label}: ${cachedVision}`;
const existing = mediaAnalysisMap.get(targetId) ?? [];
existing.push(analysisText);
mediaAnalysisMap.set(targetId, existing);
return;
}
}
if (candidate.stickerName && isStickerCacheReady()) {
try {
const cached = await getStickerFromCache(candidate.stickerName);
if (cached && cached.imageUrl) {
const part: MessageImagePart = {
type: "image_url",
image_url: { url: cached.imageUrl },
sourceLabel: candidate.label,
stickerName: candidate.stickerName,
};
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
return;
}
} catch (stickerErr) {
log.warn(
{
stickerName: candidate.stickerName,
error:
stickerErr instanceof Error
? stickerErr.message
: String(stickerErr),
},
"Sticker cache lookup failed — falling through to network fetch",
);
}
}
const result = await fetchUrlSafely(candidate.url);
if (result.type !== "image" || !result.data || !result.mimeType) {
log.warn(
{
url: candidate.url,
resultType: result.type,
resultHasData: !!result.data,
messageId: candidate.messageId,
label: candidate.stickerName
? `sticker:${candidate.stickerName}`
: candidate.customEmojiName
? `emoji:${candidate.customEmojiName}`
: "embed/other",
},
"Media candidate fetch did not return a usable image — skipping",
);
return;
}
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(result.data, maxDimension);
const base64 = resizedBuffer.toString("base64");
if (candidate.stickerName) {
uploadAndCacheSticker(
candidate.stickerName,
resizedBuffer,
resizedMime,
).catch(() => {});
}
const part: MessageImagePart = {
type: "image_url",
image_url: { url: `data:${resizedMime};base64,${base64}` },
sourceLabel: candidate.label,
stickerName: candidate.stickerName,
customEmojiId: candidate.customEmojiId,
customEmojiName: candidate.customEmojiName,
};
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
})(),
); );
} }
@@ -2132,3 +1886,275 @@ Kategori: spam`;
: [], : [],
}; };
} }
// ---------------------------------------------------------------------------
// Refactored helpers for prepareMediaMessage (extracted to reduce CC)
// ---------------------------------------------------------------------------
async function downloadSingleAttachment(
att: AttachmentRecord,
targetId: string,
maxDimension: number,
imageMap: Map<string, MessageImagePart[]>,
): Promise<void> {
const urlToUse = att.uploaded_url ?? att.discord_url ?? null;
if (!urlToUse) {
log.warn(
{ attachmentId: att.id, messageId: att.message_id },
"Skipping attachment: no uploaded URL available",
);
return;
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 15000);
try {
const res = await fetch(urlToUse, { signal: controller.signal });
if (!res.ok || !res.body) {
log.warn(
{ attachmentId: att.id, url: urlToUse, status: res.status },
"Failed to download attachment: HTTP error or no body",
);
return;
}
let totalBytes = 0;
const chunks: Uint8Array[] = [];
const reader = res.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) {
totalBytes += value.length;
if (totalBytes > 10 * 1024 * 1024) {
log.warn(
{ attachmentId: att.id, totalBytes },
"Attachment too large (>10MB) — skipping",
);
reader.cancel();
return;
}
chunks.push(value);
}
}
const imageBytes = Buffer.concat(chunks);
const sniffedMime = sniffImageMimeType(imageBytes);
if (!sniffedMime) {
log.warn(
{ attachmentId: att.id },
"Skipping attachment: not a recognised image format",
);
return;
}
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(imageBytes, maxDimension);
const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`;
const part: MessageImagePart = {
type: "image_url",
image_url: { url: dataUrl },
sourceLabel: `[gambar di atas adalah attachment ${att.filename} dari pesan id=${att.message_id}]`,
};
addImageToMap(imageMap, targetId, part);
} catch (err) {
log.warn(
{
attachmentId: att.id,
error: err instanceof Error ? err.message : String(err),
},
"Error downloading attachment",
);
} finally {
clearTimeout(timeoutId);
}
}
async function downloadMediaCandidate(
candidate: MediaCandidate,
targetId: string,
maxDimension: number,
imageMap: Map<string, MessageImagePart[]>,
mediaAnalysisMap: Map<string, string[]>,
): Promise<void> {
if ((imageMap.get(targetId)?.length ?? 0) >= 8) return;
if (candidate.customEmojiId || candidate.stickerName) {
const visionCacheKey = candidate.customEmojiId
? makeCustomEmojiCacheKey(candidate.customEmojiId)
: makeStickerCacheKey(candidate.stickerName!);
const cachedVision = await getCachedMediaAnalysis(visionCacheKey);
if (cachedVision) {
log.debug(
{ cacheKey: visionCacheKey },
"Vision cache HIT for media candidate — skipped download",
);
const analysisText = `[Media analysis for message ${candidate.messageId}] ${candidate.label}: ${cachedVision}`;
const existing = mediaAnalysisMap.get(targetId) ?? [];
existing.push(analysisText);
mediaAnalysisMap.set(targetId, existing);
return;
}
}
if (candidate.stickerName && isStickerCacheReady()) {
try {
const cached = await getStickerFromCache(candidate.stickerName);
if (cached && cached.imageUrl) {
const part: MessageImagePart = {
type: "image_url",
image_url: { url: cached.imageUrl },
sourceLabel: candidate.label,
stickerName: candidate.stickerName,
};
addImageToMap(imageMap, targetId, part);
return;
}
} catch (stickerErr) {
log.warn(
{
stickerName: candidate.stickerName,
error:
stickerErr instanceof Error
? stickerErr.message
: String(stickerErr),
},
"Sticker cache lookup failed — falling through to network fetch",
);
}
}
const result = await fetchUrlSafely(candidate.url);
if (result.type !== "image" || !result.data || !result.mimeType) {
log.warn(
{
url: candidate.url,
resultType: result.type,
resultHasData: !!result.data,
messageId: candidate.messageId,
label: candidate.stickerName
? `sticker:${candidate.stickerName}`
: candidate.customEmojiName
? `emoji:${candidate.customEmojiName}`
: "embed/other",
},
"Media candidate fetch did not return a usable image — skipping",
);
return;
}
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(result.data, maxDimension);
const base64 = resizedBuffer.toString("base64");
if (candidate.stickerName) {
uploadAndCacheSticker(
candidate.stickerName,
resizedBuffer,
resizedMime,
).catch(() => {});
}
const part: MessageImagePart = {
type: "image_url",
image_url: { url: `data:${resizedMime};base64,${base64}` },
sourceLabel: candidate.label,
stickerName: candidate.stickerName,
customEmojiId: candidate.customEmojiId,
customEmojiName: candidate.customEmojiName,
};
addImageToMap(imageMap, targetId, part);
}
async function fetchUrlInline(
url: string,
targetId: string,
maxDimension: number,
imageMap: Map<string, MessageImagePart[]>,
urlWebTexts: string[],
): Promise<void> {
const result = await fetchUrlSafely(url);
if (result.type === "image" && result.data && result.mimeType) {
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(result.data, maxDimension);
const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`;
const part: MessageImagePart = {
type: "image_url",
image_url: { url: dataUrl },
sourceLabel: `[gambar di atas berasal dari link ${url} pada pesan id=${targetId}]`,
};
addImageToMap(imageMap, targetId, part);
} else if (result.type === "text" && result.textContent) {
urlWebTexts.push(`[Isi Web dari ${url}]: ${result.textContent}`);
}
}
function addImageToMap(
imageMap: Map<string, MessageImagePart[]>,
targetId: string,
part: MessageImagePart,
): void {
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
}
interface MediaCandidate {
messageId: string;
url: string;
label: string;
stickerName?: string;
customEmojiId?: string;
customEmojiName?: string;
}
function buildMediaCandidates(
targetId: string,
mediaEvidence: ReturnType<typeof extractMessageMediaEvidence>,
): MediaCandidate[] {
return [
...mediaEvidence.stickers
.filter((s) => s.url)
.map(
(s): MediaCandidate => ({
messageId: targetId,
url: s.url,
label: `[gambar di atas adalah sticker "${s.name}" dari pesan id=${targetId}]`,
stickerName: s.name,
}),
),
...mediaEvidence.embeds.flatMap((embed): MediaCandidate[] =>
[
embed.image
? ({
messageId: targetId,
url: embed.image,
label: `[gambar di atas berasal dari embed image pada pesan id=${targetId}]`,
} as MediaCandidate)
: null,
embed.thumbnail
? ({
messageId: targetId,
url: embed.thumbnail,
label: `[gambar di atas berasal dari embed thumbnail pada pesan id=${targetId}]`,
} as MediaCandidate)
: null,
].filter((c): c is MediaCandidate => c !== null),
),
...mediaEvidence.customEmojis.map(
(emoji): MediaCandidate => ({
messageId: targetId,
url: emoji.url,
label: `[gambar di atas adalah custom emoji "${emoji.name}" dari pesan id=${targetId}]`,
customEmojiId: emoji.id,
customEmojiName: emoji.name,
}),
),
];
}
@@ -1,7 +1,7 @@
import { createChildLogger } from "@bete/shared/logger"; import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/config.js"; import { config } from "../../shared/config/config.js";
import { uploadToTele } from "../attachment-upload/teleUpload.js";
import { executeAll, executeGet } from "../../shared/database/drizzle.js"; import { executeAll, executeGet } from "../../shared/database/drizzle.js";
import { uploadToTele } from "../attachment-upload/teleUpload.js";
const logger = createChildLogger("sticker-cache"); const logger = createChildLogger("sticker-cache");
@@ -1,9 +1,9 @@
import { eq, and, desc } from "drizzle-orm"; import { and, desc, eq } from "drizzle-orm";
import { getDatabase } from "../../shared/database/drizzle.js"; import { getDatabase } from "../../shared/database/drizzle.js";
import { import {
userReputationsTable,
messagesTable, messagesTable,
UserReputation, UserReputation,
userReputationsTable,
} from "../../shared/database/schema.js"; } from "../../shared/database/schema.js";
/** /**
@@ -377,7 +377,9 @@ export class CommandHandler {
}; };
} }
private async handleVoiceTransmitStart(cmd: BackendCommand): Promise<CommandReply> { private async handleVoiceTransmitStart(
cmd: BackendCommand,
): Promise<CommandReply> {
if (!discordPlayer.isConnected()) { if (!discordPlayer.isConnected()) {
return { return {
id: cmd.id, id: cmd.id,
@@ -412,7 +414,9 @@ export class CommandHandler {
} }
} }
private async handleVoiceTransmitStop(cmd: BackendCommand): Promise<CommandReply> { private async handleVoiceTransmitStop(
cmd: BackendCommand,
): Promise<CommandReply> {
try { try {
await voiceTransmitter.stop(); await voiceTransmitter.stop();
logger.info("Voice transmit stopped"); logger.info("Voice transmit stopped");
@@ -464,9 +468,7 @@ export class CommandHandler {
* Fire-and-forget SET using the persistent Redis publisher connection. * Fire-and-forget SET using the persistent Redis publisher connection.
*/ */
private setKey(key: string, value: string): void { private setKey(key: string, value: string): void {
this.redisPub this.redisPub.set(key, value).catch((err: unknown) => {
.set(key, value)
.catch((err: unknown) => {
const msg = err instanceof Error ? err.message : String(err); const msg = err instanceof Error ? err.message : String(err);
logger.warn({ key, error: msg }, "Failed to update Redis status key"); logger.warn({ key, error: msg }, "Failed to update Redis status key");
}); });
@@ -138,7 +138,7 @@ export class EventBroadcaster {
async voicePcmData( async voicePcmData(
pcmBuffer: Buffer, pcmBuffer: Buffer,
userId: string, userId: string,
metadata?: any, metadata?: Record<string, unknown>,
): Promise<void> { ): Promise<void> {
await this.publisher.publish("discord:voice:pcm", { await this.publisher.publish("discord:voice:pcm", {
type: "voice_pcm_data", type: "voice_pcm_data",
@@ -1,929 +0,0 @@
import { createChildLogger } from "@bete/shared/logger";
import { executeAll, executeGet } from "../../shared/database/drizzle.js";
import type { MessageRecord } from "./types.js";
const logger = createChildLogger("analytics-store");
// ── Types ──────────────────────────────────────────────────────────────
export interface HourlyBucket {
hour: string;
count: number;
clean: number;
warned: number;
flagged: number;
error: number;
}
export interface TopicTrend {
topic: string;
count: number;
score: number;
}
export interface UserStat {
user_id: string;
username: string;
avatar_url: string | null;
message_count: number;
edited_count: number;
deleted_count: number;
flagged_count: number;
last_active: number;
}
export interface ModerationBreakdown {
total: number;
clean: number;
warned: number;
flagged: number;
error: number;
pending: number;
average_score: number;
}
export interface AnalyticsOverview {
period: { start: number; end: number };
messages: ModerationBreakdown;
hourly: HourlyBucket[];
topics: TopicTrend[];
top_users: UserStat[];
active_users_count: number;
total_channels: number;
}
// ══════════════════════════════════════════════════════════════════════════
// GENERIC QUERY CACHE (reduces duplicate DB calls from 5s auto-refresh)
// ══════════════════════════════════════════════════════════════════════════
interface CacheEntry<T> {
data: T;
expiresAt: number;
}
const queryCache = new Map<string, CacheEntry<any>>();
/** Default TTL for aggregate queries 10s is long enough to prevent redundant
* calls from the 5s auto-refresh but short enough to feel real-time. */
const AGGREGATE_CACHE_TTL_MS = 10_000;
/** Topic extraction is expensive (JSON parsing). Cache longer. */
const TOPIC_CACHE_TTL_MS = 120_000;
function makeCacheKey(prefix: string, params: Record<string, any>): string {
return `${prefix}:${JSON.stringify(params)}`;
}
function getCached<T>(key: string): T | undefined {
const entry = queryCache.get(key);
if (entry && entry.expiresAt > Date.now()) return entry.data;
if (entry) queryCache.delete(key); // expired
return undefined;
}
function setCache<T>(key: string, data: T, ttl: number): void {
queryCache.set(key, { data, expiresAt: Date.now() + ttl });
// Prune old entries if cache grows too large (>200 entries)
if (queryCache.size > 200) {
const now = Date.now();
for (const [k, v] of queryCache) {
if (v.expiresAt <= now) queryCache.delete(k);
}
}
}
// ── Hourly Message Stats ───────────────────────────────────────────────
export async function getHourlyStats(input: {
guildId: string;
channelId?: string;
hours?: number;
}): Promise<HourlyBucket[]> {
const { guildId, channelId, hours = 24 } = input;
const cacheKey = makeCacheKey("hourly", { guildId, channelId, hours });
const cached = getCached<HourlyBucket[]>(cacheKey);
if (cached) return cached;
try {
const since = Date.now() - hours * 3600_000;
const hourExpr = `to_char(to_timestamp((created_at / 3600000) * 3600), 'YYYY-MM-DD HH24:MI:SS') as hour`;
const rows = await executeAll(
`
SELECT
${hourExpr},
count(*) as count,
count(case when ai_status = 'clean' then 1 end) as clean,
count(case when ai_status = 'warn' then 1 end) as warned,
count(case when ai_status = 'flagged' then 1 end) as flagged,
count(case when ai_status = 'error' then 1 end) as error
FROM messages
WHERE guild_id = ?
AND created_at >= ?
AND deleted_at IS NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
GROUP BY (created_at / 3600000)
ORDER BY hour ASC
`,
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
);
// Initialize all hour buckets (fill gaps with zeros)
const buckets = new Map<
string,
{
count: number;
clean: number;
warned: number;
flagged: number;
error: number;
}
>();
for (let h = 0; h < hours; h++) {
const ts = new Date(since + h * 3600_000);
ts.setMinutes(0, 0, 0);
const key = ts.toISOString().slice(0, 13) + ":00:00Z";
buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 });
}
for (const row of rows) {
const d = new Date(row.hour.replace(" ", "T") + "Z");
const key = d.toISOString().slice(0, 13) + ":00:00Z";
const bucket = buckets.get(key);
if (!bucket) continue;
bucket.count = row.count;
bucket.clean = row.clean;
bucket.warned = row.warned;
bucket.flagged = row.flagged;
bucket.error = row.error;
}
const result = Array.from(buckets.entries())
.sort(([a], [b]) => a.localeCompare(b))
.map(([hour, data]) => ({ hour, ...data }));
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
return result;
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get hourly stats",
);
return [];
}
}
// ── Topic Trends ───────────────────────────────────────────────────────
const STOP_WORDS = new Set([
"yang",
"dan",
"itu",
"ini",
"dengan",
"akan",
"pada",
"dari",
"di",
"ke",
"untuk",
"tidak",
"ada",
"juga",
"sudah",
"saya",
"kamu",
"dia",
"mereka",
"kami",
"aku",
"lo",
"lu",
"gua",
"gue",
"org",
"orang",
"aja",
"sama",
"kalo",
"kalau",
"bisa",
"karena",
"gak",
"nggak",
"ga",
"tak",
"belum",
"udah",
"dah",
"lah",
"kah",
"pun",
"nih",
"tuh",
"deh",
"dong",
"si",
"nya",
"kan",
"ya",
"yah",
"yuk",
"kok",
"loh",
"nah",
"wow",
"eh",
"the",
"a",
"an",
"is",
"are",
"was",
"were",
"be",
"been",
"being",
"have",
"has",
"had",
"having",
"do",
"does",
"did",
"doing",
"will",
"would",
"could",
"should",
"may",
"might",
"must",
"shall",
"i",
"you",
"he",
"she",
"it",
"we",
"they",
"me",
"him",
"her",
"us",
"them",
"my",
"your",
"his",
"its",
"our",
"their",
"and",
"but",
"or",
"nor",
"not",
"so",
"yet",
"for",
"if",
"to",
"of",
"in",
"on",
"at",
"by",
"as",
"with",
"about",
"just",
"then",
"now",
"here",
"there",
"when",
"where",
"why",
"how",
"all",
"both",
"each",
"few",
"more",
"most",
"other",
"some",
"such",
"only",
"own",
"same",
"too",
"very",
"can",
"go",
"ok",
"okay",
"yeah",
"yes",
"no",
]);
function extractTopics(messages: MessageRecord[], topN = 15): TopicTrend[] {
const topicScores = new Map<string, { count: number; score: number }>();
const wordFreq = new Map<string, number>();
const flaggedWordFreq = new Map<string, number>();
for (const msg of messages) {
if (msg.ai_analysis) {
try {
const analysis = JSON.parse(msg.ai_analysis);
const topics = analysis.topics;
if (topics && Array.isArray(topics)) {
for (const topic of topics) {
const key =
typeof topic === "string" ? topic : topic.name || topic.topic;
if (!key) continue;
const k = key.toLowerCase();
const score = msg.ai_moderation_score || 0;
const existing = topicScores.get(k);
if (existing) {
existing.count++;
existing.score += score;
} else {
topicScores.set(k, { count: 1, score });
}
}
}
if (analysis.category) {
const cat = String(analysis.category).toLowerCase();
const existing = topicScores.get(cat);
if (existing) {
existing.count++;
existing.score += msg.ai_moderation_score || 0;
} else {
topicScores.set(cat, {
count: 1,
score: msg.ai_moderation_score || 0,
});
}
}
} catch {
/* not valid JSON */
}
}
if (msg.content) {
const words = msg.content
.toLowerCase()
.replace(/[^\w\s]/g, " ")
.split(/\s+/)
.filter((w) => w.length > 2 && !STOP_WORDS.has(w));
for (const word of words) {
wordFreq.set(word, (wordFreq.get(word) || 0) + 1);
if (msg.ai_status === "flagged" || msg.ai_status === "warn") {
flaggedWordFreq.set(word, (flaggedWordFreq.get(word) || 0) + 1);
}
}
}
}
const results: TopicTrend[] = [];
for (const [topic, data] of topicScores) {
results.push({ topic, count: data.count, score: data.score });
}
const sortedWords = Array.from(wordFreq.entries())
.sort(([, a], [, b]) => b - a)
.slice(0, topN);
for (const [word, count] of sortedWords) {
if (!topicScores.has(word)) {
results.push({
topic: word,
count,
score: flaggedWordFreq.get(word) || 0,
});
}
}
return results.sort((a, b) => b.count - a.count).slice(0, topN);
}
export async function getTopicTrends(input: {
guildId: string;
channelId?: string;
hours?: number;
}): Promise<TopicTrend[]> {
const { guildId, channelId, hours = 24 } = input;
const cacheKey = makeCacheKey("topics", { guildId, channelId, hours });
const cached = getCached<TopicTrend[]>(cacheKey);
if (cached) return cached;
try {
const since = Date.now() - hours * 3600_000;
// Fetch all analyzed messages within the time window (no hard row cap).
// Messages without ai_analysis are excluded which naturally limits rows.
const rows = (await executeAll(
`
SELECT
id, content, ai_status, ai_analysis, ai_moderation_score,
ai_moderation_flags, created_at
FROM messages
WHERE guild_id = ?
AND created_at >= ?
AND deleted_at IS NULL
AND ai_analysis IS NOT NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
ORDER BY created_at DESC
`,
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
)) as MessageRecord[];
const result = extractTopics(rows);
setCache(cacheKey, result, TOPIC_CACHE_TTL_MS);
return result;
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get topic trends",
);
return [];
}
}
// ── User Leaderboard ────────────────────────────────────────────────────
export async function getUserLeaderboard(input: {
guildId: string;
channelId?: string;
hours?: number;
limit?: number;
}): Promise<UserStat[]> {
const { guildId, channelId, hours = 24, limit = 20 } = input;
const cacheKey = makeCacheKey("leaderboard", {
guildId,
channelId,
hours,
limit,
});
const cached = getCached<UserStat[]>(cacheKey);
if (cached) return cached;
try {
const since = Date.now() - hours * 3600_000;
const rows = await executeAll(
`
SELECT
user_id,
username,
avatar_url,
count(*) as message_count,
count(case when type = 'edited' then 1 end) as edited_count,
count(case when type = 'deleted' then 1 end) as deleted_count,
count(case when ai_status = 'flagged' then 1 end) as flagged_count,
max(created_at) as last_active
FROM messages
WHERE guild_id = ?
AND created_at >= ?
AND deleted_at IS NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
GROUP BY user_id, username, avatar_url
ORDER BY message_count DESC
LIMIT ?
`,
channelId
? [guildId, since, channelId, channelId, limit]
: [guildId, since, limit],
);
const result = rows as UserStat[];
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
return result;
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get user leaderboard",
);
return [];
}
}
// ── Moderation Stats ───────────────────────────────────────────────────
export async function getModerationStats(input: {
guildId: string;
channelId?: string;
hours?: number;
}): Promise<ModerationBreakdown> {
const { guildId, channelId, hours = 24 } = input;
const cacheKey = makeCacheKey("modstats", { guildId, channelId, hours });
const cached = getCached<ModerationBreakdown>(cacheKey);
if (cached) return cached;
try {
const since = Date.now() - hours * 3600_000;
const avgScoreExpr = `round(avg(ai_moderation_score)::numeric, 2)`;
const row = await executeGet(
`
SELECT
count(*) as total,
count(case when ai_status = 'clean' then 1 end) as clean,
count(case when ai_status = 'warn' then 1 end) as warned,
count(case when ai_status = 'flagged' then 1 end) as flagged,
count(case when ai_status = 'error' then 1 end) as error,
count(case when ai_status = 'pending' or ai_status IS NULL then 1 end) as pending,
${avgScoreExpr} as average_score
FROM messages
WHERE guild_id = ?
AND created_at >= ?
AND deleted_at IS NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
`,
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
);
const result: ModerationBreakdown = row
? {
total: row.total ?? 0,
clean: row.clean ?? 0,
warned: row.warned ?? 0,
flagged: row.flagged ?? 0,
error: row.error ?? 0,
pending: row.pending ?? 0,
average_score: row.average_score ?? 0,
}
: {
total: 0,
clean: 0,
warned: 0,
flagged: 0,
error: 0,
pending: 0,
average_score: 0,
};
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
return result;
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get moderation stats",
);
return {
total: 0,
clean: 0,
warned: 0,
flagged: 0,
error: 0,
pending: 0,
average_score: 0,
};
}
}
// ── Active Channels Count ──────────────────────────────────────────────
export async function getActiveChannelCount(input: {
guildId: string;
hours?: number;
}): Promise<number> {
const { guildId, hours = 24 } = input;
const cacheKey = makeCacheKey("channels", { guildId, hours });
const cached = getCached<number>(cacheKey);
if (cached !== undefined) return cached;
try {
const since = Date.now() - hours * 3600_000;
const row = await executeGet(
`
SELECT count(DISTINCT channel_id) as cnt
FROM messages
WHERE guild_id = ?
AND created_at >= ?
AND deleted_at IS NULL
`,
[guildId, since],
);
const result = row?.cnt ?? 0;
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
return result;
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get active channel count",
);
return 0;
}
}
// ── Top Violators ─────────────────────────────────────────────────────
export interface ViolatorStat {
user_id: string;
username: string;
avatar_url: string | null;
total_messages: number;
flagged_count: number;
warned_count: number;
violation_score: number;
worst_flags: string[];
last_violation: number;
}
export async function getTopViolators(input: {
guildId: string;
channelId?: string;
hours?: number;
limit?: number;
}): Promise<ViolatorStat[]> {
const { guildId, channelId, hours = 24, limit = 20 } = input;
const cacheKey = makeCacheKey("violators", {
guildId,
channelId,
hours,
limit,
});
const cached = getCached<ViolatorStat[]>(cacheKey);
if (cached) return cached;
try {
const since = Date.now() - hours * 3600_000;
const rows = await executeAll(
`
SELECT
user_id,
username,
avatar_url,
count(*) as total_messages,
count(case when ai_status = 'flagged' then 1 end) as flagged_count,
count(case when ai_status = 'warn' then 1 end) as warned_count,
max(case when ai_status in ('flagged', 'warn') then created_at else 0 end) as last_violation
FROM messages
WHERE guild_id = ?
AND created_at >= ?
AND deleted_at IS NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
GROUP BY user_id, username, avatar_url
HAVING count(case when ai_status = 'flagged' then 1 end) > 0
OR count(case when ai_status = 'warn' then 1 end) > 0
ORDER BY (
count(case when ai_status = 'flagged' then 1 end) * 3
+ count(case when ai_status = 'warn' then 1 end)
) DESC
LIMIT ?
`,
channelId
? [guildId, since, channelId, channelId, limit]
: [guildId, since, limit],
);
const violators: ViolatorStat[] = rows.map((row: any) => {
const flaggedCount = Number(row.flagged_count ?? 0);
const warnedCount = Number(row.warned_count ?? 0);
return {
user_id: row.user_id,
username: row.username,
avatar_url: row.avatar_url,
total_messages: Number(row.total_messages ?? 0),
flagged_count: flaggedCount,
warned_count: warnedCount,
violation_score: flaggedCount * 3 + warnedCount,
worst_flags: [],
last_violation: Number(row.last_violation ?? 0),
};
});
setCache(cacheKey, violators, AGGREGATE_CACHE_TTL_MS);
return violators;
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get top violators",
);
return [];
}
}
// ── Daily Trend (for multi-day line chart) ────────────────────────────
export interface TrendBucket {
date: string;
count: number;
clean: number;
warned: number;
flagged: number;
error: number;
}
export async function getDailyTrend(input: {
guildId: string;
channelId?: string;
hours?: number;
}): Promise<TrendBucket[]> {
const { guildId, channelId, hours = 168 } = input;
const cacheKey = makeCacheKey("daily_trend", { guildId, channelId, hours });
const cached = getCached<TrendBucket[]>(cacheKey);
if (cached) return cached;
try {
const since = Date.now() - hours * 3600_000;
const dateExpr = `to_char(date_trunc('day', to_timestamp(created_at / 1000)), 'YYYY-MM-DD') as date`;
const rows = await executeAll(
`
SELECT
${dateExpr},
count(*) as count,
count(case when ai_status = 'clean' then 1 end) as clean,
count(case when ai_status = 'warn' then 1 end) as warned,
count(case when ai_status = 'flagged' then 1 end) as flagged,
count(case when ai_status = 'error' then 1 end) as error
FROM messages
WHERE guild_id = ?
AND created_at >= ?
AND deleted_at IS NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
GROUP BY 1
ORDER BY 1 ASC
`,
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
);
// Initialize all day buckets (fill gaps with zeros)
const buckets = new Map<
string,
{
count: number;
clean: number;
warned: number;
flagged: number;
error: number;
}
>();
const msPerDay = 86400_000;
const startDay = Math.floor(since / msPerDay) * msPerDay;
const endDay = Math.floor(Date.now() / msPerDay) * msPerDay;
for (let d = startDay; d <= endDay; d += msPerDay) {
const key = new Date(d).toISOString().slice(0, 10);
buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 });
}
for (const row of rows) {
const bucket = buckets.get(row.date);
if (!bucket) continue;
bucket.count = row.count;
bucket.clean = row.clean;
bucket.warned = row.warned;
bucket.flagged = row.flagged;
bucket.error = row.error;
}
const result = Array.from(buckets.entries())
.sort(([a], [b]) => a.localeCompare(b))
.map(([date, data]) => ({ date, ...data }));
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
return result;
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get daily trend",
);
return [];
}
}
// ── Activity Heatmap (day-of-week × hour-of-day) ──────────────────────
export interface HeatmapCell {
dayOfWeek: number; // 0=Senin, 6=Minggu
hour: number; // 0-23
count: number;
clean: number;
warned: number;
flagged: number;
}
export async function getActivityHeatmap(input: {
guildId: string;
channelId?: string;
hours?: number;
}): Promise<HeatmapCell[]> {
const { guildId, channelId, hours = 168 } = input;
const cacheKey = makeCacheKey("heatmap", { guildId, channelId, hours });
const cached = getCached<HeatmapCell[]>(cacheKey);
if (cached) return cached;
try {
const since = Date.now() - hours * 3600_000;
const dayExpr = `(extract(isodow from to_timestamp(created_at / 1000)) % 7)::int as day_of_week`;
const hourExpr = `extract(hour from to_timestamp(created_at / 1000))::int as hour`;
const rows = await executeAll(
`
SELECT
${dayExpr},
${hourExpr},
count(*) as count,
count(case when ai_status = 'clean' then 1 end) as clean,
count(case when ai_status = 'warn' then 1 end) as warned,
count(case when ai_status = 'flagged' then 1 end) as flagged
FROM messages
WHERE guild_id = ?
AND created_at >= ?
AND deleted_at IS NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
GROUP BY day_of_week, hour
ORDER BY day_of_week, hour
`,
channelId ? [guildId, since, channelId, channelId] : [guildId, since],
);
// Initialize all 7×24 cells with zeros
const cells = new Map<
string,
{ count: number; clean: number; warned: number; flagged: number }
>();
for (let d = 0; d < 7; d++) {
for (let h = 0; h < 24; h++) {
cells.set(`${d}-${h}`, { count: 0, clean: 0, warned: 0, flagged: 0 });
}
}
for (const row of rows) {
const key = `${row.day_of_week}-${row.hour}`;
const cell = cells.get(key);
if (!cell) continue;
cell.count = row.count;
cell.clean = row.clean;
cell.warned = row.warned;
cell.flagged = row.flagged;
}
const result = Array.from(cells.entries())
.map(([key, data]) => {
const [dayOfWeek, hour] = key.split("-").map(Number);
return { dayOfWeek, hour, ...data };
})
.sort((a, b) => a.dayOfWeek - b.dayOfWeek || a.hour - b.hour);
setCache(cacheKey, result, AGGREGATE_CACHE_TTL_MS);
return result;
} catch (error) {
logger.error(
{ error: error instanceof Error ? error.message : String(error) },
"Failed to get activity heatmap",
);
return [];
}
}
// ── Cache Invalidation (called when new messages arrive) ───────────────
export function invalidateAnalyticsCache(guildId: string): void {
const now = Date.now();
const needle = `"${guildId}"`;
for (const [key, entry] of queryCache) {
if (key.includes(needle) && entry.expiresAt > now) {
entry.expiresAt = 0; // expire immediately
}
}
}
// ── Combined Overview ──────────────────────────────────────────────────
export async function getAnalyticsOverview(input: {
guildId: string;
channelId?: string;
hours?: number;
}): Promise<AnalyticsOverview> {
const { guildId, hours = 24 } = input;
const now = Date.now();
const since = now - hours * 3600_000;
const [messages, hourly, topics, topUsers, totalChannels] = await Promise.all(
[
getModerationStats(input),
getHourlyStats(input),
getTopicTrends(input),
getUserLeaderboard(input),
getActiveChannelCount({ guildId, hours }),
],
);
return {
period: { start: since, end: now },
messages,
hourly,
topics,
top_users: topUsers,
active_users_count: topUsers.length,
total_channels: totalChannels,
};
}

Some files were not shown because too many files have changed in this diff Show More