From 2557a079164af2b84da8ba80eed94ebfd4db8da9 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Wed, 10 Jun 2026 20:56:16 +0700 Subject: [PATCH] feat(core): implement data retention, metrics, and enhanced media handling This commit introduces several significant improvements across the backend and gateway services: - **Data Retention**: Added an automated cleanup scheduler in `discord-gateway` to prune expired messages, attachments, and voice recordings based on configurable retention policies. - **Observability**: Integrated `prom-client` in the `backend` service to expose Prometheus metrics via `/api/metrics` and added default Node.js runtime metrics. - **Media Handling**: Enhanced `MediaHandler` in `discord-gateway` to support media URL resolution and improved playback status tracking. - **API & Config**: Expanded the configuration endpoint to expose more system settings and reorganized `.env.example` for better readability. - **Refactoring & Cleanup**: - Removed unused `better-sqlite3` dependency. - Refactored voice channel routing. - Improved error handling and testing coverage with comprehensive unit tests for shared utilities and error classes. - **Documentation**: Added `MEMORY.md` for project context. --- .env.example | 218 +++++---- MEMORY.md | 92 ++++ package.json | 2 - packages/shared/package.json | 2 + pnpm-lock.yaml | 43 +- pnpm-workspace.yaml | 2 - .../src/modules/config/config.routes.ts | 12 + .../src/modules/health/health.controller.ts | 8 + .../src/modules/health/health.routes.ts | 10 +- .../src/modules/voice/guilds.routes.ts | 19 +- .../backend/src/modules/voice/voice.routes.ts | 4 - services/backend/tests/placeholder.test.ts | 269 ++++++++++- services/discord-gateway/src/app/bootstrap.ts | 160 +++++++ .../modules/command-handler/media.handler.ts | 84 +++- .../modules/command-handler/voice.handler.ts | 61 --- .../modules/voice-recording/mediaSource.ts | 374 ++++++++++++++++ .../discord-gateway/tests/placeholder.test.ts | 418 +++++++++++++++++- .../frontend/src/features/messages/index.tsx | 8 +- 18 files changed, 1537 insertions(+), 249 deletions(-) create mode 100644 MEMORY.md create mode 100644 services/discord-gateway/src/modules/voice-recording/mediaSource.ts diff --git a/.env.example b/.env.example index 50802c0..a2dda43 100644 --- a/.env.example +++ b/.env.example @@ -1,136 +1,120 @@ # Discord Bot Configuration -DISCORD_TOKEN=your_bot_token_here -MONITOR_GUILD_ID=your_guild_id_here -TEXT_GUILD_ID=optional_text_guild_id -TEXT_CHANNEL_ID=optional_text_channel_id +# ============================================================================= -# Recording Configuration -RECORDINGS_DIR=./recordings -RECORDING_SEGMENT_MS=5000 -VERBOSE=false +# === Discord === +DISCORD_TOKEN=your_bot_token_here # REQUIRED +MONITOR_GUILD_ID=your_guild_id_here # Target guild for text monitoring +TEXT_GUILD_ID=optional_text_guild_id # Override text capture guild (falls back to MONITOR_GUILD_ID) +TEXT_CHANNEL_ID=optional_text_channel_id # Restrict text capture to a single channel -# Decoder Configuration -DECODER_ROTATE_MS=5000 -DECODER_COOLDOWN_MS=30000 +# === Voice Channels === +# VOICE_GUILD_ID= # Guild for voice connection (optional) +# VOICE_CHANNEL_ID= # Channel for voice connection (optional) -# Audio Configuration -AUDIO_STREAM_SILENCE_DURATION_MS=3000 -PACKET_FILTER_MIN_SIZE=8 -OPUS_FRAME_SIZE=960 -AUDIO_SAMPLE_RATE=48000 -AUDIO_CHANNELS=2 -AVATAR_SIZE=64 +# === Recording === +RECORDINGS_DIR=./recordings # Audio file output directory (default: ./recordings) +RECORDING_SEGMENT_MS=5000 # OGG segment duration in ms (default: 5000) -# Webserver Configuration -WEBSERVER_PORT=3000 +# === Decoder === +DECODER_ROTATE_MS=5000 # Opus decoder rotation interval in ms (default: 5000) +DECODER_COOLDOWN_MS=30000 # Decoder error cooldown in ms (default: 30000) -# Connection Configuration -VOICE_CONNECTION_TIMEOUT_MS=15000 -RECONNECT_TIMEOUT_MS=5000 +# === Audio === +AUDIO_STREAM_SILENCE_DURATION_MS=3000 # Silence threshold in ms before stopping stream (default: 3000) +PACKET_FILTER_MIN_SIZE=8 # Minimum Opus packet size in bytes (default: 8) +OPUS_FRAME_SIZE=960 # Opus frame size in samples (default: 960) +AUDIO_SAMPLE_RATE=48000 # Audio sample rate in Hz (default: 48000) +AUDIO_CHANNELS=2 # Number of audio channels (default: 2) +AVATAR_SIZE=64 # User avatar size in pixels (default: 64) -# Voice Recording Selection -# VOICE_GUILD_ID falls back to legacy GUILD_ID when omitted. -GUILD_ID=legacy_voice_guild_id -VOICE_GUILD_ID=voice_guild_id -VOICE_CHANNEL_ID=voice_channel_id +# === Webserver === +WEBSERVER_PORT=3001 # Backend HTTP/WS server port (default: 3001) -# Logging Configuration -LOG_LEVEL=info -NODE_ENV=development +# === Connection === +VOICE_CONNECTION_TIMEOUT_MS=15000 # Voice connection timeout in ms (default: 15000) +RECONNECT_TIMEOUT_MS=5000 # Reconnect timeout in ms (default: 5000) -# Moderation Configuration -TELE_UPLOAD_URL=https://upload.asepharyana.my.id/api/upload -ATTACHMENT_UPLOAD_TIMEOUT_MS=30000 -ATTACHMENT_MAX_SIZE_MB=100 -ATTACHMENT_RETRY_ATTEMPTS=3 -BACKLOG_SYNC_HOURS=24 -BACKLOG_SYNC_BATCH_SIZE=100 +# === Logging === +LOG_LEVEL=info # Pino log level: error|warn|info|http|verbose|debug|silly (default: info) +NODE_ENV=development # Environment: development|production|test (default: development) +VERBOSE=false # Enable verbose/debug logging (default: false) -# AI Analysis Configuration -AI_ANALYSIS_ENABLED=false -AI_LLM_API_KEY=your_9router_key_here -AI_LLM_BASE_URL=https://9router.asepharyana.my.id/v1 -AI_LLM_MODEL=text -# Vision model for image/video moderation (falls back to AI_LLM_MODEL if unset) -AI_LLM_VISION_MODEL=multimodal -# Max concurrent LLM API calls (default: 5) -AI_LLM_MAX_CONCURRENT=5 -# Maximum image dimension in pixels before resize for vision API (default: 1024) -AI_LLM_IMAGE_MAX_DIMENSION=1024 -# Maximum messages per text-only moderation batch (default: 20) -AI_LLM_TEXT_BATCH_SIZE=20 -# Timeout in ms for individual media analysis calls (default: 60000) -AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS=60000 +# === Admin === +ADMIN_PASSWORD=admin123 # Backend admin password for API auth (default: admin123) -# AI Moderation Analysis Tuning (advanced) -AI_ANALYSIS_DEBOUNCE_MS=500 -AI_ANALYSIS_RECOVERY_INTERVAL_MS=15000 -AI_ANALYSIS_ERROR_COOLDOWN_MS=30000 -# Max messages fetched per conversation batch (default: 200) -AI_ANALYSIS_MAX_BATCH_SIZE=200 -AI_ANALYSIS_MAX_CONTEXT_TOKENS=8000 -# Token budget for target messages specifically (default: 4000) -AI_ANALYSIS_MAX_TARGET_TOKENS=4000 -AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT=20 -# How long a conversation is locked while being processed (default: 120000ms) -AI_ANALYSIS_PROCESSING_TIMEOUT_MS=120000 -# Max concurrent individual-fallback jobs (default: 50) -AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT=50 -# Consecutive errors before individual circuit breaker trips (default: 50) -AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD=50 - -# OpenAI Moderation (optional separate provider) -# OPENAI_MODERATION_API_KEY=your_key_here -# OPENAI_MODERATION_BASE_URL=https://api.openai.com/v1 -# OPENAI_MODERATION_MODEL=omni-moderation-latest - -# Admin -ADMIN_PASSWORD=admin123 - -# Database Configuration (PostgreSQL) -# Option 1: Use DATABASE_URL for connection string +# === Database (PostgreSQL) === +# Option 1: Connection string (overrides individual params) # DATABASE_URL=postgresql://user:password@localhost:5432/discord_bot -# Option 2: Use individual connection parameters -# POSTGRES_HOST=localhost -# POSTGRES_PORT=5432 -# POSTGRES_USER=postgres -# POSTGRES_PASSWORD=your_password_here -# POSTGRES_DB=discord_bot +# Option 2: Individual connection parameters +POSTGRES_HOST=localhost # PostgreSQL host (default: localhost) +POSTGRES_PORT=5432 # PostgreSQL port (default: 5432) +POSTGRES_USER=postgres # PostgreSQL user (optional if DATABASE_URL provided) +POSTGRES_PASSWORD=your_password_here # PostgreSQL password (optional if DATABASE_URL provided) +POSTGRES_DB=discord_bot # PostgreSQL database name (optional if DATABASE_URL provided) +POSTGRES_POOL_MIN=2 # Minimum pool connections (default: 2) +POSTGRES_POOL_MAX=10 # Maximum pool connections (default: 10) -# Redis Configuration (queue + persistent KV store) -# REDIS_URL=redis://localhost:6379 +# === Redis === +REDIS_URL=redis://localhost:6379 # Redis connection string (default: redis://localhost:6379) -# PostgreSQL Connection Pool Configuration -# POSTGRES_POOL_MIN=2 -# POSTGRES_POOL_MAX=10 +# === Attachments === +TELE_UPLOAD_URL=https://upload.asepharyana.my.id/api/upload # Attachment upload endpoint (default) +ATTACHMENT_UPLOAD_TIMEOUT_MS=30000 # Upload timeout in ms (default: 30000) +ATTACHMENT_MAX_SIZE_MB=100 # Max attachment size in MB (default: 100) +ATTACHMENT_RETRY_ATTEMPTS=3 # Upload retry count (default: 3) +BACKLOG_SYNC_HOURS=24 # Backlog sync lookback window in hours (default: 24) +BACKLOG_SYNC_BATCH_SIZE=100 # Messages per backlog batch, max 100 (default: 100) -# Auto-Delete Configuration -AUTO_DELETE_FLAGGED_ENABLED=true -AUTO_DELETE_FLAGGED_DRY_RUN=true -AUTO_DELETE_FLAGGED_DELAY_MS=0 -AUTO_DELETE_MIN_CONFIDENCE=0.50 -AUTO_DELETE_ALLOWED_SEVERITIES=critical,high,medium,low -AUTO_DELETE_NOTIFY_USER=false -# Optional: comma-separated channel/user IDs to exclude -# AUTO_DELETE_EXCLUDED_CHANNEL_IDS= -# AUTO_DELETE_EXCLUDED_USER_IDS= -# Optional: comma-separated category filter (empty = all categories) -# AUTO_DELETE_ALLOWED_CATEGORIES= -# Optional: log channel ID for auto-delete actions -# AUTO_DELETE_LOG_CHANNEL_ID= +# === AI Analysis === +AI_ANALYSIS_ENABLED=false # Enable AI content moderation (default: false) +# AI_LLM_API_KEY= # REQUIRED if AI_ANALYSIS_ENABLED=true. LLM API key +AI_LLM_BASE_URL=https://9router.asepharyana.my.id/v1 # LLM API base URL (default) +AI_LLM_MODEL=text # LLM text model name (default: text) +# AI_LLM_VISION_MODEL= # Vision model for image analysis (falls back to AI_LLM_MODEL) +AI_LLM_MAX_CONCURRENT=5 # Max concurrent LLM API calls (default: 5) +AI_LLM_IMAGE_MAX_DIMENSION=1024 # Max image dimension in pixels before resize (default: 1024) +AI_LLM_TEXT_BATCH_SIZE=20 # Max messages per text-only moderation batch (default: 20) +AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS=60000 # Timeout in ms for media analysis calls (default: 60000) -# Retention Configuration (0 = disabled) -RETENTION_MESSAGES_DAYS=0 -RETENTION_ATTACHMENTS_DAYS=0 -RETENTION_VOICE_DAYS=0 -# Cleanup interval in ms (default: 24h) -RETENTION_CLEANUP_INTERVAL_MS=86400000 -RETENTION_DRY_RUN=true +# === AI Analysis Tuning === +AI_ANALYSIS_DEBOUNCE_MS=500 # Debounce window for batching messages in ms (default: 500) +AI_ANALYSIS_RECOVERY_INTERVAL_MS=15000 # Recovery interval after errors in ms (default: 15000) +AI_ANALYSIS_ERROR_COOLDOWN_MS=30000 # Cooldown period after consecutive errors in ms (default: 30000) +AI_ANALYSIS_MAX_BATCH_SIZE=200 # Max messages fetched per conversation batch (default: 200) +AI_ANALYSIS_MAX_CONTEXT_TOKENS=8000 # Token budget for context window (default: 8000) +AI_ANALYSIS_MAX_TARGET_TOKENS=4000 # Token budget for target messages (default: 4000) +AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT=20 # Max messages in context window (default: 20) +AI_ANALYSIS_PROCESSING_TIMEOUT_MS=120000 # Conversation lock timeout in ms (default: 120000) +AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT=50 # Max concurrent individual-fallback jobs (default: 50) +AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD=50 # Consecutive errors before circuit breaker trips (default: 50) -# Database Migration Configuration -# Safe default: run migrations on startup before the app accepts traffic. -AUTO_MIGRATE_ON_STARTUP=true +# === OpenAI Moderation (optional separate provider) === +# OPENAI_MODERATION_API_KEY= # OpenAI API key for moderation endpoint +# OPENAI_MODERATION_BASE_URL=https://api.openai.com/v1 # OpenAI moderation base URL (default) +# OPENAI_MODERATION_MODEL=omni-moderation-latest # OpenAI moderation model (default) -# Worker Pool Configuration -# PISCINA_MAX_THREADS=4 +# === Auto-Delete === +AUTO_DELETE_FLAGGED_ENABLED=true # Enable auto-deletion of flagged messages (default: true) +AUTO_DELETE_FLAGGED_DRY_RUN=true # Dry-run mode: log but do not delete (default: false) +AUTO_DELETE_FLAGGED_DELAY_MS=0 # Delay before auto-delete in ms (default: 0) +AUTO_DELETE_MIN_CONFIDENCE=0.5 # Minimum AI confidence threshold 0-1 (default: 0.5) +AUTO_DELETE_ALLOWED_SEVERITIES=critical,high,medium,low # Comma-separated severities (default) +AUTO_DELETE_ALLOWED_CATEGORIES= # Comma-separated category filter (empty = all) +AUTO_DELETE_EXCLUDED_CHANNEL_IDS= # Comma-separated channel IDs to exclude +AUTO_DELETE_EXCLUDED_USER_IDS= # Comma-separated user IDs to exclude +AUTO_DELETE_NOTIFY_USER=false # Notify user when their message is auto-deleted (default: false) +AUTO_DELETE_LOG_CHANNEL_ID= # Channel ID to log auto-delete actions + +# === Retention (0 = disabled) === +RETENTION_MESSAGES_DAYS=0 # Message retention in days (default: 0 = off) +RETENTION_ATTACHMENTS_DAYS=0 # Attachment retention in days (default: 0 = off) +RETENTION_VOICE_DAYS=0 # Voice recording retention in days (default: 0 = off) +RETENTION_CLEANUP_INTERVAL_MS=86400000 # Cleanup interval in ms (default: 24h) +RETENTION_DRY_RUN=true # Dry-run: log but do not delete (default: true) + +# === Migration === +AUTO_MIGRATE_ON_STARTUP=true # Run database migrations on startup (default: true) + +# === Worker Pool === +# PISCINA_MAX_THREADS=4 # Worker thread pool size (optional, defaults to CPU cores) diff --git a/MEMORY.md b/MEMORY.md new file mode 100644 index 0000000..8ed8e90 --- /dev/null +++ b/MEMORY.md @@ -0,0 +1,92 @@ +# Durable Memory Wiki + +Consolidated knowledge and long-term facts. + +## Core Learnings + +- **Topic:** session + **Context:** Failure observation: session + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** biome check diagnosticlevelerror + **Context:** Failure observation: Exit code 1 +$ biome check --diagnostic-level=error . +src/modules/voice-recording/recorder/segment.ts:1:1 assist/source/organizeImports FIXABLE ━━━━━━━━━━ + + × Sort these imports + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** typecheck scope workspace + **Context:** Failure observation: Exit code 2 +$ pnpm -r run typecheck +Scope: 6 of 7 workspace projects +packages/shared typecheck$ tsc --noEmit +packages/shared typecheck: Done +services/backend typecheck$ tsc --noEmit +services/discord-g + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** eisdir illegal operation + **Context:** Failure observation: EISDIR: illegal operation on a directory, read '/mnt/code/bete/packages/shared/src/types/' + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** exist current working + **Context:** Failure observation: File does not exist. Note: your current working directory is /mnt/code/bete. + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** error 32603 pattern + **Context:** Failure observation: MCP error -32603: pattern must be a non-empty string. + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** errpnpmnoscript missing script + **Context:** Failure observation: Exit code 2 +[ERR_PNPM_NO_SCRIPT] Missing script: build:shared + +Command "build:shared" not found. Did you mean "pnpm run build:backend"? +$ tsc +$ pnpm --filter './services/backend' run build +$ tsc +$ pnp + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** projects matched filters + **Context:** Failure observation: Exit code 1 +No projects matched the filters "vendor/*" in "/mnt/code/bete" +Scope: 6 of 7 workspace projects +vendor/discord.js-selfbot-v13 test$ npm run lint && npm run test:typescript && npm run docs: + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** scope workspace projects + **Context:** Failure observation: Exit code 1 +$ pnpm -r run test +Scope: 6 of 7 workspace projects +vendor/discord.js-selfbot-v13 test$ npm run lint && npm run test:typescript && npm run docs:test +vendor/discord.js-selfbot-v13 test: > d + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** error 32603 include + **Context:** Failure observation: MCP error -32603: include must be an array of strings. + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** nodeinternalmodulescjsloader1522 throw error + **Context:** Failure observation: Exit code 1 +node:internal/modules/cjs/loader:1522 + throw err; + ^ + +Error: Cannot find module 'ioredis' +Require stack: +- /mnt/code/bete/[eval] + at Module._resolveFilename (node:internal/modules/cjs + *Promoted on:* 2026-06-10T03:02:46.068Z + +- **Topic:** biome check diagnosticlevelerror + **Context:** Failure observation: Exit code 1 +$ biome check --diagnostic-level=error src/ +src/features/messages/index.tsx:4:1 assist/source/organizeImports FIXABLE ━━━━━━━━━━━━━━━━━━━━━━� + *Promoted on:* 2026-06-10T09:16:35.182Z + +- **Topic:** eval1 matches found + **Context:** Failure observation: Exit code 1 +(eval):1: no matches found: tsconfig*.json + *Promoted on:* 2026-06-10T09:49:12.792Z diff --git a/package.json b/package.json index ebc704d..6710315 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,6 @@ }, "devDependencies": { "@biomejs/biome": "latest", - "@types/better-sqlite3": "^7.6.13", - "better-sqlite3": "^11.9.1", "drizzle-kit": "^0.31.10", "tsx": "^4.22.2", "typescript": "^5.9.3" diff --git a/packages/shared/package.json b/packages/shared/package.json index 9e51b81..2f6b53b 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -11,6 +11,8 @@ "./database/schema": "./dist/database/schema.js", "./errors": "./dist/errors/index.js", "./logger": "./dist/logger/index.js", + "./moderation-types": "./dist/moderation-types.js", + "./redis-channels": "./dist/redis-channels.js", "./utils": "./dist/utils/index.js" }, "scripts": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2fc699f..d5ba52a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,12 +15,6 @@ importers: '@biomejs/biome': specifier: latest version: 2.4.16 - '@types/better-sqlite3': - specifier: ^7.6.13 - version: 7.6.13 - better-sqlite3: - specifier: ^11.9.1 - version: 11.10.0 drizzle-kit: specifier: ^0.31.10 version: 0.31.10 @@ -35,7 +29,7 @@ importers: dependencies: drizzle-orm: specifier: ^0.45.2 - version: 0.45.2(@opentelemetry/api@1.9.1)(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@11.10.0)(pg@8.21.0) + version: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(pg@8.21.0) pino: specifier: ^9.0.0 version: 9.14.0 @@ -69,7 +63,7 @@ importers: version: 17.4.2 drizzle-orm: specifier: ^0.45.2 - version: 0.45.2(@opentelemetry/api@1.9.1)(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@11.10.0)(pg@8.21.0) + version: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(pg@8.21.0) express: specifier: ^5.2.1 version: 5.2.1 @@ -145,7 +139,7 @@ importers: version: 17.4.2 drizzle-orm: specifier: ^0.45.2 - version: 0.45.2(@opentelemetry/api@1.9.1)(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@11.10.0)(pg@8.21.0) + version: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(pg@8.21.0) imghash: specifier: ^1.1.4 version: 1.1.4 @@ -2192,9 +2186,6 @@ packages: '@tybys/wasm-util@0.10.2': resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==} - '@types/better-sqlite3@7.6.13': - resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} - '@types/body-parser@1.19.6': resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} @@ -2502,15 +2493,9 @@ packages: before-after-hook@2.2.3: resolution: {integrity: sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==} - better-sqlite3@11.10.0: - resolution: {integrity: sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==} - bidi-js@1.0.3: resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==} - bindings@1.5.0: - resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} - bintrees@1.0.2: resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} @@ -3212,9 +3197,6 @@ packages: resolution: {integrity: sha512-uzk64HRpUZyTGZtVuvrjP0FYxzQrBf4rojot6J65YMEbwBLB0CWm0CLojVpwpmFmxcE/lkvYICgfcGozbBq6rw==} engines: {node: '>=6'} - file-uri-to-path@1.0.0: - resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} - fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -6903,10 +6885,6 @@ snapshots: tslib: 2.8.1 optional: true - '@types/better-sqlite3@7.6.13': - dependencies: - '@types/node': 25.9.0 - '@types/body-parser@1.19.6': dependencies: '@types/connect': 3.4.38 @@ -7213,19 +7191,10 @@ snapshots: before-after-hook@2.2.3: {} - better-sqlite3@11.10.0: - dependencies: - bindings: 1.5.0 - prebuild-install: 7.1.3 - bidi-js@1.0.3: dependencies: require-from-string: 2.0.2 - bindings@1.5.0: - dependencies: - file-uri-to-path: 1.0.0 - bintrees@1.0.2: {} bl@4.1.0: @@ -7592,12 +7561,10 @@ snapshots: esbuild: 0.25.12 tsx: 4.22.1 - drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@11.10.0)(pg@8.21.0): + drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(pg@8.21.0): optionalDependencies: '@opentelemetry/api': 1.9.1 - '@types/better-sqlite3': 7.6.13 '@types/pg': 8.20.0 - better-sqlite3: 11.10.0 pg: 8.21.0 dunder-proto@1.0.1: @@ -7962,8 +7929,6 @@ snapshots: file-type@10.11.0: {} - file-uri-to-path@1.0.0: {} - fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index acd204f..30ad585 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -9,7 +9,6 @@ packages: allowBuilds: '@discordjs/opus': true '@lng2004/node-datachannel': true - better-sqlite3: true esbuild: true node-av: true node-crc: true @@ -28,7 +27,6 @@ minimumReleaseAgeExclude: onlyBuiltDependencies: - '@discordjs/opus' - '@lng2004/node-datachannel' - - better-sqlite3 - esbuild - node-av - sharp diff --git a/services/backend/src/modules/config/config.routes.ts b/services/backend/src/modules/config/config.routes.ts index 821814b..5e835bb 100644 --- a/services/backend/src/modules/config/config.routes.ts +++ b/services/backend/src/modules/config/config.routes.ts @@ -9,6 +9,18 @@ export function createConfigRouter(): Router { router.get("/config", (_req, res) => { res.json({ monitorGuildId: config.MONITOR_GUILD_ID || null, + webserverPort: config.WEBSERVER_PORT, + nodeEnv: config.NODE_ENV, + backlogSyncHours: config.BACKLOG_SYNC_HOURS, + backlogSyncBatchSize: config.BACKLOG_SYNC_BATCH_SIZE, + retentionMessagesDays: config.RETENTION_MESSAGES_DAYS, + retentionAttachmentsDays: config.RETENTION_ATTACHMENTS_DAYS, + retentionVoiceDays: config.RETENTION_VOICE_DAYS, + autoDeleteFlaggedEnabled: config.AUTO_DELETE_FLAGGED_ENABLED, + aiAnalysisEnabled: config.AI_ANALYSIS_ENABLED, + voiceGuildId: config.VOICE_GUILD_ID || null, + voiceChannelId: config.VOICE_CHANNEL_ID || null, + logLevel: config.LOG_LEVEL, }); }); diff --git a/services/backend/src/modules/health/health.controller.ts b/services/backend/src/modules/health/health.controller.ts index 0f6a638..41d9437 100644 --- a/services/backend/src/modules/health/health.controller.ts +++ b/services/backend/src/modules/health/health.controller.ts @@ -1,4 +1,5 @@ import type { Request, Response } from "express"; +import { register } from "prom-client"; import { asyncHandler } from "../../shared/middlewares/index.js"; import { healthService } from "./health.service.js"; @@ -10,3 +11,10 @@ export const handleHealthCheck = asyncHandler( res.status(status).json(result); }, ); + +export const handleMetrics = asyncHandler( + async (_req: Request, res: Response) => { + res.set("Content-Type", register.contentType); + res.end(await register.metrics()); + }, +); diff --git a/services/backend/src/modules/health/health.routes.ts b/services/backend/src/modules/health/health.routes.ts index f4ebb37..b9bf54e 100644 --- a/services/backend/src/modules/health/health.routes.ts +++ b/services/backend/src/modules/health/health.routes.ts @@ -1,6 +1,11 @@ +import { collectDefaultMetrics, register } from "prom-client"; import type { Router } from "express"; import express from "express"; -import { handleHealthCheck } from "./health.controller.js"; +import { handleHealthCheck, handleMetrics } from "./health.controller.js"; + +// Initialize default Node.js runtime metrics (event loop lag, memory, GC, etc.) +// Called once at module load, not per-request. +collectDefaultMetrics(); export function createHealthRouter(): Router { const router = express.Router(); @@ -8,5 +13,8 @@ export function createHealthRouter(): Router { // GET /api/health router.get("/health", handleHealthCheck); + // GET /api/metrics — Prometheus scrape endpoint + router.get("/metrics", handleMetrics); + return router; } diff --git a/services/backend/src/modules/voice/guilds.routes.ts b/services/backend/src/modules/voice/guilds.routes.ts index cfc20ca..255833b 100644 --- a/services/backend/src/modules/voice/guilds.routes.ts +++ b/services/backend/src/modules/voice/guilds.routes.ts @@ -2,7 +2,11 @@ import { createChildLogger } from "@bete/shared/logger"; import type { Request, Response, Router } from "express"; import express from "express"; import { asyncHandler } from "../../shared/middlewares/index.js"; -import { getGuilds, getTextChannels } from "./voice.service.js"; +import { + getGuilds, + getTextChannels, + getVoiceChannels, +} from "./voice.service.js"; const logger = createChildLogger("guilds.routes"); @@ -32,5 +36,18 @@ export function createGuildsRouter(): Router { }), ); + // GET /api/guilds/:guildId/voice-channels + router.get( + "/:guildId/voice-channels", + asyncHandler(async (req: Request, res: Response) => { + const guildId = Array.isArray(req.params.guildId) + ? req.params.guildId[0] + : req.params.guildId; + logger.debug({ guildId }, "Fetching voice channels"); + const channels = await getVoiceChannels(guildId); + res.json(channels); + }), + ); + return router; } diff --git a/services/backend/src/modules/voice/voice.routes.ts b/services/backend/src/modules/voice/voice.routes.ts index 65d46b0..f388874 100644 --- a/services/backend/src/modules/voice/voice.routes.ts +++ b/services/backend/src/modules/voice/voice.routes.ts @@ -3,7 +3,6 @@ import express from "express"; import { handleConnectVoice, handleDisconnectVoice, - handleGetVoiceChannels, handleGetVoiceStatus, handleVoiceCommand, } from "./voice.controller.js"; @@ -20,9 +19,6 @@ export function createVoiceRouter(): Router { // POST /api/disconnect router.post("/disconnect", handleDisconnectVoice); - // GET /api/guilds/:guildId/voice-channels - router.get("/guilds/:guildId/voice-channels", handleGetVoiceChannels); - // POST /api/voice/command — send arbitrary voice command (transmit start/stop) router.post("/voice/command", handleVoiceCommand); diff --git a/services/backend/tests/placeholder.test.ts b/services/backend/tests/placeholder.test.ts index dee1837..2b1d45b 100644 --- a/services/backend/tests/placeholder.test.ts +++ b/services/backend/tests/placeholder.test.ts @@ -1,7 +1,268 @@ -import { describe, expect, it } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -describe("backend", () => { - it("should load without errors", () => { - expect(true).toBe(true); +// ─── Shared Error Classes ──────────────────────────────────────────────────── +import { + AppError, + NotFoundError, + ValidationError, + UnauthorizedError, + DatabaseError, + ConfigError, +} from "@bete/shared/errors"; + +// ─── Shared utilities ───────────────────────────────────────────────────────── +import { delay, retryWithBackoff, encodeCursor, decodeCursor, pageResult } from "@bete/shared/utils"; + +// ─── Backend middleware ────────────────────────────────────────────────────── +import { asyncHandler, requireParam } from "../src/shared/middlewares/index.js"; + +// ═══════════════════════════════════════════════════════════════════════════════ +// 1. AppError / Error Hierarchy Tests +// ═══════════════════════════════════════════════════════════════════════════════ +describe("AppError subclasses", () => { + it("AppError stores message, code, statusCode, and details", () => { + const err = new AppError("custom", "CUSTOM", 418, { reason: "teapot" }); + expect(err).toBeInstanceOf(Error); + expect(err.message).toBe("custom"); + expect(err.code).toBe("CUSTOM"); + expect(err.statusCode).toBe(418); + expect(err.details).toEqual({ reason: "teapot" }); + expect(err.name).toBe("AppError"); + }); + + it("AppError defaults statusCode to 500", () => { + const err = new AppError("msg", "X"); + expect(err.statusCode).toBe(500); + }); + + it("NotFoundError has 404 status and formatted message", () => { + const err = new NotFoundError("User"); + expect(err).toBeInstanceOf(AppError); + expect(err.statusCode).toBe(404); + expect(err.code).toBe("NOT_FOUND"); + expect(err.message).toBe("User not found"); + expect(err.name).toBe("NotFoundError"); + }); + + it("NotFoundError appends id when provided", () => { + const err = new NotFoundError("Message", "abc-123"); + expect(err.message).toBe("Message not found: abc-123"); + }); + + it("ValidationError has 400 status and forwards details", () => { + const details = { field: "email" }; + const err = new ValidationError("Invalid input", details); + expect(err).toBeInstanceOf(AppError); + expect(err.statusCode).toBe(400); + expect(err.code).toBe("VALIDATION_ERROR"); + expect(err.details).toBe(details); + expect(err.name).toBe("ValidationError"); + }); + + it("UnauthorizedError has 401 status and default message", () => { + const err = new UnauthorizedError(); + expect(err.statusCode).toBe(401); + expect(err.code).toBe("UNAUTHORIZED"); + expect(err.message).toBe("Unauthorized"); + }); + + it("UnauthorizedError accepts custom message", () => { + const err = new UnauthorizedError("Access denied"); + expect(err.message).toBe("Access denied"); + }); + + it("DatabaseError has 500 status and forwards details", () => { + const err = new DatabaseError("DB down", { cause: "timeout" }); + expect(err.statusCode).toBe(500); + expect(err.code).toBe("DATABASE_ERROR"); + expect(err.details).toEqual({ cause: "timeout" }); + }); + + it("ConfigError has 500 status", () => { + const err = new ConfigError("Bad config"); + expect(err.statusCode).toBe(500); + expect(err.code).toBe("CONFIG_ERROR"); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════ +// 2. Utility Function Tests +// ═══════════════════════════════════════════════════════════════════════════════ +describe("delay", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("resolves after the given time", async () => { + vi.useFakeTimers(); + const promise = delay(500); + vi.advanceTimersByTime(500); + await expect(promise).resolves.toBeUndefined(); + }); + + it("rejects are not triggered on non-matching timer", async () => { + vi.useFakeTimers(); + const promise = delay(1000); + // Advance only part way — the timer should NOT fire yet + vi.advanceTimersByTime(500); + // The timer is still pending; the promise has not resolved yet + // We advance the rest + vi.advanceTimersByTime(500); + await expect(promise).resolves.toBeUndefined(); + }); +}); + +describe("retryWithBackoff", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("returns the result on first success without retrying", async () => { + const fn = vi.fn().mockResolvedValue("ok"); + await expect(retryWithBackoff(fn)).resolves.toBe("ok"); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it("re-throws after exhausting all retries", async () => { + const fn = vi.fn().mockRejectedValue(new Error("persistent")); + await expect( + retryWithBackoff(fn, { retries: 1, minTimeout: 1, maxTimeout: 5 }), + ).rejects.toThrow("persistent"); + // initial call + 1 retry + expect(fn.mock.calls.length).toBeGreaterThanOrEqual(2); + }); + + it("throws AbortError immediately when signal is already aborted", async () => { + const ac = new AbortController(); + ac.abort(); + const fn = vi.fn().mockResolvedValue("ok"); + await expect( + retryWithBackoff(fn, { retries: 3, signal: ac.signal }), + ).rejects.toThrow("Aborted"); + expect(fn).not.toHaveBeenCalled(); + }); + + it("respects abort signal during retry", async () => { + vi.useFakeTimers(); + const ac = new AbortController(); + const fn = vi.fn().mockRejectedValue(new Error("fail")); + + const promise = retryWithBackoff(fn, { + retries: 5, + minTimeout: 100, + signal: ac.signal, + }); + + // Schedule abort after first failure + backoff starts + setTimeout(() => ac.abort(), 150); + vi.advanceTimersByTime(200); + await vi.waitFor(async () => { + await expect(promise).rejects.toThrow("Aborted"); + }); + }); +}); + +describe("pagination utilities", () => { + it("encodeCursor produces a base64 string", () => { + const result = encodeCursor({ created_at: 1000, id: "msg-1" }); + expect(typeof result).toBe("string"); + expect(result.length).toBeGreaterThan(0); + }); + + it("encodeCursor round-trips through decodeCursor", () => { + const data = { created_at: 1234567890, id: "abc-def" }; + const cursor = encodeCursor(data); + expect(decodeCursor(cursor)).toEqual(data); + }); + + it("decodeCursor returns null for undefined / empty", () => { + expect(decodeCursor()).toBeNull(); + expect(decodeCursor("")).toBeNull(); + }); + + it("decodeCursor returns null for malformed input", () => { + // Completely invalid base64 + expect(decodeCursor("!!!not-valid!!!")).toBeNull(); + // Valid base64 but not JSON + const notJson = Buffer.from("not-json").toString("base64"); + expect(decodeCursor(notJson)).toBeNull(); + // Valid JSON but wrong shape (missing created_at / id) + const wrongShape = Buffer.from(JSON.stringify({ foo: "bar" })).toString("base64"); + expect(decodeCursor(wrongShape)).toBeNull(); + }); + + it("pageResult truncates and sets nextCursor when rows exceed limit", () => { + const rows = [ + { id: "a", created_at: 100 }, + { id: "b", created_at: 200 }, + { id: "c", created_at: 300 }, + ]; + const { data, nextCursor } = pageResult(rows, 2); + expect(data).toHaveLength(2); + expect(data[0].id).toBe("a"); + expect(nextCursor).toBeTruthy(); + }); + + it("pageResult returns null nextCursor when fewer rows than limit", () => { + const rows = [{ id: "a", created_at: 100 }]; + const { data, nextCursor } = pageResult(rows, 2); + expect(data).toHaveLength(1); + expect(nextCursor).toBeNull(); + }); + + it("pageResult returns empty data for empty input", () => { + const { data, nextCursor } = pageResult([], 10); + expect(data).toEqual([]); + expect(nextCursor).toBeNull(); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════ +// 3. Middleware Tests +// ═══════════════════════════════════════════════════════════════════════════════ +describe("asyncHandler", () => { + it("passes thrown errors to next()", async () => { + const error = new Error("handler-error"); + const wrapped = asyncHandler(async () => { + throw error; + }); + const next = vi.fn(); + + wrapped({} as any, {} as any, next); + + // .catch(next) is a microtask — flush the queue + await Promise.resolve(); + await Promise.resolve(); + expect(next).toHaveBeenCalledWith(error); + }); + + it("does not call next when handler resolves successfully", async () => { + const wrapped = asyncHandler(async (_req: any, _res: any, _next: any) => { + // no-op + }); + const next = vi.fn(); + + wrapped({} as any, {} as any, next); + await Promise.resolve(); + await Promise.resolve(); + expect(next).not.toHaveBeenCalled(); + }); +}); + +describe("requireParam", () => { + it("returns the value for a non-empty string", () => { + expect(requireParam("hello", "param", "name")).toBe("hello"); + }); + + it("throws ValidationError for undefined", () => { + expect(() => requireParam(undefined, "query", "q")).toThrow(ValidationError); + }); + + it("throws ValidationError for empty string", () => { + expect(() => requireParam("", "param", "id")).toThrow(ValidationError); + }); + + it("throws with a descriptive message", () => { + expect(() => requireParam(null, "header", "X-Token")).toThrow("Missing header: X-Token"); }); }); diff --git a/services/discord-gateway/src/app/bootstrap.ts b/services/discord-gateway/src/app/bootstrap.ts index 9603fa0..2dd5895 100644 --- a/services/discord-gateway/src/app/bootstrap.ts +++ b/services/discord-gateway/src/app/bootstrap.ts @@ -1,6 +1,8 @@ import { ConfigError, DatabaseError } from "@bete/shared/errors"; import { createChildLogger } from "@bete/shared/logger"; import { Client } from "discord.js-selfbot-v13"; +import { inArray, lt } from "drizzle-orm"; +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; import { startPendingAIAnalysisWorker } from "../modules/ai-moderation/aiAnalyzer.js"; import { CommandHandler } from "../modules/command-handler/commandHandler.js"; import { @@ -11,19 +13,174 @@ import { registerMessageCapture, setEventBroadcaster as setMessageCaptureEventBroadcaster, } from "../modules/message-capture/messageCapture.js"; +import { getExpiredMessages } from "../modules/message-capture/messageStore.js"; import { setEventBroadcaster as setRecorderEventBroadcaster } from "../modules/voice-recording/recorder.js"; import { VoiceController } from "../modules/voice-recording/voiceController.js"; import { config } from "../shared/config/config.js"; import { closeDatabase, + getDatabase, initializeDatabase, } from "../shared/database/drizzle.js"; import { runMigrations } from "../shared/database/migrate.js"; +import type * as schema from "../shared/database/schema.js"; +import { + attachmentsTable, + messagesTable, + voiceRecordingsTable, +} from "../shared/database/schema.js"; import { createDiscordClientOptions } from "../shared/discord/clientOptions.js"; import { createGracefulShutdown } from "./shutdown.js"; const logger = createChildLogger("discord-gateway"); +// ─── Retention Cleanup ───────────────────────────────────────────────────── + +function startRetentionCleanup(): void { + const intervalMs = config.RETENTION_CLEANUP_INTERVAL_MS; + const dryRun = config.RETENTION_DRY_RUN; + + logger.info( + { + intervalMs, + dryRun, + messagesDays: config.RETENTION_MESSAGES_DAYS, + attachmentsDays: config.RETENTION_ATTACHMENTS_DAYS, + voiceDays: config.RETENTION_VOICE_DAYS, + }, + "Starting retention cleanup scheduler", + ); + + async function runCleanupTick(): Promise { + const db = getDatabase() as unknown as NodePgDatabase; + + // ── Expired messages ──────────────────────────────────────────────── + if (config.RETENTION_MESSAGES_DAYS > 0) { + try { + const expiredMessages = await getExpiredMessages( + config.RETENTION_MESSAGES_DAYS, + ); + + if (expiredMessages.length > 0) { + const ids = expiredMessages.map((m: { id: string }) => m.id); + logger.info( + { count: ids.length, dryRun }, + "Expired messages found for cleanup", + ); + + if (!dryRun) { + await db + .delete(messagesTable) + .where(inArray(messagesTable.id, ids)); + logger.info({ count: ids.length }, "Expired messages deleted"); + } + } + } catch (error) { + logger.error( + { + error: error instanceof Error ? error.message : String(error), + }, + "Failed to clean up expired messages", + ); + } + } + + // ── Expired attachments ───────────────────────────────────────────── + if (config.RETENTION_ATTACHMENTS_DAYS > 0) { + try { + const cutoff = + Date.now() - config.RETENTION_ATTACHMENTS_DAYS * 24 * 60 * 60 * 1000; + + const expiredAttachments = await db + .select({ id: attachmentsTable.id }) + .from(attachmentsTable) + .where(lt(attachmentsTable.created_at, cutoff)) + .limit(1000); + + if (expiredAttachments.length > 0) { + const ids = expiredAttachments.map((a: { id: string }) => a.id); + logger.info( + { count: ids.length, dryRun }, + "Expired attachments found for cleanup", + ); + + if (!dryRun) { + await db + .delete(attachmentsTable) + .where(inArray(attachmentsTable.id, ids)); + logger.info({ count: ids.length }, "Expired attachments deleted"); + } + } + } catch (error) { + logger.error( + { + error: error instanceof Error ? error.message : String(error), + }, + "Failed to clean up expired attachments", + ); + } + } + + // ── Expired voice recordings ──────────────────────────────────────── + if (config.RETENTION_VOICE_DAYS > 0) { + try { + const cutoff = + Date.now() - config.RETENTION_VOICE_DAYS * 24 * 60 * 60 * 1000; + + const expiredRecordings = await db + .select({ id: voiceRecordingsTable.id }) + .from(voiceRecordingsTable) + .where(lt(voiceRecordingsTable.created_at, cutoff)) + .limit(1000); + + if (expiredRecordings.length > 0) { + const ids = expiredRecordings.map((r: { id: string }) => r.id); + logger.info( + { count: ids.length, dryRun }, + "Expired voice recordings found for cleanup", + ); + + if (!dryRun) { + await db + .delete(voiceRecordingsTable) + .where(inArray(voiceRecordingsTable.id, ids)); + logger.info( + { count: ids.length }, + "Expired voice recordings deleted", + ); + } + } + } catch (error) { + logger.error( + { + error: error instanceof Error ? error.message : String(error), + }, + "Failed to clean up expired voice recordings", + ); + } + } + } + + // Run immediately on start, then schedule + runCleanupTick().catch((error) => { + logger.error( + { error: error instanceof Error ? error.message : String(error) }, + "Initial retention cleanup tick failed", + ); + }); + + setInterval(() => { + runCleanupTick().catch((error) => { + logger.error( + { error: error instanceof Error ? error.message : String(error) }, + "Retention cleanup tick failed", + ); + }); + }, intervalMs); +} + +// ─── Bootstrap ───────────────────────────────────────────────────────────── + export async function initializeDiscordGateway() { if (config.AI_ANALYSIS_ENABLED && !config.AI_LLM_API_KEY) { throw new ConfigError( @@ -98,6 +255,9 @@ export async function initializeDiscordGateway() { // Start command handler after Discord is ready commandHandler.start(client, voiceController); logger.info("Command handler started"); + + // Start retention cleanup scheduler + startRetentionCleanup(); }); client.on("error", (err) => { diff --git a/services/discord-gateway/src/modules/command-handler/media.handler.ts b/services/discord-gateway/src/modules/command-handler/media.handler.ts index f1d48c5..1beedcd 100644 --- a/services/discord-gateway/src/modules/command-handler/media.handler.ts +++ b/services/discord-gateway/src/modules/command-handler/media.handler.ts @@ -1,15 +1,23 @@ import { type CommandMessage, type CommandReply } from "@bete/shared"; import { createChildLogger } from "@bete/shared/logger"; +import { StreamType } from "@discordjs/voice"; +import { resolveMediaUrl } from "../voice-recording/mediaSource.js"; import { discordPlayer } from "../voice-recording/player.js"; // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- +interface CurrentTrack { + title: string; + url: string; + duration?: number; +} + export interface MediaStatusPayload { playing: boolean; musicVolume: number; - current: unknown; + current: CurrentTrack | null; queue: unknown[]; } @@ -19,29 +27,84 @@ export interface MediaStatusPayload { export class MediaHandler { private logger = createChildLogger("media-handler"); + private currentTrack: CurrentTrack | null = null; getCurrentMediaStatus(): MediaStatusPayload { return { playing: discordPlayer.getStatus() === "playing", musicVolume: discordPlayer.getMusicVolume(), - current: null, + current: this.currentTrack, queue: [], }; } async handleMediaQueue(cmd: CommandMessage): Promise> { - this.logger.info( - "media:queue received — media queueing is handled externally", - ); - return { - id: cmd.id, - success: true, - data: this.getCurrentMediaStatus(), - }; + const url = String(cmd.payload.url ?? "").trim(); + + if (!url) { + this.logger.warn("media:queue received without a URL"); + return { + id: cmd.id, + success: false, + data: null, + error: "url is required", + }; + } + + if (!discordPlayer.isConnected()) { + this.logger.warn( + "media:queue attempted without an active voice connection", + ); + return { + id: cmd.id, + success: false, + data: null, + error: "Not connected to a voice channel. Connect to voice first.", + }; + } + + try { + this.logger.info({ url }, "Resolving media URL"); + + const resolution = await resolveMediaUrl(url); + + this.currentTrack = { + title: resolution.title ?? url, + url, + duration: resolution.duration, + }; + + discordPlayer.playStream(resolution.stream, "music", { + inputType: StreamType.Arbitrary, + inlineVolume: true, + volume: discordPlayer.getMusicVolume(), + }); + + this.logger.info( + { url, title: resolution.title }, + "Media queued and playback started", + ); + + return { + id: cmd.id, + success: true, + data: this.getCurrentMediaStatus(), + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + this.logger.error({ error: message, url }, "Failed to queue media"); + return { + id: cmd.id, + success: false, + data: null, + error: message, + }; + } } async handleMediaSkip(cmd: CommandMessage): Promise> { discordPlayer.stop("music"); + this.currentTrack = null; return { id: cmd.id, success: true, @@ -51,6 +114,7 @@ export class MediaHandler { async handleMediaStop(cmd: CommandMessage): Promise> { discordPlayer.stop("music"); + this.currentTrack = null; return { id: cmd.id, success: true, diff --git a/services/discord-gateway/src/modules/command-handler/voice.handler.ts b/services/discord-gateway/src/modules/command-handler/voice.handler.ts index 5c8f3f5..1ede355 100644 --- a/services/discord-gateway/src/modules/command-handler/voice.handler.ts +++ b/services/discord-gateway/src/modules/command-handler/voice.handler.ts @@ -111,67 +111,6 @@ export class VoiceHandler { } } - async handleGuildsList(cmd: CommandMessage): Promise> { - if (!this.client) { - return { - id: cmd.id, - success: false, - data: null, - error: "Gateway not initialized", - }; - } - - try { - const guilds = this.client.guilds.cache - .map((guild) => ({ id: guild.id, name: guild.name })) - .sort((a, b) => a.name.localeCompare(b.name)); - return { id: cmd.id, success: true, data: guilds }; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - return { id: cmd.id, success: false, data: null, error: msg }; - } - } - - async handleWatchableChannels( - cmd: CommandMessage, - ): Promise> { - if (!this.client) { - return { - id: cmd.id, - success: false, - data: null, - error: "Gateway not initialized", - }; - } - - const guildId = String(cmd.payload.guildId ?? ""); - if (!guildId) { - return { - id: cmd.id, - success: false, - data: null, - error: "guildId is required", - }; - } - - try { - const guild = await this.client.guilds.fetch(guildId); - const channels = await guild.channels.fetch(); - const textChannels = channels - .filter((c) => c?.type === "GUILD_TEXT") - .map((c) => ({ - id: c.id, - name: c.name, - type: c.type, - })); - - return { id: cmd.id, success: true, data: textChannels }; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - return { id: cmd.id, success: false, data: null, error: msg }; - } - } - async handleVoiceTransmitStart( cmd: CommandMessage, ): Promise> { diff --git a/services/discord-gateway/src/modules/voice-recording/mediaSource.ts b/services/discord-gateway/src/modules/voice-recording/mediaSource.ts new file mode 100644 index 0000000..7442d89 --- /dev/null +++ b/services/discord-gateway/src/modules/voice-recording/mediaSource.ts @@ -0,0 +1,374 @@ +import { type ChildProcess, spawn } from "node:child_process"; +import { PassThrough, Readable } from "node:stream"; +import { createChildLogger } from "@bete/shared/logger"; +import { StreamType } from "@discordjs/voice"; + +const logger = createChildLogger("media-source"); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface MediaInfo { + title: string; + duration: number; + uploader?: string; + thumbnail?: string; +} + +export interface MediaSourceResolution { + stream: Readable; + type: StreamType; + title?: string; + duration?: number; + info: MediaInfo; +} + +export interface ResolveOptions { + /** Timeout in milliseconds for the yt-dlp process. */ + timeout?: number; + /** + * yt-dlp format string override (e.g. "bestaudio[ext=m4a]"). + * Defaults to "bestaudio". + */ + quality?: string; +} + +// --------------------------------------------------------------------------- +// Internal state +// --------------------------------------------------------------------------- + +/** Tracks all spawned yt-dlp child processes for shutdown cleanup. */ +const activeProcesses: Set = new Set(); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function parseSeconds(value: string): number { + const n = Number.parseFloat(value); + return Number.isFinite(n) ? n : 0; +} + +/** + * Read the first two newline-delimited lines from a Readable stdout stream. + * + * yt-dlp with `--print before_dl:title --print before_dl:duration` outputs: + * line 1: video title + * line 2: duration in seconds (float) + * rest: raw binary audio data + * + * Returns the parsed header and a new Readable that contains all remaining + * data (the audio stream). + */ +function readFirstTwoLines(stdout: Readable): Promise<{ + title: string; + duration: number; + remaining: Readable; +}> { + return new Promise((resolve, reject) => { + const passThrough = new PassThrough(); + + let buffer = Buffer.alloc(0); + let title = ""; + let stage: "title" | "duration" | "done" = "title"; + + function cleanup() { + stdout.removeListener("data", onData); + stdout.removeListener("error", onError); + stdout.removeListener("end", onEnd); + } + + function onData(chunk: Buffer) { + if (stage === "done") return; + buffer = Buffer.concat([buffer, chunk]); + processBuffer(); + } + + function processBuffer() { + while (buffer.length > 0 && stage !== "done") { + const nl = buffer.indexOf(0x0a); // '\n' byte + if (nl === -1) break; // Need more data + + const line = buffer.subarray(0, nl).toString("utf8").trim(); + buffer = buffer.subarray(nl + 1); + + if (stage === "title") { + title = line; + stage = "duration"; + } else if (stage === "duration") { + const duration = parseSeconds(line); + stage = "done"; + cleanup(); + + // Write any buffered data that follows the second newline + if (buffer.length > 0) { + passThrough.write(buffer); + } + + // Pipe the remainder of stdout into the pass-through + stdout.pipe(passThrough); + + resolve({ title, duration, remaining: passThrough }); + return; + } + } + } + + function onError(err: Error) { + if (stage !== "done") { + cleanup(); + reject(err); + } + } + + function onEnd() { + if (stage !== "done") { + cleanup(); + reject( + new Error( + `yt-dlp stdout ended before metadata could be read. ` + + `Stage: ${stage}, partial title: "${title}"`, + ), + ); + } + } + + stdout.on("data", onData); + stdout.on("error", onError); + stdout.on("end", onEnd); + }); +} + +function buildNotInstalledError(): Error { + return new Error( + "yt-dlp is not installed or not found in PATH. " + + 'Run "pnpm run install:yt-dlp" to install it.', + ); +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Resolve a media URL (YouTube, Spotify, etc.) to a playable audio stream. + * + * Spawns `yt-dlp`, extracts the title and duration from the first two stdout + * lines, then pipes the remaining raw audio data into a Readable stream. + * + * The returned stream uses `StreamType.Arbitrary` — suitable for + * `DiscordPlayer.playStream()` with `inputType: StreamType.Arbitrary`. + * + * @throws If yt-dlp is not installed or the process exits with a non-zero code + * before the metadata headers have been parsed. + */ +export function resolveMediaUrl( + url: string, + options?: ResolveOptions, +): Promise { + return new Promise((resolve, reject) => { + const format = options?.quality ?? "bestaudio"; + const args = [ + "-f", + format, + "--audio-format", + "best", + "-o", + "-", + "--print", + "before_dl:title", + "--print", + "before_dl:duration", + url, + ]; + + logger.info({ url }, "Spawning yt-dlp for media resolution"); + + const proc = spawn("yt-dlp", args, { + stdio: ["pipe", "pipe", "pipe"], + }); + + activeProcesses.add(proc); + + let stderrBuf = ""; + let resolved = false; + + // -- helpers ----------------------------------------------------------- + + const failOnce = (err: Error) => { + if (resolved) return; + resolved = true; + activeProcesses.delete(proc); + reject(err); + }; + + // -- spawn error (ENOENT etc.) ---------------------------------------- + + proc.on("error", (err: NodeJS.ErrnoException) => { + if (err.code === "ENOENT") { + failOnce(buildNotInstalledError()); + } else { + failOnce(new Error(`yt-dlp failed to start: ${err.message}`)); + } + }); + + // -- stderr (capture for diagnostics) ---------------------------------- + + if (proc.stderr) { + proc.stderr.on("data", (chunk: Buffer) => { + stderrBuf += chunk.toString("utf8"); + }); + } + + // -- stdout: parse header, then stream audio --------------------------- + + readFirstTwoLines(proc.stdout) + .then(({ title, duration, remaining }) => { + if (resolved) return; + resolved = true; + activeProcesses.delete(proc); + + const info: MediaInfo = { title, duration }; + resolve({ + stream: remaining, + type: StreamType.Arbitrary, + title, + duration, + info, + }); + }) + .catch((err: Error) => { + failOnce(err); + }); + + // -- process exit (non-zero means failure) ----------------------------- + + proc.on("close", (code, signal) => { + activeProcesses.delete(proc); + + if (resolved) return; + + if (code !== null && code !== 0) { + const detail = stderrBuf.trim() ? `: ${stderrBuf.trim()}` : ""; + failOnce(new Error(`yt-dlp exited with code ${code}${detail}`)); + } else if (signal) { + failOnce(new Error(`yt-dlp was killed by signal ${signal}`)); + } + }); + + // -- optional timeout -------------------------------------------------- + + if (options?.timeout && options.timeout > 0) { + const timer = setTimeout(() => { + if (resolved) return; + logger.warn({ url, timeout: options.timeout }, "yt-dlp timed out"); + proc.kill("SIGTERM"); + failOnce(new Error(`yt-dlp timed out after ${options.timeout}ms`)); + }, options.timeout); + + proc.once("close", () => clearTimeout(timer)); + } + }); +} + +/** + * Extract metadata (title, duration, uploader, thumbnail) from a media URL + * without downloading the audio stream. + * + * Uses `yt-dlp --dump-json` and parses the JSON output. + * + * @throws If yt-dlp is not installed or the process exits with a non-zero + * code. + */ +export async function extractMediaInfo(url: string): Promise { + return new Promise((resolve, reject) => { + const args = ["--dump-json", "--no-warnings", url]; + + logger.debug({ url }, "Spawning yt-dlp for metadata extraction"); + + const proc = spawn("yt-dlp", args, { + stdio: ["pipe", "pipe", "pipe"], + }); + + activeProcesses.add(proc); + + let stdoutBuf = ""; + let stderrBuf = ""; + + if (proc.stdout) { + proc.stdout.on("data", (chunk: Buffer) => { + stdoutBuf += chunk.toString("utf8"); + }); + } + + if (proc.stderr) { + proc.stderr.on("data", (chunk: Buffer) => { + stderrBuf += chunk.toString("utf8"); + }); + } + + proc.on("error", (err: NodeJS.ErrnoException) => { + activeProcesses.delete(proc); + if (err.code === "ENOENT") { + reject(buildNotInstalledError()); + } else { + reject(new Error(`yt-dlp failed to start: ${err.message}`)); + } + }); + + proc.on("close", (code) => { + activeProcesses.delete(proc); + + if (code !== 0) { + const detail = stderrBuf.trim() ? `: ${stderrBuf.trim()}` : ""; + reject( + new Error( + `yt-dlp metadata extraction exited with code ${code}${detail}`, + ), + ); + return; + } + + try { + const raw = JSON.parse(stdoutBuf.trim()) as Record; + resolve({ + title: String(raw.title ?? url), + duration: typeof raw.duration === "number" ? raw.duration : 0, + uploader: String(raw.uploader ?? raw.channel ?? "") || undefined, + thumbnail: String(raw.thumbnail ?? "") || undefined, + }); + } catch (parseErr) { + reject( + new Error( + `Failed to parse yt-dlp JSON output: ${(parseErr as Error).message}`, + ), + ); + } + }); + }); +} + +/** + * Kill all active yt-dlp child processes. + * + * Call during graceful shutdown to ensure no orphan processes remain. + */ +export function cleanup(): void { + if (activeProcesses.size === 0) return; + + logger.info( + { count: activeProcesses.size }, + "Killing active yt-dlp processes", + ); + + for (const proc of activeProcesses) { + try { + proc.kill("SIGTERM"); + } catch { + // Process may already be dead — ignore + } + } + + activeProcesses.clear(); +} diff --git a/services/discord-gateway/tests/placeholder.test.ts b/services/discord-gateway/tests/placeholder.test.ts index dbe9144..804f479 100644 --- a/services/discord-gateway/tests/placeholder.test.ts +++ b/services/discord-gateway/tests/placeholder.test.ts @@ -1,7 +1,417 @@ -import { describe, expect, it } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; -describe("discord-gateway", () => { - it("should load without errors", () => { - expect(true).toBe(true); +// ═══════════════════════════════════════════════════════════════════════════════ +// 1. AppError Hierarchy +// ═══════════════════════════════════════════════════════════════════════════════ +import { + AppError, + NotFoundError, + ValidationError, + UnauthorizedError, + DatabaseError, + ConfigError, +} from "@bete/shared/errors"; + +describe("AppError subclasses", () => { + it("AppError carries code, statusCode, and details", () => { + const err = new AppError("err", "X", 500, { info: "test" }); + expect(err).toBeInstanceOf(Error); + expect(err.code).toBe("X"); + expect(err.statusCode).toBe(500); + expect(err.details).toEqual({ info: "test" }); + }); + + it("NotFoundError sets 404 status", () => { + expect(new NotFoundError("R").statusCode).toBe(404); + expect(new NotFoundError("R").code).toBe("NOT_FOUND"); + }); + + it("ValidationError sets 400 status", () => { + expect(new ValidationError("V").statusCode).toBe(400); + expect(new ValidationError("V").code).toBe("VALIDATION_ERROR"); + }); + + it("UnauthorizedError sets 401 status", () => { + expect(new UnauthorizedError().statusCode).toBe(401); + }); + + it("DatabaseError sets 500 status", () => { + expect(new DatabaseError("X").statusCode).toBe(500); + }); + + it("ConfigError sets 500 status", () => { + expect(new ConfigError("X").statusCode).toBe(500); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════ +// 2. Shared Utilities +// ═══════════════════════════════════════════════════════════════════════════════ +import { delay, retryWithBackoff, encodeCursor, decodeCursor, pageResult } from "@bete/shared/utils"; + +describe("delay", () => { + afterEach(() => { vi.useRealTimers(); }); + + it("resolves after specified time with fake timers", async () => { + vi.useFakeTimers(); + const p = delay(250); + vi.advanceTimersByTime(250); + await expect(p).resolves.toBeUndefined(); + }); +}); + +describe("retryWithBackoff", () => { + afterEach(() => { vi.useRealTimers(); }); + + it("resolves on first attempt", async () => { + const fn = vi.fn().mockResolvedValue(42); + await expect(retryWithBackoff(fn)).resolves.toBe(42); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it("throws after retries are exhausted", async () => { + const fn = vi.fn().mockRejectedValue(new Error("fail")); + await expect( + retryWithBackoff(fn, { retries: 1, minTimeout: 1, maxTimeout: 5 }), + ).rejects.toThrow("fail"); + expect(fn.mock.calls.length).toBeGreaterThanOrEqual(2); + }); + + it("rejects immediately when already aborted", async () => { + const ac = new AbortController(); + ac.abort(); + await expect( + retryWithBackoff(() => Promise.resolve("ok"), { signal: ac.signal }), + ).rejects.toThrow("Aborted"); + }); +}); + +describe("pagination utils", () => { + it("encode/decode round-trips correctly", () => { + const data = { created_at: 999, id: "id-1" }; + expect(decodeCursor(encodeCursor(data))).toEqual(data); + }); + + it("decodeCursor rejects invalid input with null", () => { + expect(decodeCursor()).toBeNull(); + expect(decodeCursor("")).toBeNull(); + expect(decodeCursor("!!!")).toBeNull(); + }); + + it("pageResult handles hasMore and no-more cases", () => { + const r1 = pageResult([{ id: "a", created_at: 1 }], 5); + expect(r1.nextCursor).toBeNull(); + + const r2 = pageResult( + [{ id: "a", created_at: 1 }, { id: "b", created_at: 2 }], + 1, + ); + expect(r2.data).toHaveLength(1); + expect(r2.nextCursor).toBeTruthy(); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════ +// 3. Redis Channel Constants +// ═══════════════════════════════════════════════════════════════════════════════ +import { + DISCORD_MESSAGE_CREATED, + DISCORD_MESSAGE_UPDATED, + DISCORD_MESSAGE_DELETED, + DISCORD_MESSAGE_ANALYZED, + DISCORD_ATTACHMENT_CREATED, + DISCORD_VOICE_STARTED, + DISCORD_VOICE_PCM, + DISCORD_ANALYSIS_QUEUE_STATUS, + BACKEND_COMMAND, + VOICE_STATUS_KEY, + MEDIA_STATUS_KEY, + COMMAND_VOICE_CONNECT, + COMMAND_VOICE_DISCONNECT, + COMMAND_GUILDS_LIST, + COMMAND_MEDIA_QUEUE, + COMMAND_MEDIA_SKIP, + COMMAND_MEDIA_STOP, + COMMAND_MEDIA_VOLUME, + COMMAND_MODERATION_ACTION, +} from "@bete/shared/redis-channels"; + +describe("Redis channel constants", () => { + it("define event channel names", () => { + expect(DISCORD_MESSAGE_CREATED).toBe("discord:message:created"); + expect(DISCORD_MESSAGE_UPDATED).toBe("discord:message:updated"); + expect(DISCORD_MESSAGE_DELETED).toBe("discord:message:deleted"); + expect(DISCORD_MESSAGE_ANALYZED).toBe("discord:message:analyzed"); + expect(DISCORD_ATTACHMENT_CREATED).toBe("discord:attachment:created"); + expect(DISCORD_VOICE_STARTED).toBe("discord:voice:started"); + expect(DISCORD_VOICE_PCM).toBe("discord:voice:pcm"); + expect(DISCORD_ANALYSIS_QUEUE_STATUS).toBe("discord:analysis:queue_status"); + }); + + it("define command channel and status keys", () => { + expect(BACKEND_COMMAND).toBe("backend:command"); + expect(VOICE_STATUS_KEY).toBe("voice:status"); + expect(MEDIA_STATUS_KEY).toBe("media:status"); + }); + + it("define command type constants", () => { + expect(COMMAND_VOICE_CONNECT).toBe("voice:connect"); + expect(COMMAND_VOICE_DISCONNECT).toBe("voice:disconnect"); + expect(COMMAND_GUILDS_LIST).toBe("guilds:list"); + expect(COMMAND_MEDIA_QUEUE).toBe("media:queue"); + expect(COMMAND_MEDIA_SKIP).toBe("media:skip"); + expect(COMMAND_MEDIA_STOP).toBe("media:stop"); + expect(COMMAND_MEDIA_VOLUME).toBe("media:volume"); + expect(COMMAND_MODERATION_ACTION).toBe("moderation:action"); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════ +// 4. Config Validation +// ═══════════════════════════════════════════════════════════════════════════════ +vi.hoisted(() => { + process.env.DISCORD_TOKEN = "test-discord-token-for-tests"; + process.env.DATABASE_URL = "postgres://test:test@localhost:5432/test"; +}); + +import { loadConfig, configSchema } from "@bete/shared/config"; + +describe("Config validation", () => { + it("loadConfig succeeds with minimal valid env", () => { + const cfg = loadConfig({ + DISCORD_TOKEN: "abc", + DATABASE_URL: "postgres://localhost/db", + }); + expect(cfg.DISCORD_TOKEN).toBe("abc"); + // Defaults + expect(cfg.RECORDINGS_DIR).toBe("./recordings"); + expect(cfg.RECORDING_SEGMENT_MS).toBe(5000); + expect(cfg.NODE_ENV).toBe("development"); + expect(cfg.WEBSERVER_PORT).toBe(3001); + expect(cfg.OPUS_FRAME_SIZE).toBe(960); + expect(cfg.AUDIO_SAMPLE_RATE).toBe(48000); + expect(cfg.LOG_LEVEL).toBe("info"); + }); + + it("loadConfig throws ConfigError when DISCORD_TOKEN is missing", () => { + expect(() => loadConfig({})).toThrow(ConfigError); + }); + + it("schema parses boolean-string transforms correctly", () => { + const result = configSchema.parse({ + DISCORD_TOKEN: "tok", + DATABASE_URL: "pg://localhost/db", + VERBOSE: "true", + AI_ANALYSIS_ENABLED: "true", + AI_LLM_API_KEY: "sk-test", + }); + expect(result.AI_ANALYSIS_ENABLED).toBe(true); + }); + + it("schema provides sensible default for NODE_ENV", () => { + const result = configSchema.parse({ + DISCORD_TOKEN: "tok", + DATABASE_URL: "pg://localhost/db", + }); + expect(result.NODE_ENV).toBe("development"); + }); + + it("gateway loadConfig adds EFFECTIVE_TEXT_GUILD_ID from MONITOR_GUILD_ID", async () => { + const { loadConfig: gwLoadConfig } = await import("../src/shared/config/config.js"); + const cfg = gwLoadConfig({ + DISCORD_TOKEN: "tok", + DATABASE_URL: "pg://localhost/db", + MONITOR_GUILD_ID: "guild-1", + }); + expect(cfg.EFFECTIVE_TEXT_GUILD_ID).toBe("guild-1"); + }); + + it("gateway loadConfig prefers TEXT_GUILD_ID over MONITOR_GUILD_ID", async () => { + const { loadConfig: gwLoadConfig } = await import("../src/shared/config/config.js"); + const cfg = gwLoadConfig({ + DISCORD_TOKEN: "tok", + DATABASE_URL: "pg://localhost/db", + TEXT_GUILD_ID: "text-guild", + MONITOR_GUILD_ID: "monitor-guild", + }); + expect(cfg.EFFECTIVE_TEXT_GUILD_ID).toBe("text-guild"); + }); +}); + +// ═══════════════════════════════════════════════════════════════════════════════ +// 5. Pure function modules +// ═══════════════════════════════════════════════════════════════════════════════ +import { sniffImageMimeType } from "../src/modules/ai-moderation/imageMimeSniffer.js"; + +describe("sniffImageMimeType", () => { + function buf(...bytes: number[]): Buffer { + const b = Buffer.alloc(12); + for (let i = 0; i < bytes.length; i++) b[i] = bytes[i]; + return b; + } + + it("detects JPEG", () => { + expect(sniffImageMimeType(buf(0xff, 0xd8, 0xff))).toBe("image/jpeg"); + }); + + it("detects PNG", () => { + expect( + sniffImageMimeType(buf(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)), + ).toBe("image/png"); + }); + + it("detects GIF", () => { + expect(sniffImageMimeType(buf(0x47, 0x49, 0x46, 0x38))).toBe("image/gif"); + }); + + it("detects WebP", () => { + expect( + sniffImageMimeType(buf(0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50)), + ).toBe("image/webp"); + }); + + it("detects AVIF", () => { + // ftyp box with avif brand at bytes 8-11 + expect( + sniffImageMimeType(buf(0, 0, 0, 0, 0x66, 0x74, 0x79, 0x70, 0x61, 0x76, 0x69, 0x66)), + ).toBe("image/avif"); + }); + + it("detects HEIC", () => { + // ftyp box with heic brand at bytes 8-11 + expect( + sniffImageMimeType(buf(0, 0, 0, 0, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63)), + ).toBe("image/heic"); + }); + + it("returns null for short buffer (< 12 bytes)", () => { + expect(sniffImageMimeType(Buffer.alloc(3))).toBeNull(); + }); + + it("returns null for unrecognised data", () => { + expect(sniffImageMimeType(Buffer.alloc(12))).toBeNull(); + }); +}); + +import { + clampScore, + deriveSeverity, + deriveRecommendedAction, + hasDeferralAnalysis, +} from "../src/modules/ai-moderation/severityDeriver.js"; + +describe("severityDeriver", () => { + describe("clampScore", () => { + it("clamps values between 0 and 1", () => { + expect(clampScore(-0.5)).toBe(0); + expect(clampScore(0.5)).toBe(0.5); + expect(clampScore(1.5)).toBe(1); + }); + + it("handles undefined and NaN with fallback", () => { + expect(clampScore(undefined)).toBe(0); + expect(clampScore(NaN)).toBe(0); + }); + + it("allows custom fallback", () => { + expect(clampScore(undefined, 0.5)).toBe(0.5); + }); + }); + + describe("deriveSeverity", () => { + it("returns none for clean status", () => { + expect(deriveSeverity("clean", 0)).toBe("none"); + }); + + it("returns low for warn with low score", () => { + expect(deriveSeverity("warn", 0.5)).toBe("low"); + }); + + it("returns medium for warn with score >= 0.65", () => { + expect(deriveSeverity("warn", 0.65)).toBe("medium"); + }); + + it("returns critical for flagged with score >= 0.9", () => { + expect(deriveSeverity("flagged", 0.9)).toBe("critical"); + }); + + it("returns high for flagged with score >= 0.75", () => { + expect(deriveSeverity("flagged", 0.75)).toBe("high"); + }); + + it("returns medium for flagged with score < 0.75", () => { + expect(deriveSeverity("flagged", 0.5)).toBe("medium"); + }); + }); + + describe("deriveRecommendedAction", () => { + it("returns none for clean status", () => { + expect(deriveRecommendedAction("clean", "none")).toBe("none"); + }); + + it("returns review for warn with medium severity", () => { + expect(deriveRecommendedAction("warn", "medium")).toBe("review"); + }); + + it("returns warn for warn with low severity", () => { + expect(deriveRecommendedAction("warn", "low")).toBe("warn"); + }); + }); + + describe("hasDeferralAnalysis", () => { + it("detects Indonesian deferral: kurang konteks", () => { + expect(hasDeferralAnalysis("kurang konteks untuk menilai")).toBe(true); + }); + + it("detects English deferral: insufficient context", () => { + expect(hasDeferralAnalysis("insufficient context to moderate")).toBe(true); + }); + + it("detects cannot determine pattern", () => { + expect(hasDeferralAnalysis("cannot determine")).toBe(true); + }); + + it("returns false for non-deferral text", () => { + expect(hasDeferralAnalysis("This message is perfectly clean")).toBe(false); + }); + + it("returns false for exception pattern (decisive verdict)", () => { + expect( + hasDeferralAnalysis("tidak bisa menentukan karena tidak ada pelanggaran"), + ).toBe(false); + }); + }); +}); + +import { extractJson } from "../src/modules/ai-moderation/jsonExtractor.js"; + +describe("extractJson", () => { + it("extracts from plain JSON string", () => { + expect(extractJson('{"a":1}')).toEqual({ a: 1 }); + }); + + it("extracts from JSON inside markdown code block", () => { + const input = '```json\n{"key": "value"}\n```'; + expect(extractJson(input)).toEqual({ key: "value" }); + }); + + it("extracts from JSON inside unlabeled code block", () => { + const input = '```\n{"nested": {"x": 42}}\n```'; + expect(extractJson(input)).toEqual({ nested: { x: 42 } }); + }); + + it("extracts JSON from surrounding text", () => { + const input = 'Here is the result: {"status": "ok"} end.'; + expect(extractJson(input)).toEqual({ status: "ok" }); + }); + + it("throws an error when no JSON is found", () => { + expect(() => extractJson("this has no json at all")).toThrow( + "No JSON object found", + ); + }); + + it("throws on empty string", () => { + expect(() => extractJson("")).toThrow("No JSON object found"); }); }); diff --git a/services/frontend/src/features/messages/index.tsx b/services/frontend/src/features/messages/index.tsx index d682e80..501e183 100644 --- a/services/frontend/src/features/messages/index.tsx +++ b/services/frontend/src/features/messages/index.tsx @@ -1,7 +1,7 @@ import { motion } from "framer-motion"; import { Filter, RotateCw, Search, X } from "lucide-react"; import { useMemo, useState } from "react"; -import type { MessageRecord } from "../../shared/api/client"; +import { type MessageRecord, request } from "../../shared/api/client"; import { cardItem, cardStagger } from "../../shared/hooks/useFramerStagger"; import { Badge, @@ -58,9 +58,9 @@ export function MessagesPanel({ setIsSearching(true); try { const params = new URLSearchParams({ q: searchQuery, limit: "50" }); - const response = await fetch(`/api/analysis/search?${params}`); - if (!response.ok) throw new Error("Search failed"); - const data = await response.json(); + const data = await request<{ results: MessageRecord[] }>( + `/api/analysis/search?${params}`, + ); setSearchResults(data.results || []); setShowSearch(true); } catch {