feat(core): implement data retention, metrics, and enhanced media handling
This commit introduces several significant improvements across the backend and gateway services:
- **Data Retention**: Added an automated cleanup scheduler in `discord-gateway` to prune expired messages, attachments, and voice recordings based on configurable retention policies.
- **Observability**: Integrated `prom-client` in the `backend` service to expose Prometheus metrics via `/api/metrics` and added default Node.js runtime metrics.
- **Media Handling**: Enhanced `MediaHandler` in `discord-gateway` to support media URL resolution and improved playback status tracking.
- **API & Config**: Expanded the configuration endpoint to expose more system settings and reorganized `.env.example` for better readability.
- **Refactoring & Cleanup**:
- Removed unused `better-sqlite3` dependency.
- Refactored voice channel routing.
- Improved error handling and testing coverage with comprehensive unit tests for shared utilities and error classes.
- **Documentation**: Added `MEMORY.md` for project context.
This commit is contained in:
MythEclipse
+101
-117
@@ -1,136 +1,120 @@
|
||||
# Discord Bot Configuration
|
||||
DISCORD_TOKEN=your_bot_token_here
|
||||
MONITOR_GUILD_ID=your_guild_id_here
|
||||
TEXT_GUILD_ID=optional_text_guild_id
|
||||
TEXT_CHANNEL_ID=optional_text_channel_id
|
||||
# =============================================================================
|
||||
|
||||
# Recording Configuration
|
||||
RECORDINGS_DIR=./recordings
|
||||
RECORDING_SEGMENT_MS=5000
|
||||
VERBOSE=false
|
||||
# === Discord ===
|
||||
DISCORD_TOKEN=your_bot_token_here # REQUIRED
|
||||
MONITOR_GUILD_ID=your_guild_id_here # Target guild for text monitoring
|
||||
TEXT_GUILD_ID=optional_text_guild_id # Override text capture guild (falls back to MONITOR_GUILD_ID)
|
||||
TEXT_CHANNEL_ID=optional_text_channel_id # Restrict text capture to a single channel
|
||||
|
||||
# Decoder Configuration
|
||||
DECODER_ROTATE_MS=5000
|
||||
DECODER_COOLDOWN_MS=30000
|
||||
# === Voice Channels ===
|
||||
# VOICE_GUILD_ID= # Guild for voice connection (optional)
|
||||
# VOICE_CHANNEL_ID= # Channel for voice connection (optional)
|
||||
|
||||
# Audio Configuration
|
||||
AUDIO_STREAM_SILENCE_DURATION_MS=3000
|
||||
PACKET_FILTER_MIN_SIZE=8
|
||||
OPUS_FRAME_SIZE=960
|
||||
AUDIO_SAMPLE_RATE=48000
|
||||
AUDIO_CHANNELS=2
|
||||
AVATAR_SIZE=64
|
||||
# === Recording ===
|
||||
RECORDINGS_DIR=./recordings # Audio file output directory (default: ./recordings)
|
||||
RECORDING_SEGMENT_MS=5000 # OGG segment duration in ms (default: 5000)
|
||||
|
||||
# Webserver Configuration
|
||||
WEBSERVER_PORT=3000
|
||||
# === Decoder ===
|
||||
DECODER_ROTATE_MS=5000 # Opus decoder rotation interval in ms (default: 5000)
|
||||
DECODER_COOLDOWN_MS=30000 # Decoder error cooldown in ms (default: 30000)
|
||||
|
||||
# Connection Configuration
|
||||
VOICE_CONNECTION_TIMEOUT_MS=15000
|
||||
RECONNECT_TIMEOUT_MS=5000
|
||||
# === Audio ===
|
||||
AUDIO_STREAM_SILENCE_DURATION_MS=3000 # Silence threshold in ms before stopping stream (default: 3000)
|
||||
PACKET_FILTER_MIN_SIZE=8 # Minimum Opus packet size in bytes (default: 8)
|
||||
OPUS_FRAME_SIZE=960 # Opus frame size in samples (default: 960)
|
||||
AUDIO_SAMPLE_RATE=48000 # Audio sample rate in Hz (default: 48000)
|
||||
AUDIO_CHANNELS=2 # Number of audio channels (default: 2)
|
||||
AVATAR_SIZE=64 # User avatar size in pixels (default: 64)
|
||||
|
||||
# Voice Recording Selection
|
||||
# VOICE_GUILD_ID falls back to legacy GUILD_ID when omitted.
|
||||
GUILD_ID=legacy_voice_guild_id
|
||||
VOICE_GUILD_ID=voice_guild_id
|
||||
VOICE_CHANNEL_ID=voice_channel_id
|
||||
# === Webserver ===
|
||||
WEBSERVER_PORT=3001 # Backend HTTP/WS server port (default: 3001)
|
||||
|
||||
# Logging Configuration
|
||||
LOG_LEVEL=info
|
||||
NODE_ENV=development
|
||||
# === Connection ===
|
||||
VOICE_CONNECTION_TIMEOUT_MS=15000 # Voice connection timeout in ms (default: 15000)
|
||||
RECONNECT_TIMEOUT_MS=5000 # Reconnect timeout in ms (default: 5000)
|
||||
|
||||
# Moderation Configuration
|
||||
TELE_UPLOAD_URL=https://upload.asepharyana.my.id/api/upload
|
||||
ATTACHMENT_UPLOAD_TIMEOUT_MS=30000
|
||||
ATTACHMENT_MAX_SIZE_MB=100
|
||||
ATTACHMENT_RETRY_ATTEMPTS=3
|
||||
BACKLOG_SYNC_HOURS=24
|
||||
BACKLOG_SYNC_BATCH_SIZE=100
|
||||
# === Logging ===
|
||||
LOG_LEVEL=info # Pino log level: error|warn|info|http|verbose|debug|silly (default: info)
|
||||
NODE_ENV=development # Environment: development|production|test (default: development)
|
||||
VERBOSE=false # Enable verbose/debug logging (default: false)
|
||||
|
||||
# AI Analysis Configuration
|
||||
AI_ANALYSIS_ENABLED=false
|
||||
AI_LLM_API_KEY=your_9router_key_here
|
||||
AI_LLM_BASE_URL=https://9router.asepharyana.my.id/v1
|
||||
AI_LLM_MODEL=text
|
||||
# Vision model for image/video moderation (falls back to AI_LLM_MODEL if unset)
|
||||
AI_LLM_VISION_MODEL=multimodal
|
||||
# Max concurrent LLM API calls (default: 5)
|
||||
AI_LLM_MAX_CONCURRENT=5
|
||||
# Maximum image dimension in pixels before resize for vision API (default: 1024)
|
||||
AI_LLM_IMAGE_MAX_DIMENSION=1024
|
||||
# Maximum messages per text-only moderation batch (default: 20)
|
||||
AI_LLM_TEXT_BATCH_SIZE=20
|
||||
# Timeout in ms for individual media analysis calls (default: 60000)
|
||||
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS=60000
|
||||
# === Admin ===
|
||||
ADMIN_PASSWORD=admin123 # Backend admin password for API auth (default: admin123)
|
||||
|
||||
# AI Moderation Analysis Tuning (advanced)
|
||||
AI_ANALYSIS_DEBOUNCE_MS=500
|
||||
AI_ANALYSIS_RECOVERY_INTERVAL_MS=15000
|
||||
AI_ANALYSIS_ERROR_COOLDOWN_MS=30000
|
||||
# Max messages fetched per conversation batch (default: 200)
|
||||
AI_ANALYSIS_MAX_BATCH_SIZE=200
|
||||
AI_ANALYSIS_MAX_CONTEXT_TOKENS=8000
|
||||
# Token budget for target messages specifically (default: 4000)
|
||||
AI_ANALYSIS_MAX_TARGET_TOKENS=4000
|
||||
AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT=20
|
||||
# How long a conversation is locked while being processed (default: 120000ms)
|
||||
AI_ANALYSIS_PROCESSING_TIMEOUT_MS=120000
|
||||
# Max concurrent individual-fallback jobs (default: 50)
|
||||
AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT=50
|
||||
# Consecutive errors before individual circuit breaker trips (default: 50)
|
||||
AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD=50
|
||||
|
||||
# OpenAI Moderation (optional separate provider)
|
||||
# OPENAI_MODERATION_API_KEY=your_key_here
|
||||
# OPENAI_MODERATION_BASE_URL=https://api.openai.com/v1
|
||||
# OPENAI_MODERATION_MODEL=omni-moderation-latest
|
||||
|
||||
# Admin
|
||||
ADMIN_PASSWORD=admin123
|
||||
|
||||
# Database Configuration (PostgreSQL)
|
||||
# Option 1: Use DATABASE_URL for connection string
|
||||
# === Database (PostgreSQL) ===
|
||||
# Option 1: Connection string (overrides individual params)
|
||||
# DATABASE_URL=postgresql://user:password@localhost:5432/discord_bot
|
||||
|
||||
# Option 2: Use individual connection parameters
|
||||
# POSTGRES_HOST=localhost
|
||||
# POSTGRES_PORT=5432
|
||||
# POSTGRES_USER=postgres
|
||||
# POSTGRES_PASSWORD=your_password_here
|
||||
# POSTGRES_DB=discord_bot
|
||||
# Option 2: Individual connection parameters
|
||||
POSTGRES_HOST=localhost # PostgreSQL host (default: localhost)
|
||||
POSTGRES_PORT=5432 # PostgreSQL port (default: 5432)
|
||||
POSTGRES_USER=postgres # PostgreSQL user (optional if DATABASE_URL provided)
|
||||
POSTGRES_PASSWORD=your_password_here # PostgreSQL password (optional if DATABASE_URL provided)
|
||||
POSTGRES_DB=discord_bot # PostgreSQL database name (optional if DATABASE_URL provided)
|
||||
POSTGRES_POOL_MIN=2 # Minimum pool connections (default: 2)
|
||||
POSTGRES_POOL_MAX=10 # Maximum pool connections (default: 10)
|
||||
|
||||
# Redis Configuration (queue + persistent KV store)
|
||||
# REDIS_URL=redis://localhost:6379
|
||||
# === Redis ===
|
||||
REDIS_URL=redis://localhost:6379 # Redis connection string (default: redis://localhost:6379)
|
||||
|
||||
# PostgreSQL Connection Pool Configuration
|
||||
# POSTGRES_POOL_MIN=2
|
||||
# POSTGRES_POOL_MAX=10
|
||||
# === Attachments ===
|
||||
TELE_UPLOAD_URL=https://upload.asepharyana.my.id/api/upload # Attachment upload endpoint (default)
|
||||
ATTACHMENT_UPLOAD_TIMEOUT_MS=30000 # Upload timeout in ms (default: 30000)
|
||||
ATTACHMENT_MAX_SIZE_MB=100 # Max attachment size in MB (default: 100)
|
||||
ATTACHMENT_RETRY_ATTEMPTS=3 # Upload retry count (default: 3)
|
||||
BACKLOG_SYNC_HOURS=24 # Backlog sync lookback window in hours (default: 24)
|
||||
BACKLOG_SYNC_BATCH_SIZE=100 # Messages per backlog batch, max 100 (default: 100)
|
||||
|
||||
# Auto-Delete Configuration
|
||||
AUTO_DELETE_FLAGGED_ENABLED=true
|
||||
AUTO_DELETE_FLAGGED_DRY_RUN=true
|
||||
AUTO_DELETE_FLAGGED_DELAY_MS=0
|
||||
AUTO_DELETE_MIN_CONFIDENCE=0.50
|
||||
AUTO_DELETE_ALLOWED_SEVERITIES=critical,high,medium,low
|
||||
AUTO_DELETE_NOTIFY_USER=false
|
||||
# Optional: comma-separated channel/user IDs to exclude
|
||||
# AUTO_DELETE_EXCLUDED_CHANNEL_IDS=
|
||||
# AUTO_DELETE_EXCLUDED_USER_IDS=
|
||||
# Optional: comma-separated category filter (empty = all categories)
|
||||
# AUTO_DELETE_ALLOWED_CATEGORIES=
|
||||
# Optional: log channel ID for auto-delete actions
|
||||
# AUTO_DELETE_LOG_CHANNEL_ID=
|
||||
# === AI Analysis ===
|
||||
AI_ANALYSIS_ENABLED=false # Enable AI content moderation (default: false)
|
||||
# AI_LLM_API_KEY= # REQUIRED if AI_ANALYSIS_ENABLED=true. LLM API key
|
||||
AI_LLM_BASE_URL=https://9router.asepharyana.my.id/v1 # LLM API base URL (default)
|
||||
AI_LLM_MODEL=text # LLM text model name (default: text)
|
||||
# AI_LLM_VISION_MODEL= # Vision model for image analysis (falls back to AI_LLM_MODEL)
|
||||
AI_LLM_MAX_CONCURRENT=5 # Max concurrent LLM API calls (default: 5)
|
||||
AI_LLM_IMAGE_MAX_DIMENSION=1024 # Max image dimension in pixels before resize (default: 1024)
|
||||
AI_LLM_TEXT_BATCH_SIZE=20 # Max messages per text-only moderation batch (default: 20)
|
||||
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS=60000 # Timeout in ms for media analysis calls (default: 60000)
|
||||
|
||||
# Retention Configuration (0 = disabled)
|
||||
RETENTION_MESSAGES_DAYS=0
|
||||
RETENTION_ATTACHMENTS_DAYS=0
|
||||
RETENTION_VOICE_DAYS=0
|
||||
# Cleanup interval in ms (default: 24h)
|
||||
RETENTION_CLEANUP_INTERVAL_MS=86400000
|
||||
RETENTION_DRY_RUN=true
|
||||
# === AI Analysis Tuning ===
|
||||
AI_ANALYSIS_DEBOUNCE_MS=500 # Debounce window for batching messages in ms (default: 500)
|
||||
AI_ANALYSIS_RECOVERY_INTERVAL_MS=15000 # Recovery interval after errors in ms (default: 15000)
|
||||
AI_ANALYSIS_ERROR_COOLDOWN_MS=30000 # Cooldown period after consecutive errors in ms (default: 30000)
|
||||
AI_ANALYSIS_MAX_BATCH_SIZE=200 # Max messages fetched per conversation batch (default: 200)
|
||||
AI_ANALYSIS_MAX_CONTEXT_TOKENS=8000 # Token budget for context window (default: 8000)
|
||||
AI_ANALYSIS_MAX_TARGET_TOKENS=4000 # Token budget for target messages (default: 4000)
|
||||
AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT=20 # Max messages in context window (default: 20)
|
||||
AI_ANALYSIS_PROCESSING_TIMEOUT_MS=120000 # Conversation lock timeout in ms (default: 120000)
|
||||
AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT=50 # Max concurrent individual-fallback jobs (default: 50)
|
||||
AI_ANALYSIS_INDIVIDUAL_CB_THRESHOLD=50 # Consecutive errors before circuit breaker trips (default: 50)
|
||||
|
||||
# Database Migration Configuration
|
||||
# Safe default: run migrations on startup before the app accepts traffic.
|
||||
AUTO_MIGRATE_ON_STARTUP=true
|
||||
# === OpenAI Moderation (optional separate provider) ===
|
||||
# OPENAI_MODERATION_API_KEY= # OpenAI API key for moderation endpoint
|
||||
# OPENAI_MODERATION_BASE_URL=https://api.openai.com/v1 # OpenAI moderation base URL (default)
|
||||
# OPENAI_MODERATION_MODEL=omni-moderation-latest # OpenAI moderation model (default)
|
||||
|
||||
# Worker Pool Configuration
|
||||
# PISCINA_MAX_THREADS=4
|
||||
# === Auto-Delete ===
|
||||
AUTO_DELETE_FLAGGED_ENABLED=true # Enable auto-deletion of flagged messages (default: true)
|
||||
AUTO_DELETE_FLAGGED_DRY_RUN=true # Dry-run mode: log but do not delete (default: false)
|
||||
AUTO_DELETE_FLAGGED_DELAY_MS=0 # Delay before auto-delete in ms (default: 0)
|
||||
AUTO_DELETE_MIN_CONFIDENCE=0.5 # Minimum AI confidence threshold 0-1 (default: 0.5)
|
||||
AUTO_DELETE_ALLOWED_SEVERITIES=critical,high,medium,low # Comma-separated severities (default)
|
||||
AUTO_DELETE_ALLOWED_CATEGORIES= # Comma-separated category filter (empty = all)
|
||||
AUTO_DELETE_EXCLUDED_CHANNEL_IDS= # Comma-separated channel IDs to exclude
|
||||
AUTO_DELETE_EXCLUDED_USER_IDS= # Comma-separated user IDs to exclude
|
||||
AUTO_DELETE_NOTIFY_USER=false # Notify user when their message is auto-deleted (default: false)
|
||||
AUTO_DELETE_LOG_CHANNEL_ID= # Channel ID to log auto-delete actions
|
||||
|
||||
# === Retention (0 = disabled) ===
|
||||
RETENTION_MESSAGES_DAYS=0 # Message retention in days (default: 0 = off)
|
||||
RETENTION_ATTACHMENTS_DAYS=0 # Attachment retention in days (default: 0 = off)
|
||||
RETENTION_VOICE_DAYS=0 # Voice recording retention in days (default: 0 = off)
|
||||
RETENTION_CLEANUP_INTERVAL_MS=86400000 # Cleanup interval in ms (default: 24h)
|
||||
RETENTION_DRY_RUN=true # Dry-run: log but do not delete (default: true)
|
||||
|
||||
# === Migration ===
|
||||
AUTO_MIGRATE_ON_STARTUP=true # Run database migrations on startup (default: true)
|
||||
|
||||
# === Worker Pool ===
|
||||
# PISCINA_MAX_THREADS=4 # Worker thread pool size (optional, defaults to CPU cores)
|
||||
|
||||
Reference in New Issue
Block a user