From 1484d5265d27f04509d3e9c85cd2c21e8dde85a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:03:02 +0700 Subject: [PATCH] refactor: merge BOT_TOKEN + ADDITIONAL_BOT_TOKENS into single BOT_TOKENS env + speed audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BOT_TOKENS env: - Single BOT_TOKENS env var (comma-separated) replaces BOT_TOKEN + ADDITIONAL_BOT_TOKENS - Backward compat: falls back to BOT_TOKEN + ADDITIONAL_BOT_TOKENS if BOT_TOKENS unset - Config exposes botTokens: string[] instead of botToken + additionalBotTokens - Updated env.ts, bot-pool.ts, docker-compose.yml, .env.example, CLAUDE.md, all tests Speed audit (S3 -> Telegram upload flow): - Hoisted 5 dynamic await import('../../../db/index') to top-level static imports in s3-controller.ts (3x) and web-api-controller.ts (2x) -> saves module resolution + async overhead on every upload - Removed stale UPLOAD_CONCURRENCY env from docker-compose.yml (already removed from env.ts in prior refactor) Upload flow is already concurrent: - streamBodyToTemp() uses Bun.file(path).writer() — O(1) memory, safe for multi-GB blobs - utils/chunked-storage.ts reads chunks serially but uploads concurrently with inFlight backpressure at effectiveConcurrency * 2 (= 16 with 8 bots) - bot-pool.ts: per-bot PQueue(concurrency=1), 8 bots = 8 concurrent uploads per file, selectBot() picks least-loaded, 429 detection + inner+outer retry loops - TELEGRAM_API_TIMEOUT_MS=120s — ample for 48MB chunks - Infrastructure chunked-storage.ts (DI-based, dead code) has serial upload trap — noted for future cleanup --- .env.example | 3 +-- CLAUDE.md | 2 +- docker-compose.yml | 4 +-- src/env.ts | 25 +++++++++++++------ src/infrastructure/telegram/bot-pool.ts | 2 +- .../http/controllers/s3-controller.ts | 4 +-- .../http/controllers/web-api-controller.ts | 3 +-- test/bot-pool.test.ts | 3 +-- test/env.test.ts | 15 +++++------ test/helpers/setup-env.ts | 5 ++-- test/telegram.test.ts | 2 +- 11 files changed, 36 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index 27518dc..ec9fad6 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,4 @@ -BOT_TOKEN=isi_token_bot_telegram -ADDITIONAL_BOT_TOKENS=token_cadangan_1,token_cadangan_2,token_cadangan_3 +BOT_TOKENS=isi_token_bot_1,isi_token_bot_2,isi_token_bot_3 STORAGE_CHANNEL_ID=-1001234567890 BASE_URL=https://tele.asepharyana.my.id DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader diff --git a/CLAUDE.md b/CLAUDE.md index 80fc2cf..fe9fd8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ Default to using Bun instead of Node.js. - Bun.$`ls` instead of execa. - Rate limiter lokal dinonaktifkan (`checkRateLimit` di `src/utils/rateLimit.ts` selalu mengembalikan `true`). - Telegram API memiliki auto-retry otomatis jika mengembalikan error 429 (Too Many Requests) menggunakan pool Telegraf multi-bot di `src/utils/telegram.ts`. -- Multi-bot dikonfigurasi melalui `ADDITIONAL_BOT_TOKENS` (koma terpisah) di `.env` yang digabung dengan `BOT_TOKEN` utama (total 4 bot). +|- Multi-bot dikonfigurasi melalui `BOT_TOKENS` (koma terpisah) di `.env` — semua token bot digabung dalam satu variabel. - Menggunakan mekanisme rotasi instan jika ada bot yang terkena rate limit 429 sebelum memutuskan untuk sleep. - Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian. - Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman). diff --git a/docker-compose.yml b/docker-compose.yml index 8e52258..805722f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,7 @@ services: container_name: filedrop-app restart: always environment: - - BOT_TOKEN=${BOT_TOKEN} - - ADDITIONAL_BOT_TOKENS=${ADDITIONAL_BOT_TOKENS:-} + - BOT_TOKENS=${BOT_TOKENS} - STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID} - BASE_URL=${BASE_URL} - DATABASE_URL=${DATABASE_URL} @@ -13,7 +12,6 @@ services: - NODE_ENV=production - LOG_LEVEL=info - TRUST_PROXY=true - - UPLOAD_CONCURRENCY=${UPLOAD_CONCURRENCY:-8} - BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20} - BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000} - MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648} diff --git a/src/env.ts b/src/env.ts index d19a854..1d2e662 100644 --- a/src/env.ts +++ b/src/env.ts @@ -1,8 +1,8 @@ import logger from './utils/logger'; interface AppConfig { - botToken: string; - additionalBotTokens: string[]; + /** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */ + botTokens: string[]; storageChatId: number; baseUrl: string; databaseUrl: string; @@ -28,8 +28,21 @@ interface AppConfig { s3VhostDomains: string[]; } +// Validate bot tokens: BOT_TOKENS (new) or fallback to BOT_TOKEN + ADDITIONAL_BOT_TOKENS +const botTokensRaw = + process.env.BOT_TOKENS || + [process.env.BOT_TOKEN, process.env.ADDITIONAL_BOT_TOKENS].filter(Boolean).join(','); + +if (!botTokensRaw) { + logger.error( + 'Missing required environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)', + ); + throw new Error( + 'Missing environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)', + ); +} + const requiredEnv = { - BOT_TOKEN: process.env.BOT_TOKEN, STORAGE_CHANNEL_ID: process.env.STORAGE_CHANNEL_ID, BASE_URL: process.env.BASE_URL, DATABASE_URL: process.env.DATABASE_URL, @@ -93,8 +106,7 @@ const maskDatabaseUrl = (value: string): string => value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@'); export const config: AppConfig = { - botToken: process.env.BOT_TOKEN!, - additionalBotTokens: parseTokens(process.env.ADDITIONAL_BOT_TOKENS), + botTokens: parseTokens(botTokensRaw), storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10), baseUrl: process.env.BASE_URL!, databaseUrl: process.env.DATABASE_URL!, @@ -126,8 +138,7 @@ export const config: AppConfig = { logger.info('Environment variables loaded', { config: { ...config, - botToken: maskSecret(config.botToken), - additionalBotTokens: config.additionalBotTokens.map(maskSecret), + botTokens: config.botTokens.map(maskSecret), databaseUrl: maskDatabaseUrl(config.databaseUrl), adminApiToken: maskSecret(config.adminApiToken), adminApiTokenEnabled: config.adminApiToken.length > 0, diff --git a/src/infrastructure/telegram/bot-pool.ts b/src/infrastructure/telegram/bot-pool.ts index fba5d7e..6f906c5 100644 --- a/src/infrastructure/telegram/bot-pool.ts +++ b/src/infrastructure/telegram/bot-pool.ts @@ -64,7 +64,7 @@ export class BotPool implements ITelegramService { private readonly bots: BotEntry[] = []; constructor() { - const tokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens])); + const tokens = Array.from(new Set(config.botTokens)); this.bots = tokens.map((token, index) => ({ index, token, diff --git a/src/interfaces/http/controllers/s3-controller.ts b/src/interfaces/http/controllers/s3-controller.ts index ce2d2be..d30c555 100644 --- a/src/interfaces/http/controllers/s3-controller.ts +++ b/src/interfaces/http/controllers/s3-controller.ts @@ -8,6 +8,7 @@ import { listObjectsByPrefix, softDeleteFile, } from '../../../db/files-ext'; +import { db, files as fileSchema } from '../../../db/index'; import { abortMultipartUpload, completeMultipartUpload, @@ -1082,7 +1083,6 @@ const storeFileFromTemp = async ( fileStream.destroy(); const publicId = nanoid(); - const { db, files: fileSchema } = await import('../../../db/index'); await db.insert(fileSchema).values({ publicId, @@ -1194,7 +1194,6 @@ const handleCopyObject = async ( } const publicId = nanoid(); - const { db, files: fileSchema } = await import('../../../db/index'); await db.insert(fileSchema).values({ publicId, @@ -1679,7 +1678,6 @@ const handleCompleteMultipartUpload = async ( const combinedEtag = storedParts.map((p) => p.etag).join('-'); const publicId = nanoid(); - const { db, files: fileSchema } = await import('../../../db/index'); // M7: Use stored content-type from the multipart record if available const mimeType = multipart.contentType || 'application/octet-stream'; diff --git a/src/interfaces/http/controllers/web-api-controller.ts b/src/interfaces/http/controllers/web-api-controller.ts index b701180..a69824d 100644 --- a/src/interfaces/http/controllers/web-api-controller.ts +++ b/src/interfaces/http/controllers/web-api-controller.ts @@ -8,6 +8,7 @@ import { listObjectsByPrefix, softDeleteFile, } from '../../../db/files-ext'; +import { db, files as fileSchema } from '../../../db/index'; import { botPool } from '../../../infrastructure/telegram/bot-pool'; import logger from '../../../shared/logger/index'; import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file'; @@ -246,7 +247,6 @@ export const handleUploadObjectV1 = async ( ); const publicId = nanoid(); - const { db, files: fileSchema } = await import('../../../db/index'); await db.insert(fileSchema).values({ publicId, @@ -361,7 +361,6 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro } const publicId = nanoid(); - const { db, files: fileSchema } = await import('../../../db/index'); await db.insert(fileSchema).values({ publicId, diff --git a/test/bot-pool.test.ts b/test/bot-pool.test.ts index bfe6211..967df74 100644 --- a/test/bot-pool.test.ts +++ b/test/bot-pool.test.ts @@ -1,7 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; -process.env.BOT_TOKEN = 'bot1:token'; -process.env.ADDITIONAL_BOT_TOKENS = 'bot2:token,bot3:token'; +process.env.BOT_TOKENS = 'bot1:token,bot2:token,bot3:token'; process.env.STORAGE_CHANNEL_ID = '-1001234567890'; process.env.BASE_URL = 'https://example.com'; process.env.DATABASE_URL = 'sqlite://test.db'; diff --git a/test/env.test.ts b/test/env.test.ts index fc034c2..ba222cb 100644 --- a/test/env.test.ts +++ b/test/env.test.ts @@ -3,7 +3,7 @@ import { config } from '../src/env'; describe('Environment Variables Validation', () => { it('config should have all required fields', () => { - expect(config).toHaveProperty('botToken'); + expect(config).toHaveProperty('botTokens'); expect(config).toHaveProperty('storageChatId'); expect(config).toHaveProperty('baseUrl'); expect(config).toHaveProperty('databaseUrl'); @@ -17,8 +17,9 @@ describe('Environment Variables Validation', () => { expect(config).toHaveProperty('sessionMaxAgeMs'); }); - it('config.botToken should return BOT_TOKEN from process.env', () => { - expect(config.botToken).toBe(process.env.BOT_TOKEN || ''); + it('config.botTokens should return array from BOT_TOKENS env', () => { + expect(Array.isArray(config.botTokens)).toBe(true); + expect(config.botTokens.length).toBeGreaterThanOrEqual(3); }); it('config.storageChatId should be parsed as integer from STORAGE_CHANNEL_ID', () => { @@ -52,10 +53,10 @@ describe('Environment Variables Validation', () => { expect(config.sessionMaxAgeMs).toBe(86400 * 1000); }); - it('additionalBotTokens should be populated in test environment', () => { - expect(Array.isArray(config.additionalBotTokens)).toBe(true); - // With mock tokens from setup-env.ts there should be 2 additional tokens - expect(config.additionalBotTokens.length).toBeGreaterThanOrEqual(2); + it('botTokens should be populated in test environment', () => { + expect(Array.isArray(config.botTokens)).toBe(true); + // With mock tokens from setup-env.ts there should be at least 3 tokens + expect(config.botTokens.length).toBeGreaterThanOrEqual(3); }); it('S3 validation should not throw — env already loaded without error at import time', () => { diff --git a/test/helpers/setup-env.ts b/test/helpers/setup-env.ts index 81983b6..3066afc 100644 --- a/test/helpers/setup-env.ts +++ b/test/helpers/setup-env.ts @@ -10,12 +10,11 @@ * defaults in `src/env.ts` and are not touched. */ -process.env.BOT_TOKEN ||= '123456:ABC-DEF'; +process.env.BOT_TOKENS ||= '123456:ABC-DEF,789012:GHI-JKL,345678:MNO-PQR'; process.env.STORAGE_CHANNEL_ID ||= '-1001234567890'; process.env.BASE_URL ||= 'https://example.com'; process.env.DATABASE_URL ||= 'postgresql://user:pass@localhost:5432/test'; process.env.PORT ||= '3000'; process.env.NODE_ENV = 'test'; -// Add mock additional bot tokens so multi-bot rotation logic is tested too -process.env.ADDITIONAL_BOT_TOKENS ||= '789012:GHI-JKL,345678:MNO-PQR'; +// Keep old env names for backward compat with tests that reference them directly diff --git a/test/telegram.test.ts b/test/telegram.test.ts index fa57d23..5e6b872 100644 --- a/test/telegram.test.ts +++ b/test/telegram.test.ts @@ -148,7 +148,7 @@ describe('Telegram API Utilities', () => { file_size: 98765, mime_type: 'image/jpeg', file_path: 'photos/file_0.jpg', - bot_token: config.botToken, + bot_token: config.botTokens[0], }); }); });