refactor: merge BOT_TOKEN + ADDITIONAL_BOT_TOKENS into single BOT_TOKENS env + speed audit
Deploy FileDrop / deploy (push) Failing after 19s
Deploy FileDrop / deploy (push) Failing after 19s
BOT_TOKENS env:
- Single BOT_TOKENS env var (comma-separated) replaces BOT_TOKEN + ADDITIONAL_BOT_TOKENS
- Backward compat: falls back to BOT_TOKEN + ADDITIONAL_BOT_TOKENS if BOT_TOKENS unset
- Config exposes botTokens: string[] instead of botToken + additionalBotTokens
- Updated env.ts, bot-pool.ts, docker-compose.yml, .env.example, CLAUDE.md, all tests
Speed audit (S3 -> Telegram upload flow):
- Hoisted 5 dynamic await import('../../../db/index') to top-level static imports
in s3-controller.ts (3x) and web-api-controller.ts (2x)
-> saves module resolution + async overhead on every upload
- Removed stale UPLOAD_CONCURRENCY env from docker-compose.yml
(already removed from env.ts in prior refactor)
Upload flow is already concurrent:
- streamBodyToTemp() uses Bun.file(path).writer() — O(1) memory, safe for multi-GB blobs
- utils/chunked-storage.ts reads chunks serially but uploads concurrently with
inFlight backpressure at effectiveConcurrency * 2 (= 16 with 8 bots)
- bot-pool.ts: per-bot PQueue(concurrency=1), 8 bots = 8 concurrent uploads per file,
selectBot() picks least-loaded, 429 detection + inner+outer retry loops
- TELEGRAM_API_TIMEOUT_MS=120s — ample for 48MB chunks
- Infrastructure chunked-storage.ts (DI-based, dead code) has serial upload trap —
noted for future cleanup
This commit is contained in:
+1
-2
@@ -1,5 +1,4 @@
|
|||||||
BOT_TOKEN=isi_token_bot_telegram
|
BOT_TOKENS=isi_token_bot_1,isi_token_bot_2,isi_token_bot_3
|
||||||
ADDITIONAL_BOT_TOKENS=token_cadangan_1,token_cadangan_2,token_cadangan_3
|
|
||||||
STORAGE_CHANNEL_ID=-1001234567890
|
STORAGE_CHANNEL_ID=-1001234567890
|
||||||
BASE_URL=https://tele.asepharyana.my.id
|
BASE_URL=https://tele.asepharyana.my.id
|
||||||
DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader
|
DATABASE_URL=postgresql://user:password@localhost:5432/telegram_uploader
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ Default to using Bun instead of Node.js.
|
|||||||
- Bun.$`ls` instead of execa.
|
- Bun.$`ls` instead of execa.
|
||||||
- Rate limiter lokal dinonaktifkan (`checkRateLimit` di `src/utils/rateLimit.ts` selalu mengembalikan `true`).
|
- Rate limiter lokal dinonaktifkan (`checkRateLimit` di `src/utils/rateLimit.ts` selalu mengembalikan `true`).
|
||||||
- Telegram API memiliki auto-retry otomatis jika mengembalikan error 429 (Too Many Requests) menggunakan pool Telegraf multi-bot di `src/utils/telegram.ts`.
|
- Telegram API memiliki auto-retry otomatis jika mengembalikan error 429 (Too Many Requests) menggunakan pool Telegraf multi-bot di `src/utils/telegram.ts`.
|
||||||
- Multi-bot dikonfigurasi melalui `ADDITIONAL_BOT_TOKENS` (koma terpisah) di `.env` yang digabung dengan `BOT_TOKEN` utama (total 4 bot).
|
|- Multi-bot dikonfigurasi melalui `BOT_TOKENS` (koma terpisah) di `.env` — semua token bot digabung dalam satu variabel.
|
||||||
- Menggunakan mekanisme rotasi instan jika ada bot yang terkena rate limit 429 sebelum memutuskan untuk sleep.
|
- Menggunakan mekanisme rotasi instan jika ada bot yang terkena rate limit 429 sebelum memutuskan untuk sleep.
|
||||||
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
|
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
|
||||||
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
|
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
|
||||||
|
|||||||
+1
-3
@@ -4,8 +4,7 @@ services:
|
|||||||
container_name: filedrop-app
|
container_name: filedrop-app
|
||||||
restart: always
|
restart: always
|
||||||
environment:
|
environment:
|
||||||
- BOT_TOKEN=${BOT_TOKEN}
|
- BOT_TOKENS=${BOT_TOKENS}
|
||||||
- ADDITIONAL_BOT_TOKENS=${ADDITIONAL_BOT_TOKENS:-}
|
|
||||||
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
|
||||||
- BASE_URL=${BASE_URL}
|
- BASE_URL=${BASE_URL}
|
||||||
- DATABASE_URL=${DATABASE_URL}
|
- DATABASE_URL=${DATABASE_URL}
|
||||||
@@ -13,7 +12,6 @@ services:
|
|||||||
- NODE_ENV=production
|
- NODE_ENV=production
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
- TRUST_PROXY=true
|
- TRUST_PROXY=true
|
||||||
- UPLOAD_CONCURRENCY=${UPLOAD_CONCURRENCY:-8}
|
|
||||||
- BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20}
|
- BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20}
|
||||||
- BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000}
|
- BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000}
|
||||||
- MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648}
|
- MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648}
|
||||||
|
|||||||
+18
-7
@@ -1,8 +1,8 @@
|
|||||||
import logger from './utils/logger';
|
import logger from './utils/logger';
|
||||||
|
|
||||||
interface AppConfig {
|
interface AppConfig {
|
||||||
botToken: string;
|
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
|
||||||
additionalBotTokens: string[];
|
botTokens: string[];
|
||||||
storageChatId: number;
|
storageChatId: number;
|
||||||
baseUrl: string;
|
baseUrl: string;
|
||||||
databaseUrl: string;
|
databaseUrl: string;
|
||||||
@@ -28,8 +28,21 @@ interface AppConfig {
|
|||||||
s3VhostDomains: string[];
|
s3VhostDomains: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate bot tokens: BOT_TOKENS (new) or fallback to BOT_TOKEN + ADDITIONAL_BOT_TOKENS
|
||||||
|
const botTokensRaw =
|
||||||
|
process.env.BOT_TOKENS ||
|
||||||
|
[process.env.BOT_TOKEN, process.env.ADDITIONAL_BOT_TOKENS].filter(Boolean).join(',');
|
||||||
|
|
||||||
|
if (!botTokensRaw) {
|
||||||
|
logger.error(
|
||||||
|
'Missing required environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)',
|
||||||
|
);
|
||||||
|
throw new Error(
|
||||||
|
'Missing environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const requiredEnv = {
|
const requiredEnv = {
|
||||||
BOT_TOKEN: process.env.BOT_TOKEN,
|
|
||||||
STORAGE_CHANNEL_ID: process.env.STORAGE_CHANNEL_ID,
|
STORAGE_CHANNEL_ID: process.env.STORAGE_CHANNEL_ID,
|
||||||
BASE_URL: process.env.BASE_URL,
|
BASE_URL: process.env.BASE_URL,
|
||||||
DATABASE_URL: process.env.DATABASE_URL,
|
DATABASE_URL: process.env.DATABASE_URL,
|
||||||
@@ -93,8 +106,7 @@ const maskDatabaseUrl = (value: string): string =>
|
|||||||
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
||||||
|
|
||||||
export const config: AppConfig = {
|
export const config: AppConfig = {
|
||||||
botToken: process.env.BOT_TOKEN!,
|
botTokens: parseTokens(botTokensRaw),
|
||||||
additionalBotTokens: parseTokens(process.env.ADDITIONAL_BOT_TOKENS),
|
|
||||||
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10),
|
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10),
|
||||||
baseUrl: process.env.BASE_URL!,
|
baseUrl: process.env.BASE_URL!,
|
||||||
databaseUrl: process.env.DATABASE_URL!,
|
databaseUrl: process.env.DATABASE_URL!,
|
||||||
@@ -126,8 +138,7 @@ export const config: AppConfig = {
|
|||||||
logger.info('Environment variables loaded', {
|
logger.info('Environment variables loaded', {
|
||||||
config: {
|
config: {
|
||||||
...config,
|
...config,
|
||||||
botToken: maskSecret(config.botToken),
|
botTokens: config.botTokens.map(maskSecret),
|
||||||
additionalBotTokens: config.additionalBotTokens.map(maskSecret),
|
|
||||||
databaseUrl: maskDatabaseUrl(config.databaseUrl),
|
databaseUrl: maskDatabaseUrl(config.databaseUrl),
|
||||||
adminApiToken: maskSecret(config.adminApiToken),
|
adminApiToken: maskSecret(config.adminApiToken),
|
||||||
adminApiTokenEnabled: config.adminApiToken.length > 0,
|
adminApiTokenEnabled: config.adminApiToken.length > 0,
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ export class BotPool implements ITelegramService {
|
|||||||
private readonly bots: BotEntry[] = [];
|
private readonly bots: BotEntry[] = [];
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
const tokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens]));
|
const tokens = Array.from(new Set(config.botTokens));
|
||||||
this.bots = tokens.map((token, index) => ({
|
this.bots = tokens.map((token, index) => ({
|
||||||
index,
|
index,
|
||||||
token,
|
token,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
listObjectsByPrefix,
|
listObjectsByPrefix,
|
||||||
softDeleteFile,
|
softDeleteFile,
|
||||||
} from '../../../db/files-ext';
|
} from '../../../db/files-ext';
|
||||||
|
import { db, files as fileSchema } from '../../../db/index';
|
||||||
import {
|
import {
|
||||||
abortMultipartUpload,
|
abortMultipartUpload,
|
||||||
completeMultipartUpload,
|
completeMultipartUpload,
|
||||||
@@ -1082,7 +1083,6 @@ const storeFileFromTemp = async (
|
|||||||
fileStream.destroy();
|
fileStream.destroy();
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await db.insert(fileSchema).values({
|
||||||
publicId,
|
publicId,
|
||||||
@@ -1194,7 +1194,6 @@ const handleCopyObject = async (
|
|||||||
}
|
}
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await db.insert(fileSchema).values({
|
||||||
publicId,
|
publicId,
|
||||||
@@ -1679,7 +1678,6 @@ const handleCompleteMultipartUpload = async (
|
|||||||
const combinedEtag = storedParts.map((p) => p.etag).join('-');
|
const combinedEtag = storedParts.map((p) => p.etag).join('-');
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
// M7: Use stored content-type from the multipart record if available
|
// M7: Use stored content-type from the multipart record if available
|
||||||
const mimeType = multipart.contentType || 'application/octet-stream';
|
const mimeType = multipart.contentType || 'application/octet-stream';
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
listObjectsByPrefix,
|
listObjectsByPrefix,
|
||||||
softDeleteFile,
|
softDeleteFile,
|
||||||
} from '../../../db/files-ext';
|
} from '../../../db/files-ext';
|
||||||
|
import { db, files as fileSchema } from '../../../db/index';
|
||||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||||
import logger from '../../../shared/logger/index';
|
import logger from '../../../shared/logger/index';
|
||||||
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
|
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
|
||||||
@@ -246,7 +247,6 @@ export const handleUploadObjectV1 = async (
|
|||||||
);
|
);
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await db.insert(fileSchema).values({
|
||||||
publicId,
|
publicId,
|
||||||
@@ -361,7 +361,6 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro
|
|||||||
}
|
}
|
||||||
|
|
||||||
const publicId = nanoid();
|
const publicId = nanoid();
|
||||||
const { db, files: fileSchema } = await import('../../../db/index');
|
|
||||||
|
|
||||||
await db.insert(fileSchema).values({
|
await db.insert(fileSchema).values({
|
||||||
publicId,
|
publicId,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
|
import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
|
||||||
|
|
||||||
process.env.BOT_TOKEN = 'bot1:token';
|
process.env.BOT_TOKENS = 'bot1:token,bot2:token,bot3:token';
|
||||||
process.env.ADDITIONAL_BOT_TOKENS = 'bot2:token,bot3:token';
|
|
||||||
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
process.env.STORAGE_CHANNEL_ID = '-1001234567890';
|
||||||
process.env.BASE_URL = 'https://example.com';
|
process.env.BASE_URL = 'https://example.com';
|
||||||
process.env.DATABASE_URL = 'sqlite://test.db';
|
process.env.DATABASE_URL = 'sqlite://test.db';
|
||||||
|
|||||||
+8
-7
@@ -3,7 +3,7 @@ import { config } from '../src/env';
|
|||||||
|
|
||||||
describe('Environment Variables Validation', () => {
|
describe('Environment Variables Validation', () => {
|
||||||
it('config should have all required fields', () => {
|
it('config should have all required fields', () => {
|
||||||
expect(config).toHaveProperty('botToken');
|
expect(config).toHaveProperty('botTokens');
|
||||||
expect(config).toHaveProperty('storageChatId');
|
expect(config).toHaveProperty('storageChatId');
|
||||||
expect(config).toHaveProperty('baseUrl');
|
expect(config).toHaveProperty('baseUrl');
|
||||||
expect(config).toHaveProperty('databaseUrl');
|
expect(config).toHaveProperty('databaseUrl');
|
||||||
@@ -17,8 +17,9 @@ describe('Environment Variables Validation', () => {
|
|||||||
expect(config).toHaveProperty('sessionMaxAgeMs');
|
expect(config).toHaveProperty('sessionMaxAgeMs');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('config.botToken should return BOT_TOKEN from process.env', () => {
|
it('config.botTokens should return array from BOT_TOKENS env', () => {
|
||||||
expect(config.botToken).toBe(process.env.BOT_TOKEN || '');
|
expect(Array.isArray(config.botTokens)).toBe(true);
|
||||||
|
expect(config.botTokens.length).toBeGreaterThanOrEqual(3);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('config.storageChatId should be parsed as integer from STORAGE_CHANNEL_ID', () => {
|
it('config.storageChatId should be parsed as integer from STORAGE_CHANNEL_ID', () => {
|
||||||
@@ -52,10 +53,10 @@ describe('Environment Variables Validation', () => {
|
|||||||
expect(config.sessionMaxAgeMs).toBe(86400 * 1000);
|
expect(config.sessionMaxAgeMs).toBe(86400 * 1000);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('additionalBotTokens should be populated in test environment', () => {
|
it('botTokens should be populated in test environment', () => {
|
||||||
expect(Array.isArray(config.additionalBotTokens)).toBe(true);
|
expect(Array.isArray(config.botTokens)).toBe(true);
|
||||||
// With mock tokens from setup-env.ts there should be 2 additional tokens
|
// With mock tokens from setup-env.ts there should be at least 3 tokens
|
||||||
expect(config.additionalBotTokens.length).toBeGreaterThanOrEqual(2);
|
expect(config.botTokens.length).toBeGreaterThanOrEqual(3);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('S3 validation should not throw — env already loaded without error at import time', () => {
|
it('S3 validation should not throw — env already loaded without error at import time', () => {
|
||||||
|
|||||||
@@ -10,12 +10,11 @@
|
|||||||
* defaults in `src/env.ts` and are not touched.
|
* defaults in `src/env.ts` and are not touched.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
process.env.BOT_TOKEN ||= '123456:ABC-DEF';
|
process.env.BOT_TOKENS ||= '123456:ABC-DEF,789012:GHI-JKL,345678:MNO-PQR';
|
||||||
process.env.STORAGE_CHANNEL_ID ||= '-1001234567890';
|
process.env.STORAGE_CHANNEL_ID ||= '-1001234567890';
|
||||||
process.env.BASE_URL ||= 'https://example.com';
|
process.env.BASE_URL ||= 'https://example.com';
|
||||||
process.env.DATABASE_URL ||= 'postgresql://user:pass@localhost:5432/test';
|
process.env.DATABASE_URL ||= 'postgresql://user:pass@localhost:5432/test';
|
||||||
process.env.PORT ||= '3000';
|
process.env.PORT ||= '3000';
|
||||||
process.env.NODE_ENV = 'test';
|
process.env.NODE_ENV = 'test';
|
||||||
|
|
||||||
// Add mock additional bot tokens so multi-bot rotation logic is tested too
|
// Keep old env names for backward compat with tests that reference them directly
|
||||||
process.env.ADDITIONAL_BOT_TOKENS ||= '789012:GHI-JKL,345678:MNO-PQR';
|
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ describe('Telegram API Utilities', () => {
|
|||||||
file_size: 98765,
|
file_size: 98765,
|
||||||
mime_type: 'image/jpeg',
|
mime_type: 'image/jpeg',
|
||||||
file_path: 'photos/file_0.jpg',
|
file_path: 'photos/file_0.jpg',
|
||||||
bot_token: config.botToken,
|
bot_token: config.botTokens[0],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user