diff --git a/.env.example b/.env.example index ec9fad6..6f3154b 100644 --- a/.env.example +++ b/.env.example @@ -16,8 +16,8 @@ RATE_LIMIT_MAX_REQUESTS=30 # S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id # Telegram-safe internal chunking for large stored files -# Telegram max upload ~49MB, using 48MB for safety -# TELEGRAM_CHUNK_SIZE_BYTES=50331648 +# Telegram getFile download limit is 20 MB; guard rejects > 19922944 (19 MB) +# TELEGRAM_CHUNK_SIZE_BYTES=19922944 # COMPRESS_CHUNKED_UPLOADS=true # CHUNK_COMPRESSION_MIN_SIZE_BYTES=4096 diff --git a/CLAUDE.md b/CLAUDE.md index fe9fd8b..3137be2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,6 +25,12 @@ Default to using Bun instead of Node.js. - Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian. - Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman). +## Chunk size (TELEGRAM_CHUNK_SIZE_BYTES) + +- Batas keras: Telegram Bot API `getFile` hanya bisa resolve file ≤ 20 MB — di atas itu error `Bad Request: file is too big` dan part tidak bisa di-download. +- Guard fail-fast di `src/env.ts`: service MENOLAK start (exit non-zero) jika `TELEGRAM_CHUNK_SIZE_BYTES` > 19922944 (19 MB, margin aman dari limit 20 MB). Konstanta: `TELEGRAM_CHUNK_SIZE_MAX_BYTES` di `src/shared/utils/validation.ts`, juga dipakai `asSafeChunkSize()` di runtime. +- Default 19 MB; berlaku untuk chunked storage DAN S3 multipart parts (sama-sama disimpan ke Telegram lalu di-resolve via getFile). + ## Testing Use `bun test` to run tests. Jalankan tes secara spesifik (misal `bun test test/rateLimit.test.ts`) untuk menghindari polusi mock antar berkas tes ketika dijalankan bersamaan. diff --git a/src/env.ts b/src/env.ts index eff0a3d..396db34 100644 --- a/src/env.ts +++ b/src/env.ts @@ -1,4 +1,5 @@ import logger from './shared/logger/index'; +import { TELEGRAM_CHUNK_SIZE_MAX_BYTES } from './shared/utils/validation'; interface AppConfig { /** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */ @@ -107,6 +108,29 @@ const maskSecret = (value: string): string => { const maskDatabaseUrl = (value: string): string => value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@'); +// Fail-fast guard for TELEGRAM_CHUNK_SIZE_BYTES: chunked uploads store every +// part as a Telegram document and later resolve it via getFile, which only +// supports files up to 20 MB ("Bad Request: file is too big" above that). +// A chunk above the limit makes every part undownloadable — refuse to start +// instead of failing on the first large-file download. +const telegramChunkSizeBytes = parseNumber( + process.env.TELEGRAM_CHUNK_SIZE_BYTES, + TELEGRAM_CHUNK_SIZE_MAX_BYTES, +); +if (telegramChunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) { + logger.error( + `TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` + + `${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` + + 'Telegram Bot API getFile cannot download files larger than 20 MB, so every stored part would ' + + 'be undownloadable ("Bad Request: file is too big"). ' + + `Set TELEGRAM_CHUNK_SIZE_BYTES to ${TELEGRAM_CHUNK_SIZE_MAX_BYTES} or lower.`, + ); + throw new Error( + `TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` + + `${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB)`, + ); +} + export const config: AppConfig = { botTokens: parseTokens(botTokensRaw), telegramBotConcurrency: parseNumber(process.env.TELEGRAM_BOT_CONCURRENCY, 1), @@ -122,7 +146,7 @@ export const config: AppConfig = { batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20), batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024), maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024), - telegramChunkSizeBytes: parseNumber(process.env.TELEGRAM_CHUNK_SIZE_BYTES, 20 * 1024 * 1024), + telegramChunkSizeBytes, compressChunkedUploads: process.env.COMPRESS_CHUNKED_UPLOADS !== 'false', chunkCompressionMinSizeBytes: parseNumber(process.env.CHUNK_COMPRESSION_MIN_SIZE_BYTES, 4096), adminApiToken: process.env.ADMIN_API_TOKEN || '', diff --git a/src/shared/utils/validation.ts b/src/shared/utils/validation.ts index 6c20475..5a0bf19 100644 --- a/src/shared/utils/validation.ts +++ b/src/shared/utils/validation.ts @@ -1,13 +1,33 @@ +/** + * Maximum allowed chunk/part size in bytes for Telegram storage. + * + * Telegram Bot API `getFile` can only resolve files up to 20 MB; anything + * larger fails with "Bad Request: file is too big". Chunked uploads store + * each part as a Telegram document and later resolve it via `getFile`, so a + * part must never reach that limit. 19 MB (19922944 bytes) leaves a safety + * margin and is the value used in production (/etc/teleuploader/env). + */ +export const TELEGRAM_CHUNK_SIZE_MAX_BYTES = 19 * 1024 * 1024; + /** * Validates a chunk size value and returns it as a safe integer. * * @param chunkSizeBytes - The desired chunk size in bytes. - * @returns The same value if it is a positive safe integer. - * @throws {Error} If the chunk size is not a safe positive integer. + * @returns The same value if it is a positive safe integer at or below + * {@link TELEGRAM_CHUNK_SIZE_MAX_BYTES}. + * @throws {Error} If the chunk size is not a safe positive integer or exceeds + * the Telegram `getFile` limit (with margin). */ export const asSafeChunkSize = (chunkSizeBytes: number): number => { if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) { throw new Error('Invalid Telegram chunk size'); } + if (chunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) { + throw new Error( + `Telegram chunk size ${chunkSizeBytes} exceeds the maximum allowed part size ` + + `${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` + + 'Telegram getFile cannot download files larger than 20 MB, so such parts would be undownloadable.', + ); + } return chunkSizeBytes; }; diff --git a/test/env.test.ts b/test/env.test.ts index ba222cb..d46b7fd 100644 --- a/test/env.test.ts +++ b/test/env.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'bun:test'; import { config } from '../src/env'; +import { asSafeChunkSize, TELEGRAM_CHUNK_SIZE_MAX_BYTES } from '../src/shared/utils/validation'; describe('Environment Variables Validation', () => { it('config should have all required fields', () => { @@ -67,3 +68,77 @@ describe('Environment Variables Validation', () => { expect(config.s3SecretKey).toBeDefined(); }); }); + +describe('Telegram chunk size validation', () => { + it('config.telegramChunkSizeBytes should default to the safe 19 MB value when unset', () => { + // setup-env.ts does not set TELEGRAM_CHUNK_SIZE_BYTES, so the default must + // be the safe 19 MB value — never the raw 20 MB getFile limit. + expect(config.telegramChunkSizeBytes).toBe(TELEGRAM_CHUNK_SIZE_MAX_BYTES); + expect(config.telegramChunkSizeBytes).toBeLessThan(20 * 1024 * 1024); + }); + + it('asSafeChunkSize accepts sizes at or below the maximum (19 MB)', () => { + expect(asSafeChunkSize(TELEGRAM_CHUNK_SIZE_MAX_BYTES)).toBe(TELEGRAM_CHUNK_SIZE_MAX_BYTES); + expect(asSafeChunkSize(1024)).toBe(1024); + expect(asSafeChunkSize(19 * 1024 * 1024)).toBe(19 * 1024 * 1024); + }); + + it('asSafeChunkSize rejects sizes above the Telegram getFile limit (incl. the 48 MB production bug)', () => { + // The production bug value (48 MB / 50331648) must be rejected. + expect(() => asSafeChunkSize(48 * 1024 * 1024)).toThrow(/exceeds/); + // Even exactly 20 MB is at the raw Telegram limit — rejected by the margin. + expect(() => asSafeChunkSize(20 * 1024 * 1024)).toThrow(/exceeds/); + expect(() => asSafeChunkSize(TELEGRAM_CHUNK_SIZE_MAX_BYTES + 1)).toThrow(/exceeds/); + }); + + it('asSafeChunkSize rejects non-positive or non-integer sizes', () => { + expect(() => asSafeChunkSize(0)).toThrow('Invalid Telegram chunk size'); + expect(() => asSafeChunkSize(-1)).toThrow('Invalid Telegram chunk size'); + expect(() => asSafeChunkSize(1.5)).toThrow('Invalid Telegram chunk size'); + }); + + it('startup fails fast when TELEGRAM_CHUNK_SIZE_BYTES exceeds the limit', async () => { + // Spawn a real process that imports src/env with an oversized chunk size; + // it must exit non-zero with a clear error instead of starting silently. + const proc = Bun.spawn({ + cmd: ['bun', '-e', "import('./src/env')"], + cwd: `${import.meta.dir}/..`, + env: { + ...process.env, + BOT_TOKENS: '123456:ABC-DEF', + STORAGE_CHANNEL_ID: '-1001234567890', + BASE_URL: 'https://example.com', + DATABASE_URL: 'postgresql://user:***@localhost:5432/test', + PORT: '3000', + TELEGRAM_CHUNK_SIZE_BYTES: String(48 * 1024 * 1024), + }, + stdout: 'pipe', + stderr: 'pipe', + }); + const exitCode = await proc.exited; + const stderr = await new Response(proc.stderr).text(); + expect(exitCode).not.toBe(0); + expect(stderr).toContain('TELEGRAM_CHUNK_SIZE_BYTES'); + expect(stderr).toContain('exceeds'); + }); + + it('startup succeeds when TELEGRAM_CHUNK_SIZE_BYTES is at the safe limit', async () => { + const proc = Bun.spawn({ + cmd: ['bun', '-e', "import('./src/env')"], + cwd: `${import.meta.dir}/..`, + env: { + ...process.env, + BOT_TOKENS: '123456:ABC-DEF', + STORAGE_CHANNEL_ID: '-1001234567890', + BASE_URL: 'https://example.com', + DATABASE_URL: 'postgresql://user:***@localhost:5432/test', + PORT: '3000', + TELEGRAM_CHUNK_SIZE_BYTES: String(TELEGRAM_CHUNK_SIZE_MAX_BYTES), + }, + stdout: 'pipe', + stderr: 'pipe', + }); + const exitCode = await proc.exited; + expect(exitCode).toBe(0); + }); +}); diff --git a/test/helpers/setup-env.ts b/test/helpers/setup-env.ts index 3066afc..0382783 100644 --- a/test/helpers/setup-env.ts +++ b/test/helpers/setup-env.ts @@ -17,4 +17,10 @@ process.env.DATABASE_URL ||= 'postgresql://user:pass@localhost:5432/test'; process.env.PORT ||= '3000'; process.env.NODE_ENV = 'test'; +// Pin the chunk size to the safe 19 MB value UNCONDITIONALLY. Bun auto-loads +// the repo .env before preloads run, and a stale oversized value there would +// trip the fail-fast guard in src/env.ts and break every test file's import. +// Tests that need a different value set it explicitly in their own process. +process.env.TELEGRAM_CHUNK_SIZE_BYTES = String(19 * 1024 * 1024); + // Keep old env names for backward compat with tests that reference them directly