fix: fail-fast guard TELEGRAM_CHUNK_SIZE_BYTES <= 19MB (Telegram getFile limit 20MB)
Build & Deploy (Nix) / build-and-deploy (push) Successful in 56s

Chunk parts > 19MB are stored to Telegram but getFile cannot resolve files over 20MB ('Bad Request: file is too big'), making every part undownloadable (prod bug 2026-08-01: 48MB chunk -> download 500).

- src/env.ts: reject TELEGRAM_CHUNK_SIZE_BYTES > 19922944 at startup (log error + throw), default changed 20MB -> 19MB
- src/shared/utils/validation.ts: TELEGRAM_CHUNK_SIZE_MAX_BYTES constant; asSafeChunkSize now enforces the max at runtime (covers S3 multipart parts too)
- test/env.test.ts: unit tests + subprocess fail-fast tests (48MB rejected, 19MB accepted)
- test/helpers/setup-env.ts: pin safe chunk size so a stale .env can't break the suite
- .env.example + CLAUDE.md: document the 20MB getFile limit
This commit is contained in:
asepharyana
2026-08-01 13:06:42 +07:00
parent b9a3fcd828
commit 24dfb1c6b1
6 changed files with 136 additions and 5 deletions
+2 -2
View File
@@ -16,8 +16,8 @@ RATE_LIMIT_MAX_REQUESTS=30
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
# Telegram-safe internal chunking for large stored files
# Telegram max upload ~49MB, using 48MB for safety
# TELEGRAM_CHUNK_SIZE_BYTES=50331648
# Telegram getFile download limit is 20 MB; guard rejects > 19922944 (19 MB)
# TELEGRAM_CHUNK_SIZE_BYTES=19922944
# COMPRESS_CHUNKED_UPLOADS=true
# CHUNK_COMPRESSION_MIN_SIZE_BYTES=4096
+6
View File
@@ -25,6 +25,12 @@ Default to using Bun instead of Node.js.
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
## Chunk size (TELEGRAM_CHUNK_SIZE_BYTES)
- Batas keras: Telegram Bot API `getFile` hanya bisa resolve file ≤ 20 MB — di atas itu error `Bad Request: file is too big` dan part tidak bisa di-download.
- Guard fail-fast di `src/env.ts`: service MENOLAK start (exit non-zero) jika `TELEGRAM_CHUNK_SIZE_BYTES` > 19922944 (19 MB, margin aman dari limit 20 MB). Konstanta: `TELEGRAM_CHUNK_SIZE_MAX_BYTES` di `src/shared/utils/validation.ts`, juga dipakai `asSafeChunkSize()` di runtime.
- Default 19 MB; berlaku untuk chunked storage DAN S3 multipart parts (sama-sama disimpan ke Telegram lalu di-resolve via getFile).
## Testing
Use `bun test` to run tests. Jalankan tes secara spesifik (misal `bun test test/rateLimit.test.ts`) untuk menghindari polusi mock antar berkas tes ketika dijalankan bersamaan.
+25 -1
View File
@@ -1,4 +1,5 @@
import logger from './shared/logger/index';
import { TELEGRAM_CHUNK_SIZE_MAX_BYTES } from './shared/utils/validation';
interface AppConfig {
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
@@ -107,6 +108,29 @@ const maskSecret = (value: string): string => {
const maskDatabaseUrl = (value: string): string =>
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
// Fail-fast guard for TELEGRAM_CHUNK_SIZE_BYTES: chunked uploads store every
// part as a Telegram document and later resolve it via getFile, which only
// supports files up to 20 MB ("Bad Request: file is too big" above that).
// A chunk above the limit makes every part undownloadable — refuse to start
// instead of failing on the first large-file download.
const telegramChunkSizeBytes = parseNumber(
process.env.TELEGRAM_CHUNK_SIZE_BYTES,
TELEGRAM_CHUNK_SIZE_MAX_BYTES,
);
if (telegramChunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) {
logger.error(
`TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` +
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` +
'Telegram Bot API getFile cannot download files larger than 20 MB, so every stored part would ' +
'be undownloadable ("Bad Request: file is too big"). ' +
`Set TELEGRAM_CHUNK_SIZE_BYTES to ${TELEGRAM_CHUNK_SIZE_MAX_BYTES} or lower.`,
);
throw new Error(
`TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` +
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB)`,
);
}
export const config: AppConfig = {
botTokens: parseTokens(botTokensRaw),
telegramBotConcurrency: parseNumber(process.env.TELEGRAM_BOT_CONCURRENCY, 1),
@@ -122,7 +146,7 @@ export const config: AppConfig = {
batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20),
batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024),
maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024),
telegramChunkSizeBytes: parseNumber(process.env.TELEGRAM_CHUNK_SIZE_BYTES, 20 * 1024 * 1024),
telegramChunkSizeBytes,
compressChunkedUploads: process.env.COMPRESS_CHUNKED_UPLOADS !== 'false',
chunkCompressionMinSizeBytes: parseNumber(process.env.CHUNK_COMPRESSION_MIN_SIZE_BYTES, 4096),
adminApiToken: process.env.ADMIN_API_TOKEN || '',
+22 -2
View File
@@ -1,13 +1,33 @@
/**
* Maximum allowed chunk/part size in bytes for Telegram storage.
*
* Telegram Bot API `getFile` can only resolve files up to 20 MB; anything
* larger fails with "Bad Request: file is too big". Chunked uploads store
* each part as a Telegram document and later resolve it via `getFile`, so a
* part must never reach that limit. 19 MB (19922944 bytes) leaves a safety
* margin and is the value used in production (/etc/teleuploader/env).
*/
export const TELEGRAM_CHUNK_SIZE_MAX_BYTES = 19 * 1024 * 1024;
/**
* Validates a chunk size value and returns it as a safe integer.
*
* @param chunkSizeBytes - The desired chunk size in bytes.
* @returns The same value if it is a positive safe integer.
* @throws {Error} If the chunk size is not a safe positive integer.
* @returns The same value if it is a positive safe integer at or below
* {@link TELEGRAM_CHUNK_SIZE_MAX_BYTES}.
* @throws {Error} If the chunk size is not a safe positive integer or exceeds
* the Telegram `getFile` limit (with margin).
*/
export const asSafeChunkSize = (chunkSizeBytes: number): number => {
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
throw new Error('Invalid Telegram chunk size');
}
if (chunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) {
throw new Error(
`Telegram chunk size ${chunkSizeBytes} exceeds the maximum allowed part size ` +
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` +
'Telegram getFile cannot download files larger than 20 MB, so such parts would be undownloadable.',
);
}
return chunkSizeBytes;
};
+75
View File
@@ -1,5 +1,6 @@
import { describe, expect, it } from 'bun:test';
import { config } from '../src/env';
import { asSafeChunkSize, TELEGRAM_CHUNK_SIZE_MAX_BYTES } from '../src/shared/utils/validation';
describe('Environment Variables Validation', () => {
it('config should have all required fields', () => {
@@ -67,3 +68,77 @@ describe('Environment Variables Validation', () => {
expect(config.s3SecretKey).toBeDefined();
});
});
describe('Telegram chunk size validation', () => {
it('config.telegramChunkSizeBytes should default to the safe 19 MB value when unset', () => {
// setup-env.ts does not set TELEGRAM_CHUNK_SIZE_BYTES, so the default must
// be the safe 19 MB value — never the raw 20 MB getFile limit.
expect(config.telegramChunkSizeBytes).toBe(TELEGRAM_CHUNK_SIZE_MAX_BYTES);
expect(config.telegramChunkSizeBytes).toBeLessThan(20 * 1024 * 1024);
});
it('asSafeChunkSize accepts sizes at or below the maximum (19 MB)', () => {
expect(asSafeChunkSize(TELEGRAM_CHUNK_SIZE_MAX_BYTES)).toBe(TELEGRAM_CHUNK_SIZE_MAX_BYTES);
expect(asSafeChunkSize(1024)).toBe(1024);
expect(asSafeChunkSize(19 * 1024 * 1024)).toBe(19 * 1024 * 1024);
});
it('asSafeChunkSize rejects sizes above the Telegram getFile limit (incl. the 48 MB production bug)', () => {
// The production bug value (48 MB / 50331648) must be rejected.
expect(() => asSafeChunkSize(48 * 1024 * 1024)).toThrow(/exceeds/);
// Even exactly 20 MB is at the raw Telegram limit — rejected by the margin.
expect(() => asSafeChunkSize(20 * 1024 * 1024)).toThrow(/exceeds/);
expect(() => asSafeChunkSize(TELEGRAM_CHUNK_SIZE_MAX_BYTES + 1)).toThrow(/exceeds/);
});
it('asSafeChunkSize rejects non-positive or non-integer sizes', () => {
expect(() => asSafeChunkSize(0)).toThrow('Invalid Telegram chunk size');
expect(() => asSafeChunkSize(-1)).toThrow('Invalid Telegram chunk size');
expect(() => asSafeChunkSize(1.5)).toThrow('Invalid Telegram chunk size');
});
it('startup fails fast when TELEGRAM_CHUNK_SIZE_BYTES exceeds the limit', async () => {
// Spawn a real process that imports src/env with an oversized chunk size;
// it must exit non-zero with a clear error instead of starting silently.
const proc = Bun.spawn({
cmd: ['bun', '-e', "import('./src/env')"],
cwd: `${import.meta.dir}/..`,
env: {
...process.env,
BOT_TOKENS: '123456:ABC-DEF',
STORAGE_CHANNEL_ID: '-1001234567890',
BASE_URL: 'https://example.com',
DATABASE_URL: 'postgresql://user:***@localhost:5432/test',
PORT: '3000',
TELEGRAM_CHUNK_SIZE_BYTES: String(48 * 1024 * 1024),
},
stdout: 'pipe',
stderr: 'pipe',
});
const exitCode = await proc.exited;
const stderr = await new Response(proc.stderr).text();
expect(exitCode).not.toBe(0);
expect(stderr).toContain('TELEGRAM_CHUNK_SIZE_BYTES');
expect(stderr).toContain('exceeds');
});
it('startup succeeds when TELEGRAM_CHUNK_SIZE_BYTES is at the safe limit', async () => {
const proc = Bun.spawn({
cmd: ['bun', '-e', "import('./src/env')"],
cwd: `${import.meta.dir}/..`,
env: {
...process.env,
BOT_TOKENS: '123456:ABC-DEF',
STORAGE_CHANNEL_ID: '-1001234567890',
BASE_URL: 'https://example.com',
DATABASE_URL: 'postgresql://user:***@localhost:5432/test',
PORT: '3000',
TELEGRAM_CHUNK_SIZE_BYTES: String(TELEGRAM_CHUNK_SIZE_MAX_BYTES),
},
stdout: 'pipe',
stderr: 'pipe',
});
const exitCode = await proc.exited;
expect(exitCode).toBe(0);
});
});
+6
View File
@@ -17,4 +17,10 @@ process.env.DATABASE_URL ||= 'postgresql://user:pass@localhost:5432/test';
process.env.PORT ||= '3000';
process.env.NODE_ENV = 'test';
// Pin the chunk size to the safe 19 MB value UNCONDITIONALLY. Bun auto-loads
// the repo .env before preloads run, and a stale oversized value there would
// trip the fail-fast guard in src/env.ts and break every test file's import.
// Tests that need a different value set it explicitly in their own process.
process.env.TELEGRAM_CHUNK_SIZE_BYTES = String(19 * 1024 * 1024);
// Keep old env names for backward compat with tests that reference them directly