fix: fail-fast guard TELEGRAM_CHUNK_SIZE_BYTES <= 19MB (Telegram getFile limit 20MB)
Build & Deploy (Nix) / build-and-deploy (push) Successful in 56s
Build & Deploy (Nix) / build-and-deploy (push) Successful in 56s
Chunk parts > 19MB are stored to Telegram but getFile cannot resolve files over 20MB ('Bad Request: file is too big'), making every part undownloadable (prod bug 2026-08-01: 48MB chunk -> download 500).
- src/env.ts: reject TELEGRAM_CHUNK_SIZE_BYTES > 19922944 at startup (log error + throw), default changed 20MB -> 19MB
- src/shared/utils/validation.ts: TELEGRAM_CHUNK_SIZE_MAX_BYTES constant; asSafeChunkSize now enforces the max at runtime (covers S3 multipart parts too)
- test/env.test.ts: unit tests + subprocess fail-fast tests (48MB rejected, 19MB accepted)
- test/helpers/setup-env.ts: pin safe chunk size so a stale .env can't break the suite
- .env.example + CLAUDE.md: document the 20MB getFile limit
This commit is contained in:
+2
-2
@@ -16,8 +16,8 @@ RATE_LIMIT_MAX_REQUESTS=30
|
|||||||
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
|
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
|
||||||
|
|
||||||
# Telegram-safe internal chunking for large stored files
|
# Telegram-safe internal chunking for large stored files
|
||||||
# Telegram max upload ~49MB, using 48MB for safety
|
# Telegram getFile download limit is 20 MB; guard rejects > 19922944 (19 MB)
|
||||||
# TELEGRAM_CHUNK_SIZE_BYTES=50331648
|
# TELEGRAM_CHUNK_SIZE_BYTES=19922944
|
||||||
# COMPRESS_CHUNKED_UPLOADS=true
|
# COMPRESS_CHUNKED_UPLOADS=true
|
||||||
# CHUNK_COMPRESSION_MIN_SIZE_BYTES=4096
|
# CHUNK_COMPRESSION_MIN_SIZE_BYTES=4096
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ Default to using Bun instead of Node.js.
|
|||||||
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
|
- Pengiriman berkas ke Telegram dieksekusi secara responsif dan paralel penuh tanpa batas konkurensi/antrian.
|
||||||
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
|
- Berkas API upload ditulis secara sementara ke disk `/tmp/teleuploader-*` dan di-stream ke Telegram menggunakan `fs.createReadStream` (RAM-optimized) lalu dihapus otomatis setelah 50ms (timeout aman).
|
||||||
|
|
||||||
|
## Chunk size (TELEGRAM_CHUNK_SIZE_BYTES)
|
||||||
|
|
||||||
|
- Batas keras: Telegram Bot API `getFile` hanya bisa resolve file ≤ 20 MB — di atas itu error `Bad Request: file is too big` dan part tidak bisa di-download.
|
||||||
|
- Guard fail-fast di `src/env.ts`: service MENOLAK start (exit non-zero) jika `TELEGRAM_CHUNK_SIZE_BYTES` > 19922944 (19 MB, margin aman dari limit 20 MB). Konstanta: `TELEGRAM_CHUNK_SIZE_MAX_BYTES` di `src/shared/utils/validation.ts`, juga dipakai `asSafeChunkSize()` di runtime.
|
||||||
|
- Default 19 MB; berlaku untuk chunked storage DAN S3 multipart parts (sama-sama disimpan ke Telegram lalu di-resolve via getFile).
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
Use `bun test` to run tests. Jalankan tes secara spesifik (misal `bun test test/rateLimit.test.ts`) untuk menghindari polusi mock antar berkas tes ketika dijalankan bersamaan.
|
Use `bun test` to run tests. Jalankan tes secara spesifik (misal `bun test test/rateLimit.test.ts`) untuk menghindari polusi mock antar berkas tes ketika dijalankan bersamaan.
|
||||||
|
|||||||
+25
-1
@@ -1,4 +1,5 @@
|
|||||||
import logger from './shared/logger/index';
|
import logger from './shared/logger/index';
|
||||||
|
import { TELEGRAM_CHUNK_SIZE_MAX_BYTES } from './shared/utils/validation';
|
||||||
|
|
||||||
interface AppConfig {
|
interface AppConfig {
|
||||||
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
|
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
|
||||||
@@ -107,6 +108,29 @@ const maskSecret = (value: string): string => {
|
|||||||
const maskDatabaseUrl = (value: string): string =>
|
const maskDatabaseUrl = (value: string): string =>
|
||||||
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
||||||
|
|
||||||
|
// Fail-fast guard for TELEGRAM_CHUNK_SIZE_BYTES: chunked uploads store every
|
||||||
|
// part as a Telegram document and later resolve it via getFile, which only
|
||||||
|
// supports files up to 20 MB ("Bad Request: file is too big" above that).
|
||||||
|
// A chunk above the limit makes every part undownloadable — refuse to start
|
||||||
|
// instead of failing on the first large-file download.
|
||||||
|
const telegramChunkSizeBytes = parseNumber(
|
||||||
|
process.env.TELEGRAM_CHUNK_SIZE_BYTES,
|
||||||
|
TELEGRAM_CHUNK_SIZE_MAX_BYTES,
|
||||||
|
);
|
||||||
|
if (telegramChunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) {
|
||||||
|
logger.error(
|
||||||
|
`TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` +
|
||||||
|
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` +
|
||||||
|
'Telegram Bot API getFile cannot download files larger than 20 MB, so every stored part would ' +
|
||||||
|
'be undownloadable ("Bad Request: file is too big"). ' +
|
||||||
|
`Set TELEGRAM_CHUNK_SIZE_BYTES to ${TELEGRAM_CHUNK_SIZE_MAX_BYTES} or lower.`,
|
||||||
|
);
|
||||||
|
throw new Error(
|
||||||
|
`TELEGRAM_CHUNK_SIZE_BYTES=${telegramChunkSizeBytes} exceeds the maximum allowed chunk size ` +
|
||||||
|
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export const config: AppConfig = {
|
export const config: AppConfig = {
|
||||||
botTokens: parseTokens(botTokensRaw),
|
botTokens: parseTokens(botTokensRaw),
|
||||||
telegramBotConcurrency: parseNumber(process.env.TELEGRAM_BOT_CONCURRENCY, 1),
|
telegramBotConcurrency: parseNumber(process.env.TELEGRAM_BOT_CONCURRENCY, 1),
|
||||||
@@ -122,7 +146,7 @@ export const config: AppConfig = {
|
|||||||
batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20),
|
batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20),
|
||||||
batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024),
|
batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024),
|
||||||
maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024),
|
maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024),
|
||||||
telegramChunkSizeBytes: parseNumber(process.env.TELEGRAM_CHUNK_SIZE_BYTES, 20 * 1024 * 1024),
|
telegramChunkSizeBytes,
|
||||||
compressChunkedUploads: process.env.COMPRESS_CHUNKED_UPLOADS !== 'false',
|
compressChunkedUploads: process.env.COMPRESS_CHUNKED_UPLOADS !== 'false',
|
||||||
chunkCompressionMinSizeBytes: parseNumber(process.env.CHUNK_COMPRESSION_MIN_SIZE_BYTES, 4096),
|
chunkCompressionMinSizeBytes: parseNumber(process.env.CHUNK_COMPRESSION_MIN_SIZE_BYTES, 4096),
|
||||||
adminApiToken: process.env.ADMIN_API_TOKEN || '',
|
adminApiToken: process.env.ADMIN_API_TOKEN || '',
|
||||||
|
|||||||
@@ -1,13 +1,33 @@
|
|||||||
|
/**
|
||||||
|
* Maximum allowed chunk/part size in bytes for Telegram storage.
|
||||||
|
*
|
||||||
|
* Telegram Bot API `getFile` can only resolve files up to 20 MB; anything
|
||||||
|
* larger fails with "Bad Request: file is too big". Chunked uploads store
|
||||||
|
* each part as a Telegram document and later resolve it via `getFile`, so a
|
||||||
|
* part must never reach that limit. 19 MB (19922944 bytes) leaves a safety
|
||||||
|
* margin and is the value used in production (/etc/teleuploader/env).
|
||||||
|
*/
|
||||||
|
export const TELEGRAM_CHUNK_SIZE_MAX_BYTES = 19 * 1024 * 1024;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validates a chunk size value and returns it as a safe integer.
|
* Validates a chunk size value and returns it as a safe integer.
|
||||||
*
|
*
|
||||||
* @param chunkSizeBytes - The desired chunk size in bytes.
|
* @param chunkSizeBytes - The desired chunk size in bytes.
|
||||||
* @returns The same value if it is a positive safe integer.
|
* @returns The same value if it is a positive safe integer at or below
|
||||||
* @throws {Error} If the chunk size is not a safe positive integer.
|
* {@link TELEGRAM_CHUNK_SIZE_MAX_BYTES}.
|
||||||
|
* @throws {Error} If the chunk size is not a safe positive integer or exceeds
|
||||||
|
* the Telegram `getFile` limit (with margin).
|
||||||
*/
|
*/
|
||||||
export const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
export const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
||||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
||||||
throw new Error('Invalid Telegram chunk size');
|
throw new Error('Invalid Telegram chunk size');
|
||||||
}
|
}
|
||||||
|
if (chunkSizeBytes > TELEGRAM_CHUNK_SIZE_MAX_BYTES) {
|
||||||
|
throw new Error(
|
||||||
|
`Telegram chunk size ${chunkSizeBytes} exceeds the maximum allowed part size ` +
|
||||||
|
`${TELEGRAM_CHUNK_SIZE_MAX_BYTES} bytes (${TELEGRAM_CHUNK_SIZE_MAX_BYTES / (1024 * 1024)} MB). ` +
|
||||||
|
'Telegram getFile cannot download files larger than 20 MB, so such parts would be undownloadable.',
|
||||||
|
);
|
||||||
|
}
|
||||||
return chunkSizeBytes;
|
return chunkSizeBytes;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { describe, expect, it } from 'bun:test';
|
import { describe, expect, it } from 'bun:test';
|
||||||
import { config } from '../src/env';
|
import { config } from '../src/env';
|
||||||
|
import { asSafeChunkSize, TELEGRAM_CHUNK_SIZE_MAX_BYTES } from '../src/shared/utils/validation';
|
||||||
|
|
||||||
describe('Environment Variables Validation', () => {
|
describe('Environment Variables Validation', () => {
|
||||||
it('config should have all required fields', () => {
|
it('config should have all required fields', () => {
|
||||||
@@ -67,3 +68,77 @@ describe('Environment Variables Validation', () => {
|
|||||||
expect(config.s3SecretKey).toBeDefined();
|
expect(config.s3SecretKey).toBeDefined();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('Telegram chunk size validation', () => {
|
||||||
|
it('config.telegramChunkSizeBytes should default to the safe 19 MB value when unset', () => {
|
||||||
|
// setup-env.ts does not set TELEGRAM_CHUNK_SIZE_BYTES, so the default must
|
||||||
|
// be the safe 19 MB value — never the raw 20 MB getFile limit.
|
||||||
|
expect(config.telegramChunkSizeBytes).toBe(TELEGRAM_CHUNK_SIZE_MAX_BYTES);
|
||||||
|
expect(config.telegramChunkSizeBytes).toBeLessThan(20 * 1024 * 1024);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('asSafeChunkSize accepts sizes at or below the maximum (19 MB)', () => {
|
||||||
|
expect(asSafeChunkSize(TELEGRAM_CHUNK_SIZE_MAX_BYTES)).toBe(TELEGRAM_CHUNK_SIZE_MAX_BYTES);
|
||||||
|
expect(asSafeChunkSize(1024)).toBe(1024);
|
||||||
|
expect(asSafeChunkSize(19 * 1024 * 1024)).toBe(19 * 1024 * 1024);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('asSafeChunkSize rejects sizes above the Telegram getFile limit (incl. the 48 MB production bug)', () => {
|
||||||
|
// The production bug value (48 MB / 50331648) must be rejected.
|
||||||
|
expect(() => asSafeChunkSize(48 * 1024 * 1024)).toThrow(/exceeds/);
|
||||||
|
// Even exactly 20 MB is at the raw Telegram limit — rejected by the margin.
|
||||||
|
expect(() => asSafeChunkSize(20 * 1024 * 1024)).toThrow(/exceeds/);
|
||||||
|
expect(() => asSafeChunkSize(TELEGRAM_CHUNK_SIZE_MAX_BYTES + 1)).toThrow(/exceeds/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('asSafeChunkSize rejects non-positive or non-integer sizes', () => {
|
||||||
|
expect(() => asSafeChunkSize(0)).toThrow('Invalid Telegram chunk size');
|
||||||
|
expect(() => asSafeChunkSize(-1)).toThrow('Invalid Telegram chunk size');
|
||||||
|
expect(() => asSafeChunkSize(1.5)).toThrow('Invalid Telegram chunk size');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('startup fails fast when TELEGRAM_CHUNK_SIZE_BYTES exceeds the limit', async () => {
|
||||||
|
// Spawn a real process that imports src/env with an oversized chunk size;
|
||||||
|
// it must exit non-zero with a clear error instead of starting silently.
|
||||||
|
const proc = Bun.spawn({
|
||||||
|
cmd: ['bun', '-e', "import('./src/env')"],
|
||||||
|
cwd: `${import.meta.dir}/..`,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
BOT_TOKENS: '123456:ABC-DEF',
|
||||||
|
STORAGE_CHANNEL_ID: '-1001234567890',
|
||||||
|
BASE_URL: 'https://example.com',
|
||||||
|
DATABASE_URL: 'postgresql://user:***@localhost:5432/test',
|
||||||
|
PORT: '3000',
|
||||||
|
TELEGRAM_CHUNK_SIZE_BYTES: String(48 * 1024 * 1024),
|
||||||
|
},
|
||||||
|
stdout: 'pipe',
|
||||||
|
stderr: 'pipe',
|
||||||
|
});
|
||||||
|
const exitCode = await proc.exited;
|
||||||
|
const stderr = await new Response(proc.stderr).text();
|
||||||
|
expect(exitCode).not.toBe(0);
|
||||||
|
expect(stderr).toContain('TELEGRAM_CHUNK_SIZE_BYTES');
|
||||||
|
expect(stderr).toContain('exceeds');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('startup succeeds when TELEGRAM_CHUNK_SIZE_BYTES is at the safe limit', async () => {
|
||||||
|
const proc = Bun.spawn({
|
||||||
|
cmd: ['bun', '-e', "import('./src/env')"],
|
||||||
|
cwd: `${import.meta.dir}/..`,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
BOT_TOKENS: '123456:ABC-DEF',
|
||||||
|
STORAGE_CHANNEL_ID: '-1001234567890',
|
||||||
|
BASE_URL: 'https://example.com',
|
||||||
|
DATABASE_URL: 'postgresql://user:***@localhost:5432/test',
|
||||||
|
PORT: '3000',
|
||||||
|
TELEGRAM_CHUNK_SIZE_BYTES: String(TELEGRAM_CHUNK_SIZE_MAX_BYTES),
|
||||||
|
},
|
||||||
|
stdout: 'pipe',
|
||||||
|
stderr: 'pipe',
|
||||||
|
});
|
||||||
|
const exitCode = await proc.exited;
|
||||||
|
expect(exitCode).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -17,4 +17,10 @@ process.env.DATABASE_URL ||= 'postgresql://user:pass@localhost:5432/test';
|
|||||||
process.env.PORT ||= '3000';
|
process.env.PORT ||= '3000';
|
||||||
process.env.NODE_ENV = 'test';
|
process.env.NODE_ENV = 'test';
|
||||||
|
|
||||||
|
// Pin the chunk size to the safe 19 MB value UNCONDITIONALLY. Bun auto-loads
|
||||||
|
// the repo .env before preloads run, and a stale oversized value there would
|
||||||
|
// trip the fail-fast guard in src/env.ts and break every test file's import.
|
||||||
|
// Tests that need a different value set it explicitly in their own process.
|
||||||
|
process.env.TELEGRAM_CHUNK_SIZE_BYTES = String(19 * 1024 * 1024);
|
||||||
|
|
||||||
// Keep old env names for backward compat with tests that reference them directly
|
// Keep old env names for backward compat with tests that reference them directly
|
||||||
|
|||||||
Reference in New Issue
Block a user