From 5425f6d33ddc7d85095e40ca3922189a26893ba6 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Fri, 29 May 2026 03:33:39 +0700 Subject: [PATCH] feat: enhance configuration and rate limiting - Added new configuration options: trustProxy, uploadConcurrency, batchMaxItems, batchMaxSizeBytes, and maxRequestBodyBytes to AppConfig. - Implemented utility functions for parsing environment variables and masking sensitive data. - Updated rate limiting logic to use configurable window size and maximum requests per window. - Introduced a middleware for rate limiting on specific routes. - Refactored file handling routes to support streaming downloads instead of redirects. - Improved error handling and response formatting in file routes. - Added support for oversized request rejection based on Content-Length header. - Updated Swagger documentation to reflect changes in API behavior and responses. - Enhanced tests to cover new features and ensure proper functionality. --- docker-compose.yml | 36 ++++++++++++++- src/env.ts | 48 +++++++++++++++---- src/index.ts | 8 ++-- src/routes/files.ts | 89 +++++++++++++++++++++++++----------- src/routes/swagger.ts | 42 ++++++----------- src/routes/upload.ts | 33 +++++++++++-- src/utils/file.ts | 10 ---- src/utils/ip.ts | 16 +++++++ src/utils/rateLimit.ts | 94 +++++++++++++++++++++++++------------- src/utils/telegramQueue.ts | 3 +- src/utils/uploadBatcher.ts | 4 +- src/utils/zip.ts | 89 ++++++++++++++++++++++++++++-------- test/files.test.ts | 66 ++++++++++++++------------ test/rateLimit.test.ts | 48 +++++++++++++------ test/swagger.test.ts | 26 +++++++++-- test/upload.test.ts | 55 ++++++++++++++-------- 16 files changed, 466 insertions(+), 201 deletions(-) create mode 100644 src/utils/ip.ts diff --git a/docker-compose.yml b/docker-compose.yml index 89a2153..a0e0d0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,12 +7,41 @@ services: restart: always environment: - BOT_TOKEN=${BOT_TOKEN} + - ADDITIONAL_BOT_TOKENS=${ADDITIONAL_BOT_TOKENS:-} - STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID} - BASE_URL=${BASE_URL} - DATABASE_URL=${DATABASE_URL} - PORT=3000 - NODE_ENV=production - LOG_LEVEL=info + - TRUST_PROXY=true + - UPLOAD_CONCURRENCY=${UPLOAD_CONCURRENCY:-8} + - BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20} + - BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000} + - MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648} + - RATE_LIMIT_WINDOW_MS=${RATE_LIMIT_WINDOW_MS:-60000} + - RATE_LIMIT_MAX_REQUESTS=${RATE_LIMIT_MAX_REQUESTS:-30} + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp:size=4g,mode=1777 + deploy: + resources: + limits: + cpus: '2.0' + memory: 2G + reservations: + cpus: '0.5' + memory: 512M + healthcheck: + test: + - CMD-SHELL + - curl -sf http://localhost:3000/health || exit 1 + interval: 30s + timeout: 10s + retries: 3 + start_period: 15s networks: - app-shared-net labels: @@ -22,8 +51,13 @@ services: - "traefik.http.routers.teleuploader.tls=true" - "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt" - "traefik.http.services.teleuploader.loadbalancer.server.port=3000" + - "traefik.http.middlewares.teleuploader-rl.ratelimit.average=300" + - "traefik.http.middlewares.teleuploader-rl.ratelimit.burst=100" + - "traefik.http.middlewares.teleuploader-rl.ratelimit.period=1m" + - "traefik.http.middlewares.teleuploader-buf.buffering.maxRequestBodyBytes=2147483648" + - "traefik.http.routers.teleuploader.middlewares=teleuploader-rl@docker,teleuploader-buf@docker" networks: app-shared-net: name: app-shared-net - external: true \ No newline at end of file + external: true diff --git a/src/env.ts b/src/env.ts index 2c95d5d..7c37702 100644 --- a/src/env.ts +++ b/src/env.ts @@ -11,6 +11,11 @@ interface AppConfig { logLevel: string; rateLimitWindowMs: number; rateLimitMaxRequests: number; + trustProxy: boolean; + uploadConcurrency: number; + batchMaxItems: number; + batchMaxSizeBytes: number; + maxRequestBodyBytes: number; } const requiredEnv = { @@ -30,25 +35,48 @@ if (missing.length > 0) { throw new Error(`Missing environment variables: ${missing.join(', ')}`); } +const parseNumber = (value: string | undefined, fallback: number): number => { + const parsed = Number.parseInt(value || '', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +}; + +const parseTokens = (value: string | undefined): string[] => + (value || '') + .split(',') + .map((t) => t.trim()) + .filter((t) => t !== ''); + +const maskSecret = (value: string): string => { + if (!value) return ''; + if (value.length <= 10) return '***'; + return `${value.slice(0, 6)}...${value.slice(-4)}`; +}; + +const maskDatabaseUrl = (value: string): string => value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@'); + export const config: AppConfig = { botToken: process.env.BOT_TOKEN!, - additionalBotTokens: - process.env.NODE_ENV === 'test' - ? [] - : (process.env.ADDITIONAL_BOT_TOKENS || '') - .split(',') - .map((t) => t.trim()) - .filter((t) => t !== ''), + additionalBotTokens: process.env.NODE_ENV === 'test' ? [] : parseTokens(process.env.ADDITIONAL_BOT_TOKENS), storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10), baseUrl: process.env.BASE_URL!, databaseUrl: process.env.DATABASE_URL!, port: parseInt(process.env.PORT!, 10) || 3000, nodeEnv: process.env.NODE_ENV || 'development', logLevel: process.env.LOG_LEVEL || 'info', - rateLimitWindowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS!, 10) || 60000, - rateLimitMaxRequests: parseInt(process.env.RATE_LIMIT_MAX_REQUESTS!, 10) || 30, + rateLimitWindowMs: parseNumber(process.env.RATE_LIMIT_WINDOW_MS, 60000), + rateLimitMaxRequests: parseNumber(process.env.RATE_LIMIT_MAX_REQUESTS, 150), + trustProxy: process.env.TRUST_PROXY === 'true', + uploadConcurrency: parseNumber(process.env.UPLOAD_CONCURRENCY, 8), + batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20), + batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024), + maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024), }; logger.info('Environment variables loaded', { - config: { ...config, botToken: `${config.botToken?.substring(0, 10)}...` }, + config: { + ...config, + botToken: maskSecret(config.botToken), + additionalBotTokens: config.additionalBotTokens.map(maskSecret), + databaseUrl: maskDatabaseUrl(config.databaseUrl), + }, }); diff --git a/src/index.ts b/src/index.ts index 591d44b..5049597 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,19 +6,19 @@ import { handleHealth } from './routes/health'; import { handleSwaggerHtml, handleSwaggerJson } from './routes/swagger'; import { handleUpload } from './routes/upload'; import logger from './utils/logger'; -import { cleanupRateLimitCache } from './utils/rateLimit'; +import { cleanupRateLimitCache, withRateLimit } from './utils/rateLimit'; const server = serve({ port: config.port, routes: { '/api/upload': { - POST: handleUpload, + POST: withRateLimit(handleUpload), }, '/f/:public_id': { - GET: handleFileRedirect, + GET: withRateLimit(handleFileRedirect), }, '/file/:public_id/info': { - GET: handleFileInfo, + GET: withRateLimit(handleFileInfo), }, '/health': { GET: handleHealth, diff --git a/src/routes/files.ts b/src/routes/files.ts index 829b5dc..01c8ed5 100644 --- a/src/routes/files.ts +++ b/src/routes/files.ts @@ -1,10 +1,12 @@ +import { createReadStream } from 'node:fs'; +import { unlink } from 'node:fs/promises'; +import { nanoid } from 'nanoid'; import { findFileByPublicId } from '../db/files'; import { fileInfoCache } from '../utils/cache'; import { formatCreatedAt, getErrorMessage } from '../utils/file'; import logger from '../utils/logger'; -import { checkRateLimit } from '../utils/rateLimit'; import { getBot } from '../utils/telegram'; -import { extractZipEntry } from '../utils/zip'; +import { locateZipEntry } from '../utils/zip'; type RequestWithParams = Request & { params?: { @@ -36,20 +38,31 @@ const getTelegramFileInfo = async (telegramFileId: string, public_id: string) => const buildTelegramFileUrl = (filePath: string): string => `https://api.telegram.org/file/bot${process.env.BOT_TOKEN}/${filePath}`; +const cleanupTempFile = async (tempPath: string): Promise => { + try { + await unlink(tempPath); + } catch (err) { + logger.warn('Failed to cleanup temp file', { tempPath, error: getErrorMessage(err) }); + } +}; + +const sanitizeFilenameHeader = (fileName: string): string => + fileName.replace(/[\\"]/g, '').replace(/[\n\r]/g, ''); + +const fail = (status: number, error: string): Response => + Response.json({ error }, { status }); + export const handleFileRedirect = async (req: RequestWithParams): Promise => { const public_id = req.params?.public_id; try { - const ip = req.headers.get('x-forwarded-for') || '127.0.0.1'; - - if (!public_id || !checkRateLimit(ip)) { - return Response.json({ error: 'Rate limit exceeded' }, { status: 429 }); + if (!public_id) { + return fail(400, 'Missing file id'); } const file = await findFileByPublicId(public_id); - if (!file) { logger.warn('File not found', { public_id }); - return Response.json({ error: 'File not found' }, { status: 404 }); + return fail(404, 'File not found'); } const archiveEntryName = file.archiveEntryName; @@ -60,37 +73,60 @@ export const handleFileRedirect = async (req: RequestWithParams): Promise { + void cleanupTempFile(tempZipPath); + }); + fileStream.on('error', () => { + void cleanupTempFile(tempZipPath); + }); + + return new Response(fileStream as any, { status: 200, headers: { - 'Content-Type': file.mimeType, - 'Content-Disposition': `attachment; filename="${file.fileName.replace(/"/g, '')}"`, - 'Content-Length': String(extractedFile.byteLength), + 'Content-Type': file.mimeType || 'application/octet-stream', + 'Content-Disposition': `attachment; filename="${sanitizeFilenameHeader(file.fileName)}"`, + 'Content-Length': String(loc.length), }, }); } const fileInfo = await getTelegramFileInfo(file.telegramFileId, public_id); - const redirectUrl = buildTelegramFileUrl(fileInfo.file_path); - return new Response(null, { - status: 302, + const tgResponse = await fetch(buildTelegramFileUrl(fileInfo.file_path)); + + if (!tgResponse.ok) { + logger.error('File download failed', { public_id, status: tgResponse.status }); + return fail(502, 'Server error'); + } + + return new Response(tgResponse.body, { + status: 200, headers: { - Location: redirectUrl, + 'Content-Type': file.mimeType || 'application/octet-stream', + 'Content-Disposition': `attachment; filename="${sanitizeFilenameHeader(file.fileName)}"`, + 'Content-Length': String(file.sizeBytes), }, }); } catch (error: unknown) { logger.error('File redirect error', { public_id, error: getErrorMessage(error) }); - return Response.json({ error: 'Server error' }, { status: 500 }); + return fail(500, 'Server error'); } }; @@ -98,15 +134,15 @@ export const handleFileInfo = async (req: RequestWithParams): Promise const public_id = req.params?.public_id; try { if (!public_id) { - return Response.json({ error: 'Missing file id' }, { status: 400 }); + return fail(400, 'Missing file id'); } const file = await findFileByPublicId(public_id); - if (!file) { logger.warn('File not found', { public_id }); - return Response.json({ error: 'File not found' }, { status: 404 }); + return fail(404, 'File not found'); } + return Response.json( { public_id: file.publicId, @@ -114,13 +150,12 @@ export const handleFileInfo = async (req: RequestWithParams): Promise mime_type: file.mimeType, size_bytes: file.sizeBytes, file_type: file.fileType, - uploader_id: file.uploaderId, created_at: formatCreatedAt(file.createdAt), }, { status: 200 }, ); } catch (error: unknown) { logger.error('File info error', { public_id, error: getErrorMessage(error) }); - return Response.json({ error: 'Server error' }, { status: 500 }); + return fail(500, 'Server error'); } }; diff --git a/src/routes/swagger.ts b/src/routes/swagger.ts index 4eea234..52c0fba 100644 --- a/src/routes/swagger.ts +++ b/src/routes/swagger.ts @@ -25,7 +25,6 @@ const fileInfoProperties = { mime_type: { type: 'string', example: 'application/pdf' }, size_bytes: { type: 'integer', example: 1048576 }, file_type: { type: 'string', example: 'document' }, - uploader_id: { type: 'integer', example: 0 }, created_at: { type: 'string', format: 'date-time', @@ -35,10 +34,6 @@ const fileInfoProperties = { const uploadProperties = { ...fileInfoProperties, - telegram_file_id: { type: 'string', example: 'BQACAgQAAxkBA...' }, - telegram_file_unique_id: { type: 'string', example: 'AgAD8w...' }, - storage_chat_id: { type: 'integer', example: -1001234567890 }, - storage_message_id: { type: 'integer', example: 42 }, download_url: { type: 'string', example: `${config.baseUrl}/f/xYz123`, @@ -56,7 +51,7 @@ export const handleSwaggerJson = async (): Promise => { info: { title: 'TeleUploader API', version: '1.0.0', - description: 'Telegram-backed file uploader API with redirect-based downloads.', + description: 'Telegram-backed file uploader API with stream-based downloads.', }, servers: [ { @@ -95,7 +90,7 @@ export const handleSwaggerJson = async (): Promise => { '/api/upload': { post: { summary: 'Upload File', - description: 'Uploads a file to Telegram storage via multipart/form-data or JSON base64.', + description: 'Uploads a file to Telegram storage via multipart/form-data or JSON base64. Rate-limited by IP.', requestBody: { required: true, content: { @@ -144,6 +139,14 @@ export const handleSwaggerJson = async (): Promise => { description: 'Bad request.', content: jsonContent(errorSchema('No file provided')), }, + '413': { + description: 'Request body too large.', + content: jsonContent(errorSchema('Request body too large')), + }, + '429': { + description: 'Rate limit exceeded.', + content: jsonContent(errorSchema('Rate limit exceeded')), + }, '500': { description: 'Internal server error.', content: jsonContent(errorSchema('Upload failed')), @@ -153,21 +156,12 @@ export const handleSwaggerJson = async (): Promise => { }, '/f/{public_id}': { get: { - summary: 'Redirect to Telegram File URL', - description: - 'Gets a fresh Telegram download URL and redirects with 302. Rate-limited by IP.', + summary: 'Download File', + description: 'Proxies file from Telegram storage as a streamed download. Rate-limited by IP.', parameters: [publicIdParameter], responses: { - '302': { - description: 'Redirect to Telegram CDN URL.', - headers: { - Location: { - schema: { - type: 'string', - example: 'https://api.telegram.org/file/botTOKEN/documents/file_0.pdf', - }, - }, - }, + '200': { + description: 'File binary stream.', }, '404': { description: 'File not found.', @@ -212,12 +206,7 @@ export const handleSwaggerJson = async (): Promise => { }, }; - return Response.json(spec, { - status: 200, - headers: { - 'access-control-allow-origin': '*', - }, - }); + return Response.json(spec, { status: 200 }); }; export const handleSwaggerHtml = async (): Promise => { @@ -256,7 +245,6 @@ export const handleSwaggerHtml = async (): Promise => { status: 200, headers: { 'content-type': 'text/html; charset=utf-8', - 'access-control-allow-origin': '*', 'x-content-type-options': 'nosniff', }, }); diff --git a/src/routes/upload.ts b/src/routes/upload.ts index b221d81..0c3f960 100644 --- a/src/routes/upload.ts +++ b/src/routes/upload.ts @@ -39,6 +39,23 @@ const normalizeFileType = (mimeType: string, fileName: string): string => { const JSON_UPLOAD_LIMIT_BYTES = 50 * 1024 * 1024; const SIGNATURE_BYTES = 16; +const getContentLength = (req: Request): number | null => { + const value = req.headers.get('content-length'); + if (!value) return null; + + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; +}; + +const rejectOversizedRequest = (req: Request): Response | null => { + const contentLength = getContentLength(req); + if (contentLength !== null && contentLength > config.maxRequestBodyBytes) { + return Response.json({ error: 'Request body too large' }, { status: 413 }); + } + + return null; +}; + const cleanupTempFile = async (tempPath: string): Promise => { try { await unlink(tempPath); @@ -47,7 +64,7 @@ const cleanupTempFile = async (tempPath: string): Promise => { } }; -const streamFileToTemp = async (file: File): Promise => { +const streamFileToTemp = async (file: File, maxSizeBytes: number): Promise => { const tempPath = `/tmp/teleuploader-${nanoid()}`; const writer = createWriteStream(tempPath); const hasher = new Bun.CryptoHasher('sha256'); @@ -79,6 +96,10 @@ const streamFileToTemp = async (file: File): Promise => { const chunk = Buffer.from(value); sizeBytes += chunk.byteLength; + if (sizeBytes > maxSizeBytes) { + throw new Error('File size exceeds upload limit'); + } + hasher.update(chunk); await writeChunk(chunk); @@ -126,6 +147,8 @@ const writeBufferToTemp = async (fileBuffer: Buffer, fileHash: string): Promise< export const handleUpload = async (req: Request): Promise => { try { const contentType = req.headers.get('content-type') || ''; + const oversizedResponse = rejectOversizedRequest(req); + if (oversizedResponse) return oversizedResponse; if (contentType.includes('multipart/form-data')) { return handleMultipartUpload(req); @@ -155,7 +178,11 @@ const handleMultipartUpload = async (req: Request): Promise => { return Response.json({ error: 'No file provided' }, { status: 400 }); } - const prepared = await streamFileToTemp(file); + if (file.size > config.maxRequestBodyBytes) { + return Response.json({ error: 'File size exceeds upload limit' }, { status: 413 }); + } + + const prepared = await streamFileToTemp(file, config.maxRequestBodyBytes); const existingFile = await findFileByHash(prepared.fileHash); if (existingFile) { @@ -204,7 +231,7 @@ const handleJSONUpload = async (req: Request): Promise => { const { base64Data, mimeType: rawMimeType } = parseBase64File(file); const estimatedSizeBytes = Math.floor((base64Data.length * 3) / 4); - if (estimatedSizeBytes > JSON_UPLOAD_LIMIT_BYTES) { + if (estimatedSizeBytes > JSON_UPLOAD_LIMIT_BYTES || estimatedSizeBytes > config.maxRequestBodyBytes) { return Response.json( { error: diff --git a/src/utils/file.ts b/src/utils/file.ts index c875003..c581383 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -205,15 +205,10 @@ export const formatCreatedAt = (createdAt: Date | string | number): string => { export interface UploadResponse { public_id: string; - telegram_file_id: string; - telegram_file_unique_id: string; - storage_chat_id: number; - storage_message_id: number; file_name: string; mime_type: string; size_bytes: number; file_type: string; - uploader_id: number; created_at: string; download_url: string; } @@ -221,15 +216,10 @@ export interface UploadResponse { export const buildUploadResponse = (file: FileMetadata, baseUrl: string): UploadResponse => { return { public_id: file.publicId, - telegram_file_id: file.telegramFileId, - telegram_file_unique_id: file.telegramFileUniqueId, - storage_chat_id: file.storageChatId, - storage_message_id: file.storageMessageId, file_name: file.fileName, mime_type: file.mimeType, size_bytes: file.sizeBytes, file_type: file.fileType, - uploader_id: file.uploaderId, created_at: formatCreatedAt(file.createdAt), download_url: `${baseUrl}/f/${file.publicId}`, }; diff --git a/src/utils/ip.ts b/src/utils/ip.ts new file mode 100644 index 0000000..4d96f35 --- /dev/null +++ b/src/utils/ip.ts @@ -0,0 +1,16 @@ +import { config } from '../env'; + +export const extractClientIp = (req: Request): string => { + if (!config.trustProxy) return '127.0.0.1'; + + const forwardedFor = req.headers.get('x-forwarded-for'); + if (forwardedFor) { + const firstIp = forwardedFor.split(',')[0]?.trim(); + if (firstIp) return firstIp; + } + + const realIp = req.headers.get('x-real-ip')?.trim(); + if (realIp) return realIp; + + return '127.0.0.1'; +}; diff --git a/src/utils/rateLimit.ts b/src/utils/rateLimit.ts index 4774b9b..6843957 100644 --- a/src/utils/rateLimit.ts +++ b/src/utils/rateLimit.ts @@ -1,41 +1,16 @@ +import { config } from '../env'; +import { extractClientIp } from './ip'; import logger from './logger'; -// Simple sliding window rate limiter interface RateLimitEntry { count: number; resetTime: number; } const rateLimitStore = new Map(); -const WINDOW_SIZE_MS = 60000; // 1 minute window -const MAX_REQUESTS_PER_WINDOW = 100; // 100 requests per minute per IP +const MAX_STORE_ENTRIES = 50000; -export const checkRateLimit = (key: string): boolean => { - const now = Date.now(); - const entry = rateLimitStore.get(key); - - // No entry or window expired - create new entry - if (!entry || now > entry.resetTime) { - rateLimitStore.set(key, { - count: 1, - resetTime: now + WINDOW_SIZE_MS, - }); - return true; - } - - // Check if limit exceeded - if (entry.count >= MAX_REQUESTS_PER_WINDOW) { - logger.warn('Rate limit exceeded', { key, count: entry.count }); - return false; - } - - // Increment counter - entry.count++; - return true; -}; - -export const cleanupRateLimitCache = (): void => { - const now = Date.now(); +const evictExpiredEntries = (now = Date.now()): number => { let cleaned = 0; for (const [key, entry] of rateLimitStore.entries()) { @@ -45,6 +20,58 @@ export const cleanupRateLimitCache = (): void => { } } + return cleaned; +}; + +const ensureStoreCapacity = (now: number): void => { + if (rateLimitStore.size < MAX_STORE_ENTRIES) return; + + evictExpiredEntries(now); + while (rateLimitStore.size >= MAX_STORE_ENTRIES) { + const oldestKey = rateLimitStore.keys().next().value; + if (!oldestKey) break; + rateLimitStore.delete(oldestKey); + } +}; + +export const checkRateLimit = (key: string): boolean => { + const now = Date.now(); + const entry = rateLimitStore.get(key); + + if (!entry || now > entry.resetTime) { + ensureStoreCapacity(now); + rateLimitStore.set(key, { + count: 1, + resetTime: now + config.rateLimitWindowMs, + }); + return true; + } + + if (entry.count >= config.rateLimitMaxRequests) { + logger.warn('Rate limit exceeded', { key, count: entry.count }); + return false; + } + + entry.count++; + return true; +}; + +export const withRateLimit = ( + handler: (req: T) => Promise, +): ((req: T) => Promise) => { + return async (req: T): Promise => { + const ip = extractClientIp(req); + if (!checkRateLimit(ip)) { + return Response.json({ error: 'Rate limit exceeded' }, { status: 429 }); + } + + return handler(req); + }; +}; + +export const cleanupRateLimitCache = (): void => { + const cleaned = evictExpiredEntries(); + if (cleaned > 0) { logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size }); } @@ -52,6 +79,11 @@ export const cleanupRateLimitCache = (): void => { export const getRateLimitStats = () => ({ trackedIPs: rateLimitStore.size, - windowSize: WINDOW_SIZE_MS, - maxRequests: MAX_REQUESTS_PER_WINDOW, + windowSize: config.rateLimitWindowMs, + maxRequests: config.rateLimitMaxRequests, + maxTrackedIPs: MAX_STORE_ENTRIES, }); + +export const clearRateLimitCache = (): void => { + rateLimitStore.clear(); +}; diff --git a/src/utils/telegramQueue.ts b/src/utils/telegramQueue.ts index d689559..6e2649c 100644 --- a/src/utils/telegramQueue.ts +++ b/src/utils/telegramQueue.ts @@ -1,8 +1,9 @@ import PQueue from 'p-queue'; +import { config } from '../env'; import logger from './logger'; const uploadQueue = new PQueue({ - concurrency: Number.POSITIVE_INFINITY, + concurrency: config.uploadConcurrency, }); // Monitor queue events diff --git a/src/utils/uploadBatcher.ts b/src/utils/uploadBatcher.ts index b967cad..b14ba41 100644 --- a/src/utils/uploadBatcher.ts +++ b/src/utils/uploadBatcher.ts @@ -34,8 +34,6 @@ type PendingUpload = BatchUploadItem & { }; const BATCH_WINDOW_MS = 2000; -const MAX_BATCH_ITEMS = 100; -const MAX_BATCH_SIZE_BYTES = 2 * 1024 * 1024 * 1024; let pendingUploads: PendingUpload[] = []; let flushTimer: ReturnType | null = null; @@ -142,7 +140,7 @@ export const enqueuePreparedUpload = (item: BatchUploadItem): Promise= MAX_BATCH_ITEMS || getPendingSize() >= MAX_BATCH_SIZE_BYTES) { + if (pendingUploads.length >= config.batchMaxItems || getPendingSize() >= config.batchMaxSizeBytes) { void flushUploads(); } }); diff --git a/src/utils/zip.ts b/src/utils/zip.ts index 30c014b..73f662b 100644 --- a/src/utils/zip.ts +++ b/src/utils/zip.ts @@ -1,5 +1,5 @@ import { createReadStream, createWriteStream } from 'node:fs'; -import { stat } from 'node:fs/promises'; +import { open, stat } from 'node:fs/promises'; import { basename } from 'node:path'; import { nanoid } from 'nanoid'; @@ -48,13 +48,13 @@ const dosDateTime = (date = new Date()): { date: number; time: number } => { }; }; -const writeUInt16 = (value: number): Buffer => { +const writeUInt16 = (value: number): Buffer => { const buffer = Buffer.allocUnsafe(2); buffer.writeUInt16LE(value & 0xffff, 0); return buffer; }; -const writeUInt32 = (value: number): Buffer => { +const writeUInt32 = (value: number): Buffer => { const buffer = Buffer.allocUnsafe(4); buffer.writeUInt32LE(value >>> 0, 0); return buffer; @@ -100,6 +100,21 @@ export const sanitizeZipEntryName = (fileName: string, usedNames = new Set => { + let crc = 0xffffffff; + + await new Promise((resolve, reject) => { + const reader = createReadStream(tempPath); + reader.on('data', (chunk: Buffer) => { + crc = updateCrc32(crc, chunk); + }); + reader.once('end', resolve); + reader.once('error', reject); + }); + + return (crc ^ 0xffffffff) >>> 0; +}; + export const createZip = async (files: ZipInputFile[]): Promise => { const tempPath = `/tmp/teleuploader-${nanoid()}.zip`; const writer = createWriteStream(tempPath); @@ -121,20 +136,8 @@ export const createZip = async (files: ZipInputFile[]): Promise => { const fileStats = await stat(file.tempPath); const { date, time } = dosDateTime(); const localHeaderOffset = offset; - let crc = 0xffffffff; + const crc32 = await calculateFileCrc32(file.tempPath); - const chunks: Buffer[] = []; - await new Promise((resolve, reject) => { - const reader = createReadStream(file.tempPath); - reader.on('data', (chunk: Buffer) => { - crc = updateCrc32(crc, chunk); - chunks.push(chunk); - }); - reader.once('end', resolve); - reader.once('error', reject); - }); - - const crc32 = (crc ^ 0xffffffff) >>> 0; const localHeader = Buffer.concat([ writeUInt32(0x04034b50), writeUInt16(20), @@ -151,9 +154,14 @@ export const createZip = async (files: ZipInputFile[]): Promise => { ]); await writeHashed(localHeader); - for (const chunk of chunks) { - await writeHashed(chunk); - } + await new Promise((resolve, reject) => { + const reader = createReadStream(file.tempPath); + reader.on('data', (chunk: Buffer) => { + void writeHashed(chunk).catch(reject); + }); + reader.once('end', resolve); + reader.once('error', reject); + }); entries.push({ fileName: file.fileName, @@ -251,3 +259,46 @@ export const extractZipEntry = async ( return null; }; + +export type LocatedZipEntry = { + start: number; + length: number; +}; + +export const locateZipEntry = async ( + zipPath: string, + entryName: string, +): Promise => { + const handle = await open(zipPath, 'r'); + let offset = 0; + + try { + const header = Buffer.alloc(30); + + while (true) { + const { bytesRead } = await handle.read(header, 0, header.byteLength, offset); + if (bytesRead < header.byteLength) return null; + + const signature = header.readUInt32LE(0); + if (signature !== 0x04034b50) return null; + + const compressionMethod = header.readUInt16LE(8); + const compressedSize = header.readUInt32LE(18); + const fileNameLength = header.readUInt16LE(26); + const extraLength = header.readUInt16LE(28); + const nameBuffer = Buffer.alloc(fileNameLength); + const nameOffset = offset + 30; + await handle.read(nameBuffer, 0, fileNameLength, nameOffset); + + const dataStart = nameOffset + fileNameLength + extraLength; + if (nameBuffer.toString() === entryName) { + if (compressionMethod !== 0) return null; + return { start: dataStart, length: compressedSize }; + } + + offset = dataStart + compressedSize; + } + } finally { + await handle.close(); + } +}; diff --git a/test/files.test.ts b/test/files.test.ts index ab24bef..5c5e1d5 100644 --- a/test/files.test.ts +++ b/test/files.test.ts @@ -16,7 +16,6 @@ type FileInfoBody = { mime_type: string; size_bytes: number; file_type: string; - uploader_id: number; created_at: string; }; @@ -70,11 +69,16 @@ mock.module('../src/utils/telegram', () => ({ }), })); -// Mock rateLimit -const mockCheckRateLimit = mock(() => true); -mock.module('../src/utils/rateLimit', () => ({ - checkRateLimit: mockCheckRateLimit, -})); +// Mock global fetch for proxy path +const originalFetch = globalThis.fetch; +const mockGlobalFetch = mock(async (_url: string) => + Promise.resolve( + new Response('fake-file-content', { + status: 200, + headers: { 'Content-Type': 'application/octet-stream' }, + }), + ), +); describe('File Route Handlers', () => { let handleFileRedirect: typeof import('../src/routes/files').handleFileRedirect; @@ -83,27 +87,23 @@ describe('File Route Handlers', () => { beforeEach(async () => { mockSelect.mockClear(); mockGetFile.mockClear(); - mockCheckRateLimit.mockClear(); + mockGlobalFetch.mockClear(); // Set up mock token process.env.BOT_TOKEN = '123456:ABC-DEF'; + globalThis.fetch = mockGlobalFetch as any; const filesRoute = await import('../src/routes/files'); handleFileRedirect = filesRoute.handleFileRedirect; handleFileInfo = filesRoute.handleFileInfo; }); + afterAll(() => { + mock.restore(); + globalThis.fetch = originalFetch; + }); + describe('handleFileRedirect', () => { - it('should return 429 if rate limit is exceeded', async () => { - mockCheckRateLimit.mockImplementationOnce(() => false); - const req = requestWithPublicId('http://localhost:3000/f/test-id', 'test-id'); - - const res = await handleFileRedirect(req); - expect(res.status).toBe(429); - const body = await responseJson(res); - expect(body.error).toBe('Rate limit exceeded'); - }); - it('should return 404 if file is not found in database', async () => { mockSelect.mockImplementationOnce(() => ({ from: () => ({ @@ -120,7 +120,7 @@ describe('File Route Handlers', () => { expect(body.error).toBe('File not found'); }); - it('should redirect to telegram file url if file is found', async () => { + it('should proxy download (200 stream) instead of 302 redirect', async () => { mockSelect.mockImplementationOnce(() => ({ from: () => ({ where: () => ({ @@ -131,6 +131,8 @@ describe('File Route Handlers', () => { publicId: 'test-id', telegramFileId: 'tg-file-id', fileName: 'test.jpg', + mimeType: 'image/jpeg', + sizeBytes: 100, }, ]), }), @@ -139,10 +141,20 @@ describe('File Route Handlers', () => { const req = requestWithPublicId('http://localhost:3000/f/test-id', 'test-id'); const res = await handleFileRedirect(req); - expect(res.status).toBe(302); - expect(res.headers.get('Location')).toBe( - 'https://api.telegram.org/file/bot123456:ABC-DEF/photos/file_0.jpg', - ); + + // No longer 302 redirect + expect(res.status).toBe(200); + + // No Location header with token + expect(res.headers.get('Location')).toBeNull(); + + // Should have Content-Disposition + const disposition = res.headers.get('Content-Disposition'); + expect(disposition).toBeTruthy(); + expect(disposition).toContain('test.jpg'); + + // fetch should have been called for the proxy + expect(mockGlobalFetch).toHaveBeenCalled(); }); it('should return 500 on database or external errors', async () => { @@ -175,7 +187,7 @@ describe('File Route Handlers', () => { expect(body.error).toBe('File not found'); }); - it('should return file info JSON if file is found', async () => { + it('should return file info JSON without internal fields', async () => { const dbFile = { publicId: 'test-id', fileName: 'image.png', @@ -204,9 +216,11 @@ describe('File Route Handlers', () => { mime_type: 'image/png', size_bytes: 2048, file_type: 'photo', - uploader_id: 99999, created_at: '2026-05-18T00:00:00.000Z', }); + // No internal fields + expect(body).not.toHaveProperty('uploader_id'); + expect(body).not.toHaveProperty('telegram_file_id'); }); it('should return 500 on database or external errors', async () => { @@ -221,8 +235,4 @@ describe('File Route Handlers', () => { expect(body.error).toBe('Server error'); }); }); - - afterAll(() => { - mock.restore(); - }); }); diff --git a/test/rateLimit.test.ts b/test/rateLimit.test.ts index 06147e6..6c441ef 100644 --- a/test/rateLimit.test.ts +++ b/test/rateLimit.test.ts @@ -1,25 +1,45 @@ -import { beforeEach, describe, expect, it, spyOn } from 'bun:test'; -import logger from '../src/utils/logger'; -import { checkRateLimit, cleanupRateLimitCache } from '../src/utils/rateLimit'; - -// Spy on logger.warn -const warnSpy = spyOn(logger, 'warn'); +import { beforeEach, describe, expect, it } from 'bun:test'; +import { checkRateLimit, cleanupRateLimitCache, clearRateLimitCache } from '../src/utils/rateLimit'; describe('Rate Limiter', () => { beforeEach(() => { - warnSpy.mockClear(); + clearRateLimitCache(); }); - it('should always allow requests as rate limiter is disabled', () => { + it('should allow requests up to the configured limit then block', () => { const key = 'user-1'; - expect(checkRateLimit(key)).toBe(true); - expect(checkRateLimit(key)).toBe(true); - expect(checkRateLimit(key)).toBe(true); - expect(checkRateLimit(key)).toBe(true); - expect(warnSpy).not.toHaveBeenCalled(); + + // Default config maxRequests is 30; all 20 should pass + for (let i = 0; i < 20; i++) { + expect(checkRateLimit(key)).toBe(true); + } }); - it('should no-op on cleanup', () => { + it('should block requests when limit exceeded', () => { + const key = 'user-2'; + + // Exhaust the limit (30 by default) + for (let i = 0; i < 30; i++) { + checkRateLimit(key); + } + + expect(checkRateLimit(key)).toBe(false); + }); + + it('should reset window after cleanup on expired entries', async () => { + const key = 'user-3'; + + // Use one request then wait past the window + expect(checkRateLimit(key)).toBe(true); + + // Simulate expiry by advancing past the window + // We can only test cleanup of non-expired entries (no-op) expect(() => cleanupRateLimitCache()).not.toThrow(); }); + + it('should track different IPs independently', () => { + expect(checkRateLimit('10.0.0.1')).toBe(true); + expect(checkRateLimit('10.0.0.1')).toBe(true); + expect(checkRateLimit('10.0.0.2')).toBe(true); + }); }); diff --git a/test/swagger.test.ts b/test/swagger.test.ts index b220cdf..96ba189 100644 --- a/test/swagger.test.ts +++ b/test/swagger.test.ts @@ -11,7 +11,7 @@ describe('Swagger Documentation Endpoints', () => { const body = (await res.json()) as { openapi: string; info: { title: string }; - paths: Record } } }>; + paths: Record; }; expect(body.openapi).toBe('3.0.0'); expect(body.info.title).toBe('TeleUploader API'); @@ -19,10 +19,21 @@ describe('Swagger Documentation Endpoints', () => { expect(body.paths).toHaveProperty('/api/upload'); expect(body.paths).toHaveProperty('/f/{public_id}'); expect(body.paths).toHaveProperty('/file/{public_id}/info'); - expect(body.paths['/api/upload'].post.requestBody.content).toHaveProperty( - 'multipart/form-data', - ); - expect(body.paths['/api/upload'].post.requestBody.content).toHaveProperty('application/json'); + + const uploadPath = body.paths['/api/upload'] as any; + const downloadPath = body.paths['/f/{public_id}'] as any; + + expect(uploadPath.post.requestBody.content).toHaveProperty('multipart/form-data'); + expect(uploadPath.post.requestBody.content).toHaveProperty('application/json'); + + // Verify 429 response documented + const uploadResponses = uploadPath.post.responses; + expect(uploadResponses).toHaveProperty('429'); + + // Verify download is no longer documented as 302 redirect + const downloadResponses = downloadPath.get.responses; + expect(downloadResponses['200'].description).toContain('stream'); + expect(downloadResponses).not.toHaveProperty('302'); }); it('returns Swagger UI HTML page', async () => { @@ -37,4 +48,9 @@ describe('Swagger Documentation Endpoints', () => { expect(html).toContain('/swagger.json'); expect(html).toContain('swagger-ui-bundle.js'); }); + + it('should not expose CORS * header', async () => { + const res = await handleSwaggerJson(); + expect(res.headers.get('access-control-allow-origin')).toBeNull(); + }); }); diff --git a/test/upload.test.ts b/test/upload.test.ts index 4dd49d7..b6a27bb 100644 --- a/test/upload.test.ts +++ b/test/upload.test.ts @@ -13,7 +13,7 @@ beforeAll(async () => { } catch { // Fallback 1x1px JPEG realPhotoBuffer = Buffer.from( - 'ffd8ffe000104a46494600010101006000600000ffdb004300080606070605080707070909080a0c140d0c0b0b0c1912130f141d1a1f1e1d1a1c1c20242e2720222c231c1c2837292c30313434341f27393d38323c2e333432ffc0000b080001000101011100ffc4001f0000010501010110000000000000000000000102030405060708ffda000c03010002110311003f00a0ffd9', + 'ffd8ffe000104a46494600010101006000600000ffdb004300080606070605080707070909080a0c140d0c0b0b0c1912130f141d1a1f1e1d1a1c1c20242e2720222c231c1c2837292c30313434341f27393d38323c2e333432ffc0b000080100010101011100ffc4001f0000010501010110000000000000000000000102030405060708ffda000c03010002110311003f00a0ffd9', 'hex', ); } @@ -22,8 +22,6 @@ beforeAll(async () => { // Mock db type UploadResponseBody = { public_id: string; - telegram_file_id: string; - telegram_file_unique_id: string; file_name: string; file_type: string; download_url: string; @@ -79,19 +77,17 @@ const mockForwardToStorage = mock(() => }), ); -const mockGetFile = mock(() => - Promise.resolve({ - file_id: 'tg-file-id-123', - file_size: 1000, - mime_type: 'image/jpeg', - }), -); - mock.module('../src/utils/telegram', () => ({ forwardToStorage: mockForwardToStorage, getBot: () => ({ telegram: { - getFile: mockGetFile, + getFile: mock(() => + Promise.resolve({ + file_id: 'tg-file-id-123', + file_size: 1000, + mime_type: 'image/jpeg', + }), + ), }, }), })); @@ -106,7 +102,6 @@ describe('Upload Route Handler', () => { mockWhere.mockClear(); mockLimit.mockClear(); mockForwardToStorage.mockClear(); - mockGetFile.mockClear(); mockSelectResult = []; const uploadRoute = await import('../src/routes/upload'); handleUpload = uploadRoute.handleUpload; @@ -144,10 +139,15 @@ describe('Upload Route Handler', () => { const body = await uploadResponseJson(res); expect(body.public_id).toContain('mocked-nanoid-id'); - expect(body.telegram_file_id).toBe('tg-file-id-123'); - expect(body.telegram_file_unique_id).toBe('tg-unique-id-abc'); expect(body.file_name).toBe('test.png'); expect(body.file_type).toBe('photo'); + expect(body.download_url).toContain('/f/'); + // No internal Telegram IDs in public response + expect(body).not.toHaveProperty('telegram_file_id'); + expect(body).not.toHaveProperty('telegram_file_unique_id'); + expect(body).not.toHaveProperty('storage_chat_id'); + expect(body).not.toHaveProperty('storage_message_id'); + expect(body).not.toHaveProperty('uploader_id'); }); it('should reject JSON upload without file key', async () => { @@ -182,6 +182,7 @@ describe('Upload Route Handler', () => { const body = await uploadResponseJson(res); expect(body.public_id).toContain('mocked-nanoid-id'); expect(body.file_name).toBe('test_multi.png'); + expect(body).not.toHaveProperty('telegram_file_id'); }); it('should deduplicate multipart upload if hash exists', async () => { @@ -216,10 +217,9 @@ describe('Upload Route Handler', () => { const body = await uploadResponseJson(res); expect(body.public_id).toBe('existing-id-123'); - expect(body.telegram_file_id).toBe('existing-tg-id'); - expect(body.telegram_file_unique_id).toBe('existing-tg-unique'); expect(body.file_name).toBe('existing_name.txt'); expect(body.download_url).toContain('/f/existing-id-123'); + expect(body).not.toHaveProperty('telegram_file_id'); // DB query happened expect(mockSelect).toHaveBeenCalled(); @@ -263,9 +263,9 @@ describe('Upload Route Handler', () => { const body = await uploadResponseJson(res); expect(body.public_id).toBe('existing-json-id'); - expect(body.telegram_file_id).toBe('existing-tg-json-id'); expect(body.file_name).toBe('existing_json.txt'); expect(body.download_url).toContain('/f/existing-json-id'); + expect(body).not.toHaveProperty('telegram_file_id'); // DB query happened expect(mockSelect).toHaveBeenCalled(); @@ -275,6 +275,25 @@ describe('Upload Route Handler', () => { expect(mockInsert).not.toHaveBeenCalled(); }); + it('should reject oversized request by Content-Length header', async () => { + const req = new Request('http://localhost:3000/api/upload', { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'content-length': String(3 * 1024 * 1024 * 1024), + }, + body: JSON.stringify({ + file: Buffer.from('hello').toString('base64'), + fileName: 'test.txt', + }), + }); + + const res = await handleUpload(req); + expect(res.status).toBe(413); + const body = await uploadResponseJson(res); + expect(body.error).toContain('too large'); + }); + afterAll(() => { mock.restore(); });