MythEclipse/TeleUploader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: enhance configuration and rate limiting

Browse Source 
- Added new configuration options: trustProxy, uploadConcurrency, batchMaxItems, batchMaxSizeBytes, and maxRequestBodyBytes to AppConfig.
- Implemented utility functions for parsing environment variables and masking sensitive data.
- Updated rate limiting logic to use configurable window size and maximum requests per window.
- Introduced a middleware for rate limiting on specific routes.
- Refactored file handling routes to support streaming downloads instead of redirects.
- Improved error handling and response formatting in file routes.
- Added support for oversized request rejection based on Content-Length header.
- Updated Swagger documentation to reflect changes in API behavior and responses.
- Enhanced tests to cover new features and ensure proper functionality.
This commit is contained in:
MythEclipse
2026-05-29 03:33:39 +07:00
parent be813b1c0e
commit 5425f6d33d
16 changed files with 466 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.yml
+34
View File
@@ -7,12 +7,41 @@ services:
restart: always
environment:
- BOT_TOKEN=${BOT_TOKEN}
- ADDITIONAL_BOT_TOKENS=${ADDITIONAL_BOT_TOKENS:-}
- STORAGE_CHANNEL_ID=${STORAGE_CHANNEL_ID}
- BASE_URL=${BASE_URL}
- DATABASE_URL=${DATABASE_URL}
- PORT=3000
- NODE_ENV=production
- LOG_LEVEL=info
- TRUST_PROXY=true
- UPLOAD_CONCURRENCY=${UPLOAD_CONCURRENCY:-8}
- BATCH_MAX_ITEMS=${BATCH_MAX_ITEMS:-20}
- BATCH_MAX_SIZE_BYTES=${BATCH_MAX_SIZE_BYTES:-524288000}
- MAX_REQUEST_BODY_BYTES=${MAX_REQUEST_BODY_BYTES:-2147483648}
- RATE_LIMIT_WINDOW_MS=${RATE_LIMIT_WINDOW_MS:-60000}
- RATE_LIMIT_MAX_REQUESTS=${RATE_LIMIT_MAX_REQUESTS:-30}
security_opt:
- no-new-privileges:true
read_only: true
tmpfs:
- /tmp:size=4g,mode=1777
deploy:
resources:
limits:
cpus: '2.0'
memory: 2G
reservations:
cpus: '0.5'
memory: 512M
healthcheck:
test:
- CMD-SHELL
- curl -sf http://localhost:3000/health || exit 1
interval: 30s
timeout: 10s
retries: 3
start_period: 15s
networks:
- app-shared-net
labels:
@@ -22,6 +51,11 @@ services:
- "traefik.http.routers.teleuploader.tls=true"
- "traefik.http.routers.teleuploader.tls.certresolver=letsencrypt"
- "traefik.http.services.teleuploader.loadbalancer.server.port=3000"
- "traefik.http.middlewares.teleuploader-rl.ratelimit.average=300"
- "traefik.http.middlewares.teleuploader-rl.ratelimit.burst=100"
- "traefik.http.middlewares.teleuploader-rl.ratelimit.period=1m"
- "traefik.http.middlewares.teleuploader-buf.buffering.maxRequestBodyBytes=2147483648"
- "traefik.http.routers.teleuploader.middlewares=teleuploader-rl@docker,teleuploader-buf@docker"
networks:
app-shared-net:
src/env.ts
+38 -10
View File
@@ -11,6 +11,11 @@ interface AppConfig {
logLevel: string;
rateLimitWindowMs: number;
rateLimitMaxRequests: number;
trustProxy: boolean;
uploadConcurrency: number;
batchMaxItems: number;
batchMaxSizeBytes: number;
maxRequestBodyBytes: number;
}
const requiredEnv = {
@@ -30,25 +35,48 @@ if (missing.length > 0) {
throw new Error(`Missing environment variables: ${missing.join(', ')}`);
}
export const config: AppConfig = {
botToken: process.env.BOT_TOKEN!,
additionalBotTokens:
process.env.NODE_ENV === 'test'
? []
: (process.env.ADDITIONAL_BOT_TOKENS || '')
const parseNumber = (value: string | undefined, fallback: number): number => {
const parsed = Number.parseInt(value || '', 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
};
const parseTokens = (value: string | undefined): string[] =>
(value || '')
.split(',')
.map((t) => t.trim())
.filter((t) => t !== ''),
.filter((t) => t !== '');
const maskSecret = (value: string): string => {
if (!value) return '';
if (value.length <= 10) return '***';
return `${value.slice(0, 6)}...${value.slice(-4)}`;
};
const maskDatabaseUrl = (value: string): string => value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
export const config: AppConfig = {
botToken: process.env.BOT_TOKEN!,
additionalBotTokens: process.env.NODE_ENV === 'test' ? [] : parseTokens(process.env.ADDITIONAL_BOT_TOKENS),
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10),
baseUrl: process.env.BASE_URL!,
databaseUrl: process.env.DATABASE_URL!,
port: parseInt(process.env.PORT!, 10) || 3000,
nodeEnv: process.env.NODE_ENV || 'development',
logLevel: process.env.LOG_LEVEL || 'info',
rateLimitWindowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS!, 10) || 60000,
rateLimitMaxRequests: parseInt(process.env.RATE_LIMIT_MAX_REQUESTS!, 10) || 30,
rateLimitWindowMs: parseNumber(process.env.RATE_LIMIT_WINDOW_MS, 60000),
rateLimitMaxRequests: parseNumber(process.env.RATE_LIMIT_MAX_REQUESTS, 150),
trustProxy: process.env.TRUST_PROXY === 'true',
uploadConcurrency: parseNumber(process.env.UPLOAD_CONCURRENCY, 8),
batchMaxItems: parseNumber(process.env.BATCH_MAX_ITEMS, 20),
batchMaxSizeBytes: parseNumber(process.env.BATCH_MAX_SIZE_BYTES, 500 * 1024 * 1024),
maxRequestBodyBytes: parseNumber(process.env.MAX_REQUEST_BODY_BYTES, 2 * 1024 * 1024 * 1024),
};
logger.info('Environment variables loaded', {
config: { ...config, botToken: `${config.botToken?.substring(0, 10)}...` },
config: {
...config,
botToken: maskSecret(config.botToken),
additionalBotTokens: config.additionalBotTokens.map(maskSecret),
databaseUrl: maskDatabaseUrl(config.databaseUrl),
},
});
src/index.ts
+4 -4
View File
@@ -6,19 +6,19 @@ import { handleHealth } from './routes/health';
import { handleSwaggerHtml, handleSwaggerJson } from './routes/swagger';
import { handleUpload } from './routes/upload';
import logger from './utils/logger';
import { cleanupRateLimitCache } from './utils/rateLimit';
import { cleanupRateLimitCache, withRateLimit } from './utils/rateLimit';
const server = serve({
port: config.port,
routes: {
'/api/upload': {
POST: handleUpload,
POST: withRateLimit(handleUpload),
},
'/f/:public_id': {
GET: handleFileRedirect,
GET: withRateLimit(handleFileRedirect),
},
'/file/:public_id/info': {
GET: handleFileInfo,
GET: withRateLimit(handleFileInfo),
},
'/health': {
GET: handleHealth,
src/routes/files.ts
+62 -27
View File
@@ -1,10 +1,12 @@
import { createReadStream } from 'node:fs';
import { unlink } from 'node:fs/promises';
import { nanoid } from 'nanoid';
import { findFileByPublicId } from '../db/files';
import { fileInfoCache } from '../utils/cache';
import { formatCreatedAt, getErrorMessage } from '../utils/file';
import logger from '../utils/logger';
import { checkRateLimit } from '../utils/rateLimit';
import { getBot } from '../utils/telegram';
import { extractZipEntry } from '../utils/zip';
import { locateZipEntry } from '../utils/zip';
type RequestWithParams = Request & {
params?: {
@@ -36,20 +38,31 @@ const getTelegramFileInfo = async (telegramFileId: string, public_id: string) =>
const buildTelegramFileUrl = (filePath: string): string =>
`https://api.telegram.org/file/bot${process.env.BOT_TOKEN}/${filePath}`;
const cleanupTempFile = async (tempPath: string): Promise<void> => {
try {
await unlink(tempPath);
} catch (err) {
logger.warn('Failed to cleanup temp file', { tempPath, error: getErrorMessage(err) });
}
};
const sanitizeFilenameHeader = (fileName: string): string =>
fileName.replace(/[\\"]/g, '').replace(/[\n\r]/g, '');
const fail = (status: number, error: string): Response =>
Response.json({ error }, { status });
export const handleFileRedirect = async (req: RequestWithParams): Promise<Response> => {
const public_id = req.params?.public_id;
try {
const ip = req.headers.get('x-forwarded-for') || '127.0.0.1';
if (!public_id || !checkRateLimit(ip)) {
return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
if (!public_id) {
return fail(400, 'Missing file id');
}
const file = await findFileByPublicId(public_id);
if (!file) {
logger.warn('File not found', { public_id });
return Response.json({ error: 'File not found' }, { status: 404 });
return fail(404, 'File not found');
}
const archiveEntryName = file.archiveEntryName;
@@ -60,37 +73,60 @@ export const handleFileRedirect = async (req: RequestWithParams): Promise<Respon
if (!archiveResponse.ok) {
logger.error('Archive download failed', { public_id, status: archiveResponse.status });
return Response.json({ error: 'Server error' }, { status: 500 });
return fail(500, 'Server error');
}
const archiveBuffer = Buffer.from(await archiveResponse.arrayBuffer());
const extractedFile = await extractZipEntry(archiveBuffer, archiveEntryName);
if (!extractedFile) {
const tempZipPath = `/tmp/teleuploader-dl-${nanoid()}.zip`;
await Bun.write(tempZipPath, archiveResponse);
const loc = await locateZipEntry(tempZipPath, archiveEntryName);
if (!loc) {
await cleanupTempFile(tempZipPath);
logger.error('Archive entry not found', { public_id, archiveEntryName });
return Response.json({ error: 'File not found' }, { status: 404 });
return fail(404, 'File not found');
}
return new Response(extractedFile, {
const fileStream = createReadStream(tempZipPath, {
start: loc.start,
end: loc.start + loc.length - 1,
});
fileStream.on('close', () => {
void cleanupTempFile(tempZipPath);
});
fileStream.on('error', () => {
void cleanupTempFile(tempZipPath);
});
return new Response(fileStream as any, {
status: 200,
headers: {
'Content-Type': file.mimeType,
'Content-Disposition': `attachment; filename="${file.fileName.replace(/"/g, '')}"`,
'Content-Length': String(extractedFile.byteLength),
'Content-Type': file.mimeType || 'application/octet-stream',
'Content-Disposition': `attachment; filename="${sanitizeFilenameHeader(file.fileName)}"`,
'Content-Length': String(loc.length),
},
});
}
const fileInfo = await getTelegramFileInfo(file.telegramFileId, public_id);
const redirectUrl = buildTelegramFileUrl(fileInfo.file_path);
return new Response(null, {
status: 302,
const tgResponse = await fetch(buildTelegramFileUrl(fileInfo.file_path));
if (!tgResponse.ok) {
logger.error('File download failed', { public_id, status: tgResponse.status });
return fail(502, 'Server error');
}
return new Response(tgResponse.body, {
status: 200,
headers: {
Location: redirectUrl,
'Content-Type': file.mimeType || 'application/octet-stream',
'Content-Disposition': `attachment; filename="${sanitizeFilenameHeader(file.fileName)}"`,
'Content-Length': String(file.sizeBytes),
},
});
} catch (error: unknown) {
logger.error('File redirect error', { public_id, error: getErrorMessage(error) });
return Response.json({ error: 'Server error' }, { status: 500 });
return fail(500, 'Server error');
}
};
@@ -98,15 +134,15 @@ export const handleFileInfo = async (req: RequestWithParams): Promise<Response>
const public_id = req.params?.public_id;
try {
if (!public_id) {
return Response.json({ error: 'Missing file id' }, { status: 400 });
return fail(400, 'Missing file id');
}
const file = await findFileByPublicId(public_id);
if (!file) {
logger.warn('File not found', { public_id });
return Response.json({ error: 'File not found' }, { status: 404 });
return fail(404, 'File not found');
}
return Response.json(
{
public_id: file.publicId,
@@ -114,13 +150,12 @@ export const handleFileInfo = async (req: RequestWithParams): Promise<Response>
mime_type: file.mimeType,
size_bytes: file.sizeBytes,
file_type: file.fileType,
uploader_id: file.uploaderId,
created_at: formatCreatedAt(file.createdAt),
},
{ status: 200 },
);
} catch (error: unknown) {
logger.error('File info error', { public_id, error: getErrorMessage(error) });
return Response.json({ error: 'Server error' }, { status: 500 });
return fail(500, 'Server error');
}
};
src/routes/swagger.ts
+15 -27
View File
@@ -25,7 +25,6 @@ const fileInfoProperties = {
mime_type: { type: 'string', example: 'application/pdf' },
size_bytes: { type: 'integer', example: 1048576 },
file_type: { type: 'string', example: 'document' },
uploader_id: { type: 'integer', example: 0 },
created_at: {
type: 'string',
format: 'date-time',
@@ -35,10 +34,6 @@ const fileInfoProperties = {
const uploadProperties = {
...fileInfoProperties,
telegram_file_id: { type: 'string', example: 'BQACAgQAAxkBA...' },
telegram_file_unique_id: { type: 'string', example: 'AgAD8w...' },
storage_chat_id: { type: 'integer', example: -1001234567890 },
storage_message_id: { type: 'integer', example: 42 },
download_url: {
type: 'string',
example: `${config.baseUrl}/f/xYz123`,
@@ -56,7 +51,7 @@ export const handleSwaggerJson = async (): Promise<Response> => {
info: {
title: 'TeleUploader API',
version: '1.0.0',
description: 'Telegram-backed file uploader API with redirect-based downloads.',
description: 'Telegram-backed file uploader API with stream-based downloads.',
},
servers: [
{
@@ -95,7 +90,7 @@ export const handleSwaggerJson = async (): Promise<Response> => {
'/api/upload': {
post: {
summary: 'Upload File',
description: 'Uploads a file to Telegram storage via multipart/form-data or JSON base64.',
description: 'Uploads a file to Telegram storage via multipart/form-data or JSON base64. Rate-limited by IP.',
requestBody: {
required: true,
content: {
@@ -144,6 +139,14 @@ export const handleSwaggerJson = async (): Promise<Response> => {
description: 'Bad request.',
content: jsonContent(errorSchema('No file provided')),
},
'413': {
description: 'Request body too large.',
content: jsonContent(errorSchema('Request body too large')),
},
'429': {
description: 'Rate limit exceeded.',
content: jsonContent(errorSchema('Rate limit exceeded')),
},
'500': {
description: 'Internal server error.',
content: jsonContent(errorSchema('Upload failed')),
@@ -153,21 +156,12 @@ export const handleSwaggerJson = async (): Promise<Response> => {
},
'/f/{public_id}': {
get: {
summary: 'Redirect to Telegram File URL',
description:
'Gets a fresh Telegram download URL and redirects with 302. Rate-limited by IP.',
summary: 'Download File',
description: 'Proxies file from Telegram storage as a streamed download. Rate-limited by IP.',
parameters: [publicIdParameter],
responses: {
'302': {
description: 'Redirect to Telegram CDN URL.',
headers: {
Location: {
schema: {
type: 'string',
example: 'https://api.telegram.org/file/botTOKEN/documents/file_0.pdf',
},
},
},
'200': {
description: 'File binary stream.',
},
'404': {
description: 'File not found.',
@@ -212,12 +206,7 @@ export const handleSwaggerJson = async (): Promise<Response> => {
},
};
return Response.json(spec, {
status: 200,
headers: {
'access-control-allow-origin': '*',
},
});
return Response.json(spec, { status: 200 });
};
export const handleSwaggerHtml = async (): Promise<Response> => {
@@ -256,7 +245,6 @@ export const handleSwaggerHtml = async (): Promise<Response> => {
status: 200,
headers: {
'content-type': 'text/html; charset=utf-8',
'access-control-allow-origin': '*',
'x-content-type-options': 'nosniff',
},
});
src/routes/upload.ts
+30 -3
View File
@@ -39,6 +39,23 @@ const normalizeFileType = (mimeType: string, fileName: string): string => {
const JSON_UPLOAD_LIMIT_BYTES = 50 * 1024 * 1024;
const SIGNATURE_BYTES = 16;
const getContentLength = (req: Request): number | null => {
const value = req.headers.get('content-length');
if (!value) return null;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
};
const rejectOversizedRequest = (req: Request): Response | null => {
const contentLength = getContentLength(req);
if (contentLength !== null && contentLength > config.maxRequestBodyBytes) {
return Response.json({ error: 'Request body too large' }, { status: 413 });
}
return null;
};
const cleanupTempFile = async (tempPath: string): Promise<void> => {
try {
await unlink(tempPath);
@@ -47,7 +64,7 @@ const cleanupTempFile = async (tempPath: string): Promise<void> => {
}
};
const streamFileToTemp = async (file: File): Promise<PreparedUpload> => {
const streamFileToTemp = async (file: File, maxSizeBytes: number): Promise<PreparedUpload> => {
const tempPath = `/tmp/teleuploader-${nanoid()}`;
const writer = createWriteStream(tempPath);
const hasher = new Bun.CryptoHasher('sha256');
@@ -79,6 +96,10 @@ const streamFileToTemp = async (file: File): Promise<PreparedUpload> => {
const chunk = Buffer.from(value);
sizeBytes += chunk.byteLength;
if (sizeBytes > maxSizeBytes) {
throw new Error('File size exceeds upload limit');
}
hasher.update(chunk);
await writeChunk(chunk);
@@ -126,6 +147,8 @@ const writeBufferToTemp = async (fileBuffer: Buffer, fileHash: string): Promise<
export const handleUpload = async (req: Request): Promise<Response> => {
try {
const contentType = req.headers.get('content-type') || '';
const oversizedResponse = rejectOversizedRequest(req);
if (oversizedResponse) return oversizedResponse;
if (contentType.includes('multipart/form-data')) {
return handleMultipartUpload(req);
@@ -155,7 +178,11 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
return Response.json({ error: 'No file provided' }, { status: 400 });
}
const prepared = await streamFileToTemp(file);
if (file.size > config.maxRequestBodyBytes) {
return Response.json({ error: 'File size exceeds upload limit' }, { status: 413 });
}
const prepared = await streamFileToTemp(file, config.maxRequestBodyBytes);
const existingFile = await findFileByHash(prepared.fileHash);
if (existingFile) {
@@ -204,7 +231,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
const { base64Data, mimeType: rawMimeType } = parseBase64File(file);
const estimatedSizeBytes = Math.floor((base64Data.length * 3) / 4);
if (estimatedSizeBytes > JSON_UPLOAD_LIMIT_BYTES) {
if (estimatedSizeBytes > JSON_UPLOAD_LIMIT_BYTES || estimatedSizeBytes > config.maxRequestBodyBytes) {
return Response.json(
{
error:
src/utils/file.ts
-10
View File
@@ -205,15 +205,10 @@ export const formatCreatedAt = (createdAt: Date | string | number): string => {
export interface UploadResponse {
public_id: string;
telegram_file_id: string;
telegram_file_unique_id: string;
storage_chat_id: number;
storage_message_id: number;
file_name: string;
mime_type: string;
size_bytes: number;
file_type: string;
uploader_id: number;
created_at: string;
download_url: string;
}
@@ -221,15 +216,10 @@ export interface UploadResponse {
export const buildUploadResponse = (file: FileMetadata, baseUrl: string): UploadResponse => {
return {
public_id: file.publicId,
telegram_file_id: file.telegramFileId,
telegram_file_unique_id: file.telegramFileUniqueId,
storage_chat_id: file.storageChatId,
storage_message_id: file.storageMessageId,
file_name: file.fileName,
mime_type: file.mimeType,
size_bytes: file.sizeBytes,
file_type: file.fileType,
uploader_id: file.uploaderId,
created_at: formatCreatedAt(file.createdAt),
download_url: `${baseUrl}/f/${file.publicId}`,
};
src/utils/ip.ts
+16
View File
@@ -0,0 +1,16 @@
import { config } from '../env';
export const extractClientIp = (req: Request): string => {
if (!config.trustProxy) return '127.0.0.1';
const forwardedFor = req.headers.get('x-forwarded-for');
if (forwardedFor) {
const firstIp = forwardedFor.split(',')[0]?.trim();
if (firstIp) return firstIp;
}
const realIp = req.headers.get('x-real-ip')?.trim();
if (realIp) return realIp;
return '127.0.0.1';
};
src/utils/rateLimit.ts
+63 -31
View File
@@ -1,41 +1,16 @@
import { config } from '../env';
import { extractClientIp } from './ip';
import logger from './logger';
// Simple sliding window rate limiter
interface RateLimitEntry {
count: number;
resetTime: number;
}
const rateLimitStore = new Map<string, RateLimitEntry>();
const WINDOW_SIZE_MS = 60000; // 1 minute window
const MAX_REQUESTS_PER_WINDOW = 100; // 100 requests per minute per IP
const MAX_STORE_ENTRIES = 50000;
export const checkRateLimit = (key: string): boolean => {
const now = Date.now();
const entry = rateLimitStore.get(key);
// No entry or window expired - create new entry
if (!entry || now > entry.resetTime) {
rateLimitStore.set(key, {
count: 1,
resetTime: now + WINDOW_SIZE_MS,
});
return true;
}
// Check if limit exceeded
if (entry.count >= MAX_REQUESTS_PER_WINDOW) {
logger.warn('Rate limit exceeded', { key, count: entry.count });
return false;
}
// Increment counter
entry.count++;
return true;
};
export const cleanupRateLimitCache = (): void => {
const now = Date.now();
const evictExpiredEntries = (now = Date.now()): number => {
let cleaned = 0;
for (const [key, entry] of rateLimitStore.entries()) {
@@ -45,6 +20,58 @@ export const cleanupRateLimitCache = (): void => {
}
}
return cleaned;
};
const ensureStoreCapacity = (now: number): void => {
if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
evictExpiredEntries(now);
while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
const oldestKey = rateLimitStore.keys().next().value;
if (!oldestKey) break;
rateLimitStore.delete(oldestKey);
}
};
export const checkRateLimit = (key: string): boolean => {
const now = Date.now();
const entry = rateLimitStore.get(key);
if (!entry || now > entry.resetTime) {
ensureStoreCapacity(now);
rateLimitStore.set(key, {
count: 1,
resetTime: now + config.rateLimitWindowMs,
});
return true;
}
if (entry.count >= config.rateLimitMaxRequests) {
logger.warn('Rate limit exceeded', { key, count: entry.count });
return false;
}
entry.count++;
return true;
};
export const withRateLimit = <T extends Request>(
handler: (req: T) => Promise<Response>,
): ((req: T) => Promise<Response>) => {
return async (req: T): Promise<Response> => {
const ip = extractClientIp(req);
if (!checkRateLimit(ip)) {
return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
}
return handler(req);
};
};
export const cleanupRateLimitCache = (): void => {
const cleaned = evictExpiredEntries();
if (cleaned > 0) {
logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
}
@@ -52,6 +79,11 @@ export const cleanupRateLimitCache = (): void => {
export const getRateLimitStats = () => ({
trackedIPs: rateLimitStore.size,
windowSize: WINDOW_SIZE_MS,
maxRequests: MAX_REQUESTS_PER_WINDOW,
windowSize: config.rateLimitWindowMs,
maxRequests: config.rateLimitMaxRequests,
maxTrackedIPs: MAX_STORE_ENTRIES,
});
export const clearRateLimitCache = (): void => {
rateLimitStore.clear();
};
src/utils/telegramQueue.ts
+2 -1
View File
@@ -1,8 +1,9 @@
import PQueue from 'p-queue';
import { config } from '../env';
import logger from './logger';
const uploadQueue = new PQueue({
concurrency: Number.POSITIVE_INFINITY,
concurrency: config.uploadConcurrency,
});
// Monitor queue events
src/utils/uploadBatcher.ts
+1 -3
View File
@@ -34,8 +34,6 @@ type PendingUpload = BatchUploadItem & {
};
const BATCH_WINDOW_MS = 2000;
const MAX_BATCH_ITEMS = 100;
const MAX_BATCH_SIZE_BYTES = 2 * 1024 * 1024 * 1024;
let pendingUploads: PendingUpload[] = [];
let flushTimer: ReturnType<typeof setTimeout> | null = null;
@@ -142,7 +140,7 @@ export const enqueuePreparedUpload = (item: BatchUploadItem): Promise<UploadedFi
}, BATCH_WINDOW_MS);
}
if (pendingUploads.length >= MAX_BATCH_ITEMS || getPendingSize() >= MAX_BATCH_SIZE_BYTES) {
if (pendingUploads.length >= config.batchMaxItems || getPendingSize() >= config.batchMaxSizeBytes) {
void flushUploads();
}
});
src/utils/zip.ts
+70 -19
View File
@@ -1,5 +1,5 @@
import { createReadStream, createWriteStream } from 'node:fs';
import { stat } from 'node:fs/promises';
import { open, stat } from 'node:fs/promises';
import { basename } from 'node:path';
import { nanoid } from 'nanoid';
@@ -48,13 +48,13 @@ const dosDateTime = (date = new Date()): { date: number; time: number } => {
};
};
const writeUInt16 = (value: number): Buffer => {
const writeUInt16 = (value: number): Buffer<ArrayBuffer> => {
const buffer = Buffer.allocUnsafe(2);
buffer.writeUInt16LE(value & 0xffff, 0);
return buffer;
};
const writeUInt32 = (value: number): Buffer => {
const writeUInt32 = (value: number): Buffer<ArrayBuffer> => {
const buffer = Buffer.allocUnsafe(4);
buffer.writeUInt32LE(value >>> 0, 0);
return buffer;
@@ -100,6 +100,21 @@ export const sanitizeZipEntryName = (fileName: string, usedNames = new Set<strin
return candidate;
};
const calculateFileCrc32 = async (tempPath: string): Promise<number> => {
let crc = 0xffffffff;
await new Promise<void>((resolve, reject) => {
const reader = createReadStream(tempPath);
reader.on('data', (chunk: Buffer) => {
crc = updateCrc32(crc, chunk);
});
reader.once('end', resolve);
reader.once('error', reject);
});
return (crc ^ 0xffffffff) >>> 0;
};
export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
const tempPath = `/tmp/teleuploader-${nanoid()}.zip`;
const writer = createWriteStream(tempPath);
@@ -121,20 +136,8 @@ export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
const fileStats = await stat(file.tempPath);
const { date, time } = dosDateTime();
const localHeaderOffset = offset;
let crc = 0xffffffff;
const crc32 = await calculateFileCrc32(file.tempPath);
const chunks: Buffer[] = [];
await new Promise<void>((resolve, reject) => {
const reader = createReadStream(file.tempPath);
reader.on('data', (chunk: Buffer) => {
crc = updateCrc32(crc, chunk);
chunks.push(chunk);
});
reader.once('end', resolve);
reader.once('error', reject);
});
const crc32 = (crc ^ 0xffffffff) >>> 0;
const localHeader = Buffer.concat([
writeUInt32(0x04034b50),
writeUInt16(20),
@@ -151,9 +154,14 @@ export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
]);
await writeHashed(localHeader);
for (const chunk of chunks) {
await writeHashed(chunk);
}
await new Promise<void>((resolve, reject) => {
const reader = createReadStream(file.tempPath);
reader.on('data', (chunk: Buffer) => {
void writeHashed(chunk).catch(reject);
});
reader.once('end', resolve);
reader.once('error', reject);
});
entries.push({
fileName: file.fileName,
@@ -251,3 +259,46 @@ export const extractZipEntry = async (
return null;
};
export type LocatedZipEntry = {
start: number;
length: number;
};
export const locateZipEntry = async (
zipPath: string,
entryName: string,
): Promise<LocatedZipEntry | null> => {
const handle = await open(zipPath, 'r');
let offset = 0;
try {
const header = Buffer.alloc(30);
while (true) {
const { bytesRead } = await handle.read(header, 0, header.byteLength, offset);
if (bytesRead < header.byteLength) return null;
const signature = header.readUInt32LE(0);
if (signature !== 0x04034b50) return null;
const compressionMethod = header.readUInt16LE(8);
const compressedSize = header.readUInt32LE(18);
const fileNameLength = header.readUInt16LE(26);
const extraLength = header.readUInt16LE(28);
const nameBuffer = Buffer.alloc(fileNameLength);
const nameOffset = offset + 30;
await handle.read(nameBuffer, 0, fileNameLength, nameOffset);
const dataStart = nameOffset + fileNameLength + extraLength;
if (nameBuffer.toString() === entryName) {
if (compressionMethod !== 0) return null;
return { start: dataStart, length: compressedSize };
}
offset = dataStart + compressedSize;
}
} finally {
await handle.close();
}
};
test/files.test.ts
+37 -27
View File
@@ -16,7 +16,6 @@ type FileInfoBody = {
mime_type: string;
size_bytes: number;
file_type: string;
uploader_id: number;
created_at: string;
};
@@ -70,11 +69,16 @@ mock.module('../src/utils/telegram', () => ({
}),
}));
// Mock rateLimit
const mockCheckRateLimit = mock(() => true);
mock.module('../src/utils/rateLimit', () => ({
checkRateLimit: mockCheckRateLimit,
}));
// Mock global fetch for proxy path
const originalFetch = globalThis.fetch;
const mockGlobalFetch = mock(async (_url: string) =>
Promise.resolve(
new Response('fake-file-content', {
status: 200,
headers: { 'Content-Type': 'application/octet-stream' },
}),
),
);
describe('File Route Handlers', () => {
let handleFileRedirect: typeof import('../src/routes/files').handleFileRedirect;
@@ -83,27 +87,23 @@ describe('File Route Handlers', () => {
beforeEach(async () => {
mockSelect.mockClear();
mockGetFile.mockClear();
mockCheckRateLimit.mockClear();
mockGlobalFetch.mockClear();
// Set up mock token
process.env.BOT_TOKEN = '123456:ABC-DEF';
globalThis.fetch = mockGlobalFetch as any;
const filesRoute = await import('../src/routes/files');
handleFileRedirect = filesRoute.handleFileRedirect;
handleFileInfo = filesRoute.handleFileInfo;
});
describe('handleFileRedirect', () => {
it('should return 429 if rate limit is exceeded', async () => {
mockCheckRateLimit.mockImplementationOnce(() => false);
const req = requestWithPublicId('http://localhost:3000/f/test-id', 'test-id');
const res = await handleFileRedirect(req);
expect(res.status).toBe(429);
const body = await responseJson<ErrorBody>(res);
expect(body.error).toBe('Rate limit exceeded');
afterAll(() => {
mock.restore();
globalThis.fetch = originalFetch;
});
describe('handleFileRedirect', () => {
it('should return 404 if file is not found in database', async () => {
mockSelect.mockImplementationOnce(() => ({
from: () => ({
@@ -120,7 +120,7 @@ describe('File Route Handlers', () => {
expect(body.error).toBe('File not found');
});
it('should redirect to telegram file url if file is found', async () => {
it('should proxy download (200 stream) instead of 302 redirect', async () => {
mockSelect.mockImplementationOnce(() => ({
from: () => ({
where: () => ({
@@ -131,6 +131,8 @@ describe('File Route Handlers', () => {
publicId: 'test-id',
telegramFileId: 'tg-file-id',
fileName: 'test.jpg',
mimeType: 'image/jpeg',
sizeBytes: 100,
},
]),
}),
@@ -139,10 +141,20 @@ describe('File Route Handlers', () => {
const req = requestWithPublicId('http://localhost:3000/f/test-id', 'test-id');
const res = await handleFileRedirect(req);
expect(res.status).toBe(302);
expect(res.headers.get('Location')).toBe(
'https://api.telegram.org/file/bot123456:ABC-DEF/photos/file_0.jpg',
);
// No longer 302 redirect
expect(res.status).toBe(200);
// No Location header with token
expect(res.headers.get('Location')).toBeNull();
// Should have Content-Disposition
const disposition = res.headers.get('Content-Disposition');
expect(disposition).toBeTruthy();
expect(disposition).toContain('test.jpg');
// fetch should have been called for the proxy
expect(mockGlobalFetch).toHaveBeenCalled();
});
it('should return 500 on database or external errors', async () => {
@@ -175,7 +187,7 @@ describe('File Route Handlers', () => {
expect(body.error).toBe('File not found');
});
it('should return file info JSON if file is found', async () => {
it('should return file info JSON without internal fields', async () => {
const dbFile = {
publicId: 'test-id',
fileName: 'image.png',
@@ -204,9 +216,11 @@ describe('File Route Handlers', () => {
mime_type: 'image/png',
size_bytes: 2048,
file_type: 'photo',
uploader_id: 99999,
created_at: '2026-05-18T00:00:00.000Z',
});
// No internal fields
expect(body).not.toHaveProperty('uploader_id');
expect(body).not.toHaveProperty('telegram_file_id');
});
it('should return 500 on database or external errors', async () => {
@@ -221,8 +235,4 @@ describe('File Route Handlers', () => {
expect(body.error).toBe('Server error');
});
});
afterAll(() => {
mock.restore();
});
});
test/rateLimit.test.ts
+33 -13
View File
@@ -1,25 +1,45 @@
import { beforeEach, describe, expect, it, spyOn } from 'bun:test';
import logger from '../src/utils/logger';
import { checkRateLimit, cleanupRateLimitCache } from '../src/utils/rateLimit';
// Spy on logger.warn
const warnSpy = spyOn(logger, 'warn');
import { beforeEach, describe, expect, it } from 'bun:test';
import { checkRateLimit, cleanupRateLimitCache, clearRateLimitCache } from '../src/utils/rateLimit';
describe('Rate Limiter', () => {
beforeEach(() => {
warnSpy.mockClear();
clearRateLimitCache();
});
it('should always allow requests as rate limiter is disabled', () => {
it('should allow requests up to the configured limit then block', () => {
const key = 'user-1';
// Default config maxRequests is 30; all 20 should pass
for (let i = 0; i < 20; i++) {
expect(checkRateLimit(key)).toBe(true);
expect(checkRateLimit(key)).toBe(true);
expect(checkRateLimit(key)).toBe(true);
expect(checkRateLimit(key)).toBe(true);
expect(warnSpy).not.toHaveBeenCalled();
}
});
it('should no-op on cleanup', () => {
it('should block requests when limit exceeded', () => {
const key = 'user-2';
// Exhaust the limit (30 by default)
for (let i = 0; i < 30; i++) {
checkRateLimit(key);
}
expect(checkRateLimit(key)).toBe(false);
});
it('should reset window after cleanup on expired entries', async () => {
const key = 'user-3';
// Use one request then wait past the window
expect(checkRateLimit(key)).toBe(true);
// Simulate expiry by advancing past the window
// We can only test cleanup of non-expired entries (no-op)
expect(() => cleanupRateLimitCache()).not.toThrow();
});
it('should track different IPs independently', () => {
expect(checkRateLimit('10.0.0.1')).toBe(true);
expect(checkRateLimit('10.0.0.1')).toBe(true);
expect(checkRateLimit('10.0.0.2')).toBe(true);
});
});
test/swagger.test.ts
+21 -5
View File
@@ -11,7 +11,7 @@ describe('Swagger Documentation Endpoints', () => {
const body = (await res.json()) as {
openapi: string;
info: { title: string };
paths: Record<string, { post?: { requestBody: { content: Record<string, unknown> } } }>;
paths: Record<string, { get?: object; post?: object }>;
};
expect(body.openapi).toBe('3.0.0');
expect(body.info.title).toBe('TeleUploader API');
@@ -19,10 +19,21 @@ describe('Swagger Documentation Endpoints', () => {
expect(body.paths).toHaveProperty('/api/upload');
expect(body.paths).toHaveProperty('/f/{public_id}');
expect(body.paths).toHaveProperty('/file/{public_id}/info');
expect(body.paths['/api/upload'].post.requestBody.content).toHaveProperty(
'multipart/form-data',
);
expect(body.paths['/api/upload'].post.requestBody.content).toHaveProperty('application/json');
const uploadPath = body.paths['/api/upload'] as any;
const downloadPath = body.paths['/f/{public_id}'] as any;
expect(uploadPath.post.requestBody.content).toHaveProperty('multipart/form-data');
expect(uploadPath.post.requestBody.content).toHaveProperty('application/json');
// Verify 429 response documented
const uploadResponses = uploadPath.post.responses;
expect(uploadResponses).toHaveProperty('429');
// Verify download is no longer documented as 302 redirect
const downloadResponses = downloadPath.get.responses;
expect(downloadResponses['200'].description).toContain('stream');
expect(downloadResponses).not.toHaveProperty('302');
});
it('returns Swagger UI HTML page', async () => {
@@ -37,4 +48,9 @@ describe('Swagger Documentation Endpoints', () => {
expect(html).toContain('/swagger.json');
expect(html).toContain('swagger-ui-bundle.js');
});
it('should not expose CORS * header', async () => {
const res = await handleSwaggerJson();
expect(res.headers.get('access-control-allow-origin')).toBeNull();
});
});
test/upload.test.ts
+36 -17
View File
@@ -13,7 +13,7 @@ beforeAll(async () => {
} catch {
// Fallback 1x1px JPEG
realPhotoBuffer = Buffer.from(
'ffd8ffe000104a46494600010101006000600000ffdb004300080606070605080707070909080a0c140d0c0b0b0c1912130f141d1a1f1e1d1a1c1c20242e2720222c231c1c2837292c30313434341f27393d38323c2e333432ffc0000b080001000101011100ffc4001f0000010501010110000000000000000000000102030405060708ffda000c03010002110311003f00a0ffd9',
'ffd8ffe000104a46494600010101006000600000ffdb004300080606070605080707070909080a0c140d0c0b0b0c1912130f141d1a1f1e1d1a1c1c20242e2720222c231c1c2837292c30313434341f27393d38323c2e333432ffc0b000080100010101011100ffc4001f0000010501010110000000000000000000000102030405060708ffda000c03010002110311003f00a0ffd9',
'hex',
);
}
@@ -22,8 +22,6 @@ beforeAll(async () => {
// Mock db
type UploadResponseBody = {
public_id: string;
telegram_file_id: string;
telegram_file_unique_id: string;
file_name: string;
file_type: string;
download_url: string;
@@ -79,19 +77,17 @@ const mockForwardToStorage = mock(() =>
}),
);
const mockGetFile = mock(() =>
mock.module('../src/utils/telegram', () => ({
forwardToStorage: mockForwardToStorage,
getBot: () => ({
telegram: {
getFile: mock(() =>
Promise.resolve({
file_id: 'tg-file-id-123',
file_size: 1000,
mime_type: 'image/jpeg',
}),
);
mock.module('../src/utils/telegram', () => ({
forwardToStorage: mockForwardToStorage,
getBot: () => ({
telegram: {
getFile: mockGetFile,
),
},
}),
}));
@@ -106,7 +102,6 @@ describe('Upload Route Handler', () => {
mockWhere.mockClear();
mockLimit.mockClear();
mockForwardToStorage.mockClear();
mockGetFile.mockClear();
mockSelectResult = [];
const uploadRoute = await import('../src/routes/upload');
handleUpload = uploadRoute.handleUpload;
@@ -144,10 +139,15 @@ describe('Upload Route Handler', () => {
const body = await uploadResponseJson(res);
expect(body.public_id).toContain('mocked-nanoid-id');
expect(body.telegram_file_id).toBe('tg-file-id-123');
expect(body.telegram_file_unique_id).toBe('tg-unique-id-abc');
expect(body.file_name).toBe('test.png');
expect(body.file_type).toBe('photo');
expect(body.download_url).toContain('/f/');
// No internal Telegram IDs in public response
expect(body).not.toHaveProperty('telegram_file_id');
expect(body).not.toHaveProperty('telegram_file_unique_id');
expect(body).not.toHaveProperty('storage_chat_id');
expect(body).not.toHaveProperty('storage_message_id');
expect(body).not.toHaveProperty('uploader_id');
});
it('should reject JSON upload without file key', async () => {
@@ -182,6 +182,7 @@ describe('Upload Route Handler', () => {
const body = await uploadResponseJson(res);
expect(body.public_id).toContain('mocked-nanoid-id');
expect(body.file_name).toBe('test_multi.png');
expect(body).not.toHaveProperty('telegram_file_id');
});
it('should deduplicate multipart upload if hash exists', async () => {
@@ -216,10 +217,9 @@ describe('Upload Route Handler', () => {
const body = await uploadResponseJson(res);
expect(body.public_id).toBe('existing-id-123');
expect(body.telegram_file_id).toBe('existing-tg-id');
expect(body.telegram_file_unique_id).toBe('existing-tg-unique');
expect(body.file_name).toBe('existing_name.txt');
expect(body.download_url).toContain('/f/existing-id-123');
expect(body).not.toHaveProperty('telegram_file_id');
// DB query happened
expect(mockSelect).toHaveBeenCalled();
@@ -263,9 +263,9 @@ describe('Upload Route Handler', () => {
const body = await uploadResponseJson(res);
expect(body.public_id).toBe('existing-json-id');
expect(body.telegram_file_id).toBe('existing-tg-json-id');
expect(body.file_name).toBe('existing_json.txt');
expect(body.download_url).toContain('/f/existing-json-id');
expect(body).not.toHaveProperty('telegram_file_id');
// DB query happened
expect(mockSelect).toHaveBeenCalled();
@@ -275,6 +275,25 @@ describe('Upload Route Handler', () => {
expect(mockInsert).not.toHaveBeenCalled();
});
it('should reject oversized request by Content-Length header', async () => {
const req = new Request('http://localhost:3000/api/upload', {
method: 'POST',
headers: {
'content-type': 'application/json',
'content-length': String(3 * 1024 * 1024 * 1024),
},
body: JSON.stringify({
file: Buffer.from('hello').toString('base64'),
fileName: 'test.txt',
}),
});
const res = await handleUpload(req);
expect(res.status).toBe(413);
const body = await uploadResponseJson(res);
expect(body.error).toContain('too large');
});
afterAll(() => {
mock.restore();
});