feat: enhance configuration and rate limiting

- Added new configuration options: trustProxy, uploadConcurrency, batchMaxItems, batchMaxSizeBytes, and maxRequestBodyBytes to AppConfig.
- Implemented utility functions for parsing environment variables and masking sensitive data.
- Updated rate limiting logic to use configurable window size and maximum requests per window.
- Introduced a middleware for rate limiting on specific routes.
- Refactored file handling routes to support streaming downloads instead of redirects.
- Improved error handling and response formatting in file routes.
- Added support for oversized request rejection based on Content-Length header.
- Updated Swagger documentation to reflect changes in API behavior and responses.
- Enhanced tests to cover new features and ensure proper functionality.

src/utils/file.ts

@@ -205,15 +205,10 @@ export const formatCreatedAt = (createdAt: Date | string | number): string => {
 
 export interface UploadResponse {
   public_id: string;
-  telegram_file_id: string;
-  telegram_file_unique_id: string;
-  storage_chat_id: number;
-  storage_message_id: number;
   file_name: string;
   mime_type: string;
   size_bytes: number;
   file_type: string;
-  uploader_id: number;
   created_at: string;
   download_url: string;
 }
@@ -221,15 +216,10 @@ export interface UploadResponse {
 export const buildUploadResponse = (file: FileMetadata, baseUrl: string): UploadResponse => {
   return {
     public_id: file.publicId,
-    telegram_file_id: file.telegramFileId,
-    telegram_file_unique_id: file.telegramFileUniqueId,
-    storage_chat_id: file.storageChatId,
-    storage_message_id: file.storageMessageId,
     file_name: file.fileName,
     mime_type: file.mimeType,
     size_bytes: file.sizeBytes,
     file_type: file.fileType,
-    uploader_id: file.uploaderId,
     created_at: formatCreatedAt(file.createdAt),
     download_url: `${baseUrl}/f/${file.publicId}`,
   };

src/utils/ip.ts

@@ -0,0 +1,16 @@
+import { config } from '../env';
+
+export const extractClientIp = (req: Request): string => {
+  if (!config.trustProxy) return '127.0.0.1';
+
+  const forwardedFor = req.headers.get('x-forwarded-for');
+  if (forwardedFor) {
+    const firstIp = forwardedFor.split(',')[0]?.trim();
+    if (firstIp) return firstIp;
+  }
+
+  const realIp = req.headers.get('x-real-ip')?.trim();
+  if (realIp) return realIp;
+
+  return '127.0.0.1';
+};

src/utils/rateLimit.ts

@@ -1,41 +1,16 @@
+import { config } from '../env';
+import { extractClientIp } from './ip';
 import logger from './logger';
 
-// Simple sliding window rate limiter
 interface RateLimitEntry {
   count: number;
   resetTime: number;
 }
 
 const rateLimitStore = new Map<string, RateLimitEntry>();
-const WINDOW_SIZE_MS = 60000; // 1 minute window
-const MAX_REQUESTS_PER_WINDOW = 100; // 100 requests per minute per IP
+const MAX_STORE_ENTRIES = 50000;
 
-export const checkRateLimit = (key: string): boolean => {
-  const now = Date.now();
-  const entry = rateLimitStore.get(key);
-
-  // No entry or window expired - create new entry
-  if (!entry || now > entry.resetTime) {
-    rateLimitStore.set(key, {
-      count: 1,
-      resetTime: now + WINDOW_SIZE_MS,
-    });
-    return true;
-  }
-
-  // Check if limit exceeded
-  if (entry.count >= MAX_REQUESTS_PER_WINDOW) {
-    logger.warn('Rate limit exceeded', { key, count: entry.count });
-    return false;
-  }
-
-  // Increment counter
-  entry.count++;
-  return true;
-};
-
-export const cleanupRateLimitCache = (): void => {
-  const now = Date.now();
+const evictExpiredEntries = (now = Date.now()): number => {
   let cleaned = 0;
 
   for (const [key, entry] of rateLimitStore.entries()) {
@@ -45,6 +20,58 @@ export const cleanupRateLimitCache = (): void => {
     }
   }
 
+  return cleaned;
+};
+
+const ensureStoreCapacity = (now: number): void => {
+  if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
+
+  evictExpiredEntries(now);
+  while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
+    const oldestKey = rateLimitStore.keys().next().value;
+    if (!oldestKey) break;
+    rateLimitStore.delete(oldestKey);
+  }
+};
+
+export const checkRateLimit = (key: string): boolean => {
+  const now = Date.now();
+  const entry = rateLimitStore.get(key);
+
+  if (!entry || now > entry.resetTime) {
+    ensureStoreCapacity(now);
+    rateLimitStore.set(key, {
+      count: 1,
+      resetTime: now + config.rateLimitWindowMs,
+    });
+    return true;
+  }
+
+  if (entry.count >= config.rateLimitMaxRequests) {
+    logger.warn('Rate limit exceeded', { key, count: entry.count });
+    return false;
+  }
+
+  entry.count++;
+  return true;
+};
+
+export const withRateLimit = <T extends Request>(
+  handler: (req: T) => Promise<Response>,
+): ((req: T) => Promise<Response>) => {
+  return async (req: T): Promise<Response> => {
+    const ip = extractClientIp(req);
+    if (!checkRateLimit(ip)) {
+      return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
+    }
+
+    return handler(req);
+  };
+};
+
+export const cleanupRateLimitCache = (): void => {
+  const cleaned = evictExpiredEntries();
+
   if (cleaned > 0) {
     logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
   }
@@ -52,6 +79,11 @@ export const cleanupRateLimitCache = (): void => {
 
 export const getRateLimitStats = () => ({
   trackedIPs: rateLimitStore.size,
-  windowSize: WINDOW_SIZE_MS,
-  maxRequests: MAX_REQUESTS_PER_WINDOW,
+  windowSize: config.rateLimitWindowMs,
+  maxRequests: config.rateLimitMaxRequests,
+  maxTrackedIPs: MAX_STORE_ENTRIES,
 });
+
+export const clearRateLimitCache = (): void => {
+  rateLimitStore.clear();
+};

src/utils/telegramQueue.ts

@@ -1,8 +1,9 @@
 import PQueue from 'p-queue';
+import { config } from '../env';
 import logger from './logger';
 
 const uploadQueue = new PQueue({
-  concurrency: Number.POSITIVE_INFINITY,
+  concurrency: config.uploadConcurrency,
 });
 
 // Monitor queue events

src/utils/uploadBatcher.ts

@@ -34,8 +34,6 @@ type PendingUpload = BatchUploadItem & {
 };
 
 const BATCH_WINDOW_MS = 2000;
-const MAX_BATCH_ITEMS = 100;
-const MAX_BATCH_SIZE_BYTES = 2 * 1024 * 1024 * 1024;
 
 let pendingUploads: PendingUpload[] = [];
 let flushTimer: ReturnType<typeof setTimeout> | null = null;
@@ -142,7 +140,7 @@ export const enqueuePreparedUpload = (item: BatchUploadItem): Promise<UploadedFi
       }, BATCH_WINDOW_MS);
     }
 
-    if (pendingUploads.length >= MAX_BATCH_ITEMS || getPendingSize() >= MAX_BATCH_SIZE_BYTES) {
+    if (pendingUploads.length >= config.batchMaxItems || getPendingSize() >= config.batchMaxSizeBytes) {
       void flushUploads();
     }
   });

src/utils/zip.ts

@@ -1,5 +1,5 @@
 import { createReadStream, createWriteStream } from 'node:fs';
-import { stat } from 'node:fs/promises';
+import { open, stat } from 'node:fs/promises';
 import { basename } from 'node:path';
 import { nanoid } from 'nanoid';
 
@@ -48,13 +48,13 @@ const dosDateTime = (date = new Date()): { date: number; time: number } => {
   };
 };
 
-const writeUInt16 = (value: number): Buffer => {
+const writeUInt16 = (value: number): Buffer<ArrayBuffer> => {
   const buffer = Buffer.allocUnsafe(2);
   buffer.writeUInt16LE(value & 0xffff, 0);
   return buffer;
 };
 
-const writeUInt32 = (value: number): Buffer => {
+const writeUInt32 = (value: number): Buffer<ArrayBuffer> => {
   const buffer = Buffer.allocUnsafe(4);
   buffer.writeUInt32LE(value >>> 0, 0);
   return buffer;
@@ -100,6 +100,21 @@ export const sanitizeZipEntryName = (fileName: string, usedNames = new Set<strin
   return candidate;
 };
 
+const calculateFileCrc32 = async (tempPath: string): Promise<number> => {
+  let crc = 0xffffffff;
+
+  await new Promise<void>((resolve, reject) => {
+    const reader = createReadStream(tempPath);
+    reader.on('data', (chunk: Buffer) => {
+      crc = updateCrc32(crc, chunk);
+    });
+    reader.once('end', resolve);
+    reader.once('error', reject);
+  });
+
+  return (crc ^ 0xffffffff) >>> 0;
+};
+
 export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
   const tempPath = `/tmp/teleuploader-${nanoid()}.zip`;
   const writer = createWriteStream(tempPath);
@@ -121,20 +136,8 @@ export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
       const fileStats = await stat(file.tempPath);
       const { date, time } = dosDateTime();
       const localHeaderOffset = offset;
-      let crc = 0xffffffff;
+      const crc32 = await calculateFileCrc32(file.tempPath);
 
-      const chunks: Buffer[] = [];
-      await new Promise<void>((resolve, reject) => {
-        const reader = createReadStream(file.tempPath);
-        reader.on('data', (chunk: Buffer) => {
-          crc = updateCrc32(crc, chunk);
-          chunks.push(chunk);
-        });
-        reader.once('end', resolve);
-        reader.once('error', reject);
-      });
-
-      const crc32 = (crc ^ 0xffffffff) >>> 0;
       const localHeader = Buffer.concat([
         writeUInt32(0x04034b50),
         writeUInt16(20),
@@ -151,9 +154,14 @@ export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
       ]);
 
       await writeHashed(localHeader);
-      for (const chunk of chunks) {
-        await writeHashed(chunk);
-      }
+      await new Promise<void>((resolve, reject) => {
+        const reader = createReadStream(file.tempPath);
+        reader.on('data', (chunk: Buffer) => {
+          void writeHashed(chunk).catch(reject);
+        });
+        reader.once('end', resolve);
+        reader.once('error', reject);
+      });
 
       entries.push({
         fileName: file.fileName,
@@ -251,3 +259,46 @@ export const extractZipEntry = async (
 
   return null;
 };
+
+export type LocatedZipEntry = {
+  start: number;
+  length: number;
+};
+
+export const locateZipEntry = async (
+  zipPath: string,
+  entryName: string,
+): Promise<LocatedZipEntry | null> => {
+  const handle = await open(zipPath, 'r');
+  let offset = 0;
+
+  try {
+    const header = Buffer.alloc(30);
+
+    while (true) {
+      const { bytesRead } = await handle.read(header, 0, header.byteLength, offset);
+      if (bytesRead < header.byteLength) return null;
+
+      const signature = header.readUInt32LE(0);
+      if (signature !== 0x04034b50) return null;
+
+      const compressionMethod = header.readUInt16LE(8);
+      const compressedSize = header.readUInt32LE(18);
+      const fileNameLength = header.readUInt16LE(26);
+      const extraLength = header.readUInt16LE(28);
+      const nameBuffer = Buffer.alloc(fileNameLength);
+      const nameOffset = offset + 30;
+      await handle.read(nameBuffer, 0, fileNameLength, nameOffset);
+
+      const dataStart = nameOffset + fileNameLength + extraLength;
+      if (nameBuffer.toString() === entryName) {
+        if (compressionMethod !== 0) return null;
+        return { start: dataStart, length: compressedSize };
+      }
+
+      offset = dataStart + compressedSize;
+    }
+  } finally {
+    await handle.close();
+  }
+};