feat: enhance configuration and rate limiting

- Added new configuration options: trustProxy, uploadConcurrency, batchMaxItems, batchMaxSizeBytes, and maxRequestBodyBytes to AppConfig.
- Implemented utility functions for parsing environment variables and masking sensitive data.
- Updated rate limiting logic to use configurable window size and maximum requests per window.
- Introduced a middleware for rate limiting on specific routes.
- Refactored file handling routes to support streaming downloads instead of redirects.
- Improved error handling and response formatting in file routes.
- Added support for oversized request rejection based on Content-Length header.
- Updated Swagger documentation to reflect changes in API behavior and responses.
- Enhanced tests to cover new features and ensure proper functionality.

src/utils/rateLimit.ts

@@ -1,41 +1,16 @@
+import { config } from '../env';
+import { extractClientIp } from './ip';
 import logger from './logger';
 
-// Simple sliding window rate limiter
 interface RateLimitEntry {
   count: number;
   resetTime: number;
 }
 
 const rateLimitStore = new Map<string, RateLimitEntry>();
-const WINDOW_SIZE_MS = 60000; // 1 minute window
-const MAX_REQUESTS_PER_WINDOW = 100; // 100 requests per minute per IP
+const MAX_STORE_ENTRIES = 50000;
 
-export const checkRateLimit = (key: string): boolean => {
-  const now = Date.now();
-  const entry = rateLimitStore.get(key);
-
-  // No entry or window expired - create new entry
-  if (!entry || now > entry.resetTime) {
-    rateLimitStore.set(key, {
-      count: 1,
-      resetTime: now + WINDOW_SIZE_MS,
-    });
-    return true;
-  }
-
-  // Check if limit exceeded
-  if (entry.count >= MAX_REQUESTS_PER_WINDOW) {
-    logger.warn('Rate limit exceeded', { key, count: entry.count });
-    return false;
-  }
-
-  // Increment counter
-  entry.count++;
-  return true;
-};
-
-export const cleanupRateLimitCache = (): void => {
-  const now = Date.now();
+const evictExpiredEntries = (now = Date.now()): number => {
   let cleaned = 0;
 
   for (const [key, entry] of rateLimitStore.entries()) {
@@ -45,6 +20,58 @@ export const cleanupRateLimitCache = (): void => {
     }
   }
 
+  return cleaned;
+};
+
+const ensureStoreCapacity = (now: number): void => {
+  if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
+
+  evictExpiredEntries(now);
+  while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
+    const oldestKey = rateLimitStore.keys().next().value;
+    if (!oldestKey) break;
+    rateLimitStore.delete(oldestKey);
+  }
+};
+
+export const checkRateLimit = (key: string): boolean => {
+  const now = Date.now();
+  const entry = rateLimitStore.get(key);
+
+  if (!entry || now > entry.resetTime) {
+    ensureStoreCapacity(now);
+    rateLimitStore.set(key, {
+      count: 1,
+      resetTime: now + config.rateLimitWindowMs,
+    });
+    return true;
+  }
+
+  if (entry.count >= config.rateLimitMaxRequests) {
+    logger.warn('Rate limit exceeded', { key, count: entry.count });
+    return false;
+  }
+
+  entry.count++;
+  return true;
+};
+
+export const withRateLimit = <T extends Request>(
+  handler: (req: T) => Promise<Response>,
+): ((req: T) => Promise<Response>) => {
+  return async (req: T): Promise<Response> => {
+    const ip = extractClientIp(req);
+    if (!checkRateLimit(ip)) {
+      return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
+    }
+
+    return handler(req);
+  };
+};
+
+export const cleanupRateLimitCache = (): void => {
+  const cleaned = evictExpiredEntries();
+
   if (cleaned > 0) {
     logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
   }
@@ -52,6 +79,11 @@ export const cleanupRateLimitCache = (): void => {
 
 export const getRateLimitStats = () => ({
   trackedIPs: rateLimitStore.size,
-  windowSize: WINDOW_SIZE_MS,
-  maxRequests: MAX_REQUESTS_PER_WINDOW,
+  windowSize: config.rateLimitWindowMs,
+  maxRequests: config.rateLimitMaxRequests,
+  maxTrackedIPs: MAX_STORE_ENTRIES,
 });
+
+export const clearRateLimitCache = (): void => {
+  rateLimitStore.clear();
+};