From 4c216b1d9f0481c2dab99c085e3a0e0daeb56c57 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:41:50 +0700 Subject: [PATCH] refactor: full DDD + Clean Architecture refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Hapus src/utils/ (17 files) + src/db/ (8 files) dead code - Absorb 7 re-export stubs → real impl di lokasi DDD - Buat src/infrastructure/di.ts (DI container) - Rewrite 5 controllers pakai repository/DI - Fix shared/utils imports, env.ts, routes, index.ts - Update package.json build path migrate - Lint clean, build clean --- package.json | 2 +- src/db/buckets.ts | 81 --- src/db/file-parts.ts | 97 ---- src/db/files-ext.ts | 143 ------ src/db/files.ts | 26 - src/db/index.ts | 13 - src/db/migrate.ts | 53 -- src/db/multipart.ts | 137 ----- src/db/schema.ts | 71 --- src/env.ts | 2 +- src/index.ts | 2 +- src/infrastructure/di.ts | 51 ++ .../http/controllers/auth-controller.ts | 2 +- .../http/controllers/file-controller.ts | 6 +- .../http/controllers/s3-controller.ts | 122 ++--- .../http/controllers/upload-controller.ts | 17 +- .../http/controllers/web-api-controller.ts | 52 +- src/interfaces/http/middleware/rate-limit.ts | 96 +++- src/interfaces/http/routes/index.ts | 2 +- src/interfaces/s3/auth.ts | 470 +++++++++++++++++- src/interfaces/s3/headers.ts | 62 ++- src/interfaces/s3/virtual-host.ts | 28 +- src/interfaces/s3/xml.ts | 333 ++++++++++++- src/shared/logger/index.ts | 30 +- src/shared/metrics/index.ts | 112 ++++- src/shared/utils/file.ts | 2 +- src/shared/utils/retry.ts | 2 +- src/utils/auth.ts | 205 -------- src/utils/cache.ts | 74 --- src/utils/chunked-storage.ts | 249 ---------- src/utils/file.ts | 237 --------- src/utils/ip.ts | 16 - src/utils/logger.ts | 25 - src/utils/metrics.ts | 111 ----- src/utils/rateLimit.ts | 89 ---- src/utils/retry.ts | 91 ---- src/utils/s3/auth.ts | 463 ----------------- src/utils/s3/headers.ts | 52 -- src/utils/s3/object-stream.ts | 133 ----- src/utils/s3/range.ts | 46 -- src/utils/s3/virtual-host.ts | 27 - src/utils/s3/xml.ts | 311 ------------ src/utils/uploadBatcher.ts | 152 ------ src/utils/zip.ts | 292 ----------- 44 files changed, 1223 insertions(+), 3364 deletions(-) delete mode 100644 src/db/buckets.ts delete mode 100644 src/db/file-parts.ts delete mode 100644 src/db/files-ext.ts delete mode 100644 src/db/files.ts delete mode 100644 src/db/index.ts delete mode 100644 src/db/migrate.ts delete mode 100644 src/db/multipart.ts delete mode 100644 src/db/schema.ts create mode 100644 src/infrastructure/di.ts delete mode 100644 src/utils/auth.ts delete mode 100644 src/utils/cache.ts delete mode 100644 src/utils/chunked-storage.ts delete mode 100644 src/utils/file.ts delete mode 100644 src/utils/ip.ts delete mode 100644 src/utils/logger.ts delete mode 100644 src/utils/metrics.ts delete mode 100644 src/utils/rateLimit.ts delete mode 100644 src/utils/retry.ts delete mode 100644 src/utils/s3/auth.ts delete mode 100644 src/utils/s3/headers.ts delete mode 100644 src/utils/s3/object-stream.ts delete mode 100644 src/utils/s3/range.ts delete mode 100644 src/utils/s3/virtual-host.ts delete mode 100644 src/utils/s3/xml.ts delete mode 100644 src/utils/uploadBatcher.ts delete mode 100644 src/utils/zip.ts diff --git a/package.json b/package.json index d7b6fa3..cbb34ec 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "dev": "bun --hot src/index.ts", - "build": "bun build src/index.ts --target=bun --outfile=dist/index.js && bun build src/db/migrate.ts --target=bun --outfile=dist/migrate.js", + "build": "bun build src/index.ts --target=bun --outfile=dist/index.js && bun build src/infrastructure/persistence/drizzle/migrate.ts --target=bun --outfile=dist/migrate.js", "start": "NODE_ENV=production bun dist/index.js", "db:migrate": "bun dist/migrate.js", "test": "bun test --preload ./test/helpers/setup-env.ts test/rateLimit.test.ts && bun test --preload ./test/helpers/setup-env.ts test/file.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegram.test.ts && bun test --preload ./test/helpers/setup-env.ts test/upload.test.ts && bun test --preload ./test/helpers/setup-env.ts test/files.test.ts && bun test --preload ./test/helpers/setup-env.ts test/health.test.ts && bun test --preload ./test/helpers/setup-env.ts test/db.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bot.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bootstrap.test.ts && bun test --preload ./test/helpers/setup-env.ts test/swagger.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth-routes.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-operations.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-bucket-config.test.ts && bun test --preload ./test/helpers/setup-env.ts test/web-api.test.ts && bun test --preload ./test/helpers/setup-env.ts test/env.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegramQueue.test.ts", diff --git a/src/db/buckets.ts b/src/db/buckets.ts deleted file mode 100644 index 669509e..0000000 --- a/src/db/buckets.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { sql } from 'drizzle-orm'; -import { db } from './index'; - -export interface Bucket { - id: string; - name: string; - createdAt: Date; - updatedAt: Date; -} - -type QueryRow = Record; -type QueryResult = QueryRow[]; - -export const createBucket = async (name: string): Promise => { - const result = (await db.execute( - sql`INSERT INTO buckets (name) VALUES (${name}) RETURNING id, name, created_at, updated_at`, - )) as unknown as QueryResult; - const row = result[0]!; - return { - id: row.id as string, - name: row.name as string, - createdAt: new Date(row.created_at as string), - updatedAt: new Date(row.updated_at as string), - }; -}; - -export const findBucketByName = async (name: string): Promise => { - const result = (await db.execute( - sql`SELECT id, name, created_at, updated_at FROM buckets WHERE name = ${name}`, - )) as unknown as QueryResult; - if (result.length === 0) return null; - const row = result[0]!; - return { - id: row.id as string, - name: row.name as string, - createdAt: new Date(row.created_at as string), - updatedAt: new Date(row.updated_at as string), - }; -}; - -export const listBuckets = async (): Promise => { - const result = (await db.execute( - sql`SELECT id, name, created_at, updated_at FROM buckets ORDER BY name`, - )) as unknown as QueryResult; - return result.map((row) => ({ - id: row.id as string, - name: row.name as string, - createdAt: new Date(row.created_at as string), - updatedAt: new Date(row.updated_at as string), - })); -}; - -export const deleteBucket = async (name: string): Promise => { - // Cascade-delete rows that hold FK references to the bucket - await db - .execute( - sql`DELETE FROM multipart_parts WHERE upload_id IN (SELECT upload_id FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name}))`, - ) - .catch(() => {}); - await db - .execute( - sql`DELETE FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`, - ) - .catch(() => {}); - await db - .execute( - sql`DELETE FROM files WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`, - ) - .catch(() => {}); - const result = (await db.execute( - sql`DELETE FROM buckets WHERE name = ${name}`, - )) as unknown as QueryResult; - return result.length > 0; -}; - -export const bucketExists = async (name: string): Promise => { - const result = (await db.execute( - sql`SELECT 1 FROM buckets WHERE name = ${name}`, - )) as unknown as QueryResult; - return result.length > 0; -}; diff --git a/src/db/file-parts.ts b/src/db/file-parts.ts deleted file mode 100644 index 45bcc34..0000000 --- a/src/db/file-parts.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { sql } from 'drizzle-orm'; -import { db } from './index'; - -export type CompressionAlgorithm = 'gzip' | null; - -export interface FilePart { - id: number; - fileId: string; - partNumber: number; - telegramFileId: string; - telegramFileUniqueId: string; - storageChatId: number; - storageMessageId: number; - sizeBytes: number; - storedSizeBytes: number; - compressionAlgorithm: CompressionAlgorithm; - etag: string; - createdAt: Date; -} - -export type NewFilePartInput = Omit; - -const toNumber = (value: unknown): number => Number(value ?? 0); - -const mapRowToFilePart = (row: Record): FilePart => ({ - id: toNumber(row.id), - fileId: row.file_id as string, - partNumber: toNumber(row.part_number), - telegramFileId: row.telegram_file_id as string, - telegramFileUniqueId: row.telegram_file_unique_id as string, - storageChatId: toNumber(row.storage_chat_id), - storageMessageId: toNumber(row.storage_message_id), - sizeBytes: toNumber(row.size_bytes), - storedSizeBytes: toNumber(row.stored_size_bytes), - compressionAlgorithm: (row.compression_algorithm as CompressionAlgorithm) || null, - etag: row.etag as string, - createdAt: new Date(row.created_at as string), -}); - -export const insertFileParts = async (parts: NewFilePartInput[]): Promise => { - for (const part of parts) { - await db.execute( - sql`INSERT INTO file_parts ( - file_id, - part_number, - telegram_file_id, - telegram_file_unique_id, - storage_chat_id, - storage_message_id, - size_bytes, - stored_size_bytes, - compression_algorithm, - etag - ) VALUES ( - ${part.fileId}::uuid, - ${part.partNumber}, - ${part.telegramFileId}, - ${part.telegramFileUniqueId}, - ${part.storageChatId}, - ${part.storageMessageId}, - ${part.sizeBytes}, - ${part.storedSizeBytes}, - ${part.compressionAlgorithm}, - ${part.etag} - )`, - ); - } -}; - -export const listFileParts = async (fileId: string): Promise => { - const result = (await db.execute( - sql`SELECT id, - file_id, - part_number, - telegram_file_id, - telegram_file_unique_id, - storage_chat_id, - storage_message_id, - size_bytes, - stored_size_bytes, - compression_algorithm, - etag, - created_at - FROM file_parts - WHERE file_id = ${fileId}::uuid - ORDER BY part_number`, - )) as unknown as Record[]; - - return result.map(mapRowToFilePart); -}; - -export const countFileParts = async (fileId: string): Promise => { - const result = (await db.execute( - sql`SELECT COUNT(*) AS count FROM file_parts WHERE file_id = ${fileId}::uuid`, - )) as unknown as Record[]; - return toNumber(result[0]?.count); -}; diff --git a/src/db/files-ext.ts b/src/db/files-ext.ts deleted file mode 100644 index fe4426b..0000000 --- a/src/db/files-ext.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { and, eq, sql } from 'drizzle-orm'; -import { db, files as fileSchema } from './index'; -import type { File } from './schema'; - -export interface S3FileRecord extends File { - bucketId: string; - s3Key: string; -} - -export const findFileByBucketAndKey = async ( - bucketId: string, - s3Key: string, -): Promise => { - const result = await db - .select() - .from(fileSchema) - .where( - and( - eq(fileSchema.bucketId, bucketId), - eq(fileSchema.s3Key, s3Key), - eq(fileSchema.isDeleted, false), - ), - ) - .limit(1); - return result[0] || null; -}; - -const mapDbRowToS3Record = (row: Record): S3FileRecord => { - return { - id: row.id as string, - publicId: row.public_id as string, - telegramFileId: row.telegram_file_id as string, - telegramFileUniqueId: row.telegram_file_unique_id as string, - storageChatId: toNumber(row.storage_chat_id), - storageMessageId: toNumber(row.storage_message_id), - fileName: row.file_name as string, - mimeType: row.mime_type as string, - sizeBytes: toNumber(row.size_bytes), - fileType: row.file_type as string, - uploaderId: toNumber(row.uploader_id), - fileHash: row.file_hash as string | null, - archiveTelegramFileId: row.archive_telegram_file_id as string | null, - archiveStorageMessageId: - row.archive_storage_message_id === null ? null : toNumber(row.archive_storage_message_id), - archiveFileName: row.archive_file_name as string | null, - archiveEntryName: row.archive_entry_name as string | null, - archiveMimeType: row.archive_mime_type as string | null, - archiveSizeBytes: row.archive_size_bytes === null ? null : toNumber(row.archive_size_bytes), - bucketId: row.bucket_id as string, - s3Key: row.s3_key as string, - storageBackend: (row.storage_backend as string) || 'telegram', - isDeleted: row.is_deleted as boolean, - multipartUploadId: row.multipart_upload_id as string | null, - partCount: - row.part_count === null || row.part_count === undefined ? null : toNumber(row.part_count), - createdAt: new Date(row.created_at as string), - updatedAt: new Date(row.updated_at as string), - }; -}; - -const escapeLike = (s: string): string => s.replace(/[%_\\]/g, '\\$&'); - -const toNumber = (value: unknown): number => Number(value ?? 0); - -export const listObjectsByPrefix = async ( - bucketId: string, - prefix: string, - delimiter: string | null, - maxKeys: number, - startAfter: string | null, -): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => { - let query = prefix - ? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${escapeLike(prefix)}%`}` - : sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`; - - if (startAfter) { - query = sql`${query} AND s3_key > ${startAfter}`; - } - - query = sql`${query} ORDER BY s3_key LIMIT ${maxKeys + 1}`; - - const rawResult = (await db.execute(query)) as unknown as Record[]; - - if (delimiter === '/') { - const prefixSet = new Set(); - const objects: S3FileRecord[] = []; - - for (const row of rawResult) { - const s3Key = row.s3_key as string; - const relativeKey = s3Key.substring(prefix.length); - const slashIndex = relativeKey.indexOf('/'); - if (slashIndex >= 0) { - const folderPrefix = prefix + relativeKey.substring(0, slashIndex + 1); - if (folderPrefix !== prefix) { - prefixSet.add(folderPrefix); - } - } else { - objects.push(mapDbRowToS3Record(row)); - } - } - - return { - objects: objects.slice(0, maxKeys), - prefixes: Array.from(prefixSet).sort(), - }; - } - - return { - objects: rawResult.slice(0, maxKeys).map(mapDbRowToS3Record), - prefixes: [], - }; -}; - -export const softDeleteFile = async (bucketId: string, s3Key: string): Promise => { - const result = (await db.execute( - sql`UPDATE files SET is_deleted = true WHERE bucket_id = ${bucketId}::uuid AND s3_key = ${s3Key} RETURNING id`, - )) as unknown as Record[]; - return result.length > 0; -}; - -export const softDeleteFilesBatch = async (bucketId: string, keys: string[]): Promise => { - let deleted = 0; - for (const key of keys) { - const ok = await softDeleteFile(bucketId, key); - if (ok) deleted++; - } - return deleted; -}; - -export const countBucketObjects = async (bucketId: string): Promise => { - const result = (await db.execute( - sql`SELECT count(*) as count FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`, - )) as unknown as Record[]; - return Number(result[0]?.count || 0); -}; - -export const findOrphanFilesByBucket = async (bucketId: string): Promise => { - return await db - .select() - .from(fileSchema) - .where(and(eq(fileSchema.bucketId, bucketId), eq(fileSchema.isDeleted, true))) - .limit(100); -}; diff --git a/src/db/files.ts b/src/db/files.ts deleted file mode 100644 index 6a0b722..0000000 --- a/src/db/files.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { eq } from 'drizzle-orm'; -import { db, files as fileSchema } from './index'; -import type { File } from './schema'; - -export const findFileByHash = async (hash: string): Promise => { - const result = await db.select().from(fileSchema).where(eq(fileSchema.fileHash, hash)).limit(1); - return result[0] || null; -}; - -export const findFileByPublicId = async (publicId: string): Promise => { - const result = await db - .select() - .from(fileSchema) - .where(eq(fileSchema.publicId, publicId)) - .limit(1); - return result[0] || null; -}; - -export const findFileByUniqueId = async (telegramFileUniqueId: string): Promise => { - const result = await db - .select() - .from(fileSchema) - .where(eq(fileSchema.telegramFileUniqueId, telegramFileUniqueId)) - .limit(1); - return result[0] || null; -}; diff --git a/src/db/index.ts b/src/db/index.ts deleted file mode 100644 index f413737..0000000 --- a/src/db/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { drizzle } from 'drizzle-orm/postgres-js'; -import postgres from 'postgres'; -import { fileParts, files } from './schema'; - -const client = postgres(process.env.DATABASE_URL!, { - max: 10, - idle_timeout: 20, - connect_timeout: 10, -}); - -export const db = drizzle(client, { schema: { fileParts, files } }); -export { fileParts, files }; -export default db; diff --git a/src/db/migrate.ts b/src/db/migrate.ts deleted file mode 100644 index c67441e..0000000 --- a/src/db/migrate.ts +++ /dev/null @@ -1,53 +0,0 @@ -import postgres from 'postgres'; -import { config } from '../env'; -import { getErrorMessage } from '../utils/file'; -import logger from '../utils/logger'; - -/** - * Run raw SQL migration from schema.sql. - * Safe to call multiple times — all statements use IF NOT EXISTS. - */ -export const runMigration = async (): Promise => { - // In compiled dist: import.meta.dir = .../dist/ - // In source via bun --hot: import.meta.dir = .../src/db/ - const dir = import.meta.dir || ''; - const candidates = [ - `${dir}/../../schema.sql`, // from dist/ - `${dir}/../schema.sql`, // from src/ (bun --hot src/index.ts) - `${dir}/../schema.sql`, // from src/db/ (bun --hot src/db/migrate.ts) - `${dir}/schema.sql`, // from src/ (bun run db:migrate) - ]; - - let schemaSql: string | null = null; - for (const p of candidates) { - const file = Bun.file(p); - const exists = await file.exists(); - if (exists) { - schemaSql = await file.text(); - break; - } - } - - if (!schemaSql) { - logger.error(`Migration failed: schema.sql not found (tried ${candidates.join(', ')})`); - process.exitCode = 1; - return; - } - - const sql = postgres(config.databaseUrl, { max: 1 }); - - try { - await sql.unsafe(schemaSql); - logger.info('Database migration completed'); - } catch (error: unknown) { - logger.error('Database migration failed', { error: getErrorMessage(error) }); - process.exitCode = 1; - } finally { - await sql.end(); - } -}; - -// When run directly: `bun src/db/migrate.ts` or `bun dist/migrate.js` -if (import.meta.path === Bun.main) { - await runMigration(); -} diff --git a/src/db/multipart.ts b/src/db/multipart.ts deleted file mode 100644 index 3ed474c..0000000 --- a/src/db/multipart.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { sql } from 'drizzle-orm'; -import { nanoid } from 'nanoid'; -import { db } from './index'; - -export interface MultipartUpload { - uploadId: string; - bucketId: string; - s3Key: string; - initiatedAt: Date; - status: string; - initiatedBy: string; - contentType: string | null; -} - -export interface MultipartPart { - id: number; - uploadId: string; - partNumber: number; - telegramFileId: string; - telegramFileUniqueId: string; - storageMessageId: number; - sizeBytes: number; - etag: string; - createdAt: Date; -} - -export const createMultipartUpload = async ( - bucketId: string, - s3Key: string, - initiatedBy: string, - contentType?: string | null, -): Promise => { - const uploadId = nanoid(32); - await db.execute( - contentType - ? sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by, content_type) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy}, ${contentType})` - : sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy})`, - ); - return uploadId; -}; - -export const findMultipartUpload = async (uploadId: string): Promise => { - const result = (await db.execute( - sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, content_type FROM multipart_uploads WHERE upload_id = ${uploadId} AND status = 'in_progress'`, - )) as unknown as Record[]; - if (result.length === 0) return null; - const r = result[0]!; - return { - uploadId: r.upload_id as string, - bucketId: r.bucket_id as string, - s3Key: r.s3_key as string, - initiatedAt: new Date(r.initiated_at as string), - status: r.status as string, - initiatedBy: '', - contentType: (r.content_type as string | null) || null, - }; -}; - -export const completeMultipartUpload = async (uploadId: string): Promise => { - await db.execute( - sql`UPDATE multipart_uploads SET status = 'completed' WHERE upload_id = ${uploadId}`, - ); -}; - -export const abortMultipartUpload = async (uploadId: string): Promise => { - // H7: FK cascade only fires on DELETE, not UPDATE. Delete parts explicitly - // before updating the upload status. - await db.execute(sql`DELETE FROM multipart_parts WHERE upload_id = ${uploadId}`); - await db.execute( - sql`UPDATE multipart_uploads SET status = 'aborted' WHERE upload_id = ${uploadId}`, - ); -}; - -export const insertMultipartPart = async ( - part: Omit, -): Promise => { - await db.execute( - sql`INSERT INTO multipart_parts (upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag) - VALUES (${part.uploadId}, ${part.partNumber}, ${part.telegramFileId}, ${part.telegramFileUniqueId}, ${part.storageMessageId}, ${part.sizeBytes}, ${part.etag})`, - ); -}; - -export const listMultipartParts = async (uploadId: string): Promise => { - const result = (await db.execute( - sql`SELECT id, upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag, created_at - FROM multipart_parts WHERE upload_id = ${uploadId} ORDER BY part_number`, - )) as unknown as Record[]; - return result.map((r) => ({ - id: r.id as number, - uploadId: r.upload_id as string, - partNumber: r.part_number as number, - telegramFileId: r.telegram_file_id as string, - telegramFileUniqueId: r.telegram_file_unique_id as string, - storageMessageId: r.storage_message_id as number, - sizeBytes: Number(r.size_bytes), - etag: r.etag as string, - createdAt: new Date(r.created_at as string), - })); -}; - -const mapRowToMultipartUpload = (r: Record): MultipartUpload => ({ - uploadId: r.upload_id as string, - bucketId: r.bucket_id as string, - s3Key: r.s3_key as string, - initiatedAt: new Date(r.initiated_at as string), - status: r.status as string, - initiatedBy: (r.initiated_by as string | null) || '', - contentType: (r.content_type as string | null) || null, -}); - -export const listMultipartUploadsByBucket = async ( - bucketId: string, - maxUploads: number, - keyMarker: string | null, -): Promise<{ uploads: MultipartUpload[]; isTruncated: boolean; nextKeyMarker: string | null }> => { - const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000); - const result = (await db.execute( - keyMarker - ? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by - FROM multipart_uploads - WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker} - ORDER BY s3_key, initiated_at - LIMIT ${limit + 1}` - : sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by - FROM multipart_uploads - WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' - ORDER BY s3_key, initiated_at - LIMIT ${limit + 1}`, - )) as unknown as Record[]; - - const uploads = result.slice(0, limit).map(mapRowToMultipartUpload); - return { - uploads, - isTruncated: result.length > limit, - nextKeyMarker: result.length > limit ? uploads.at(-1)?.s3Key || null : null, - }; -}; diff --git a/src/db/schema.ts b/src/db/schema.ts deleted file mode 100644 index 4e00666..0000000 --- a/src/db/schema.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { eq, type InferInsertModel, type InferSelectModel } from 'drizzle-orm'; -import { - bigint, - boolean, - integer, - pgTable, - serial, - text, - timestamp, - uniqueIndex, - uuid, -} from 'drizzle-orm/pg-core'; - -export const files = pgTable( - 'files', - { - id: uuid('id').primaryKey().defaultRandom(), - publicId: text('public_id').unique().notNull(), - telegramFileId: text('telegram_file_id').notNull(), - telegramFileUniqueId: text('telegram_file_unique_id').notNull(), - storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(), - storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(), - fileName: text('file_name').notNull(), - mimeType: text('mime_type').notNull(), - sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(), - fileType: text('file_type').notNull(), - uploaderId: bigint('uploader_id', { mode: 'number' }).notNull(), - fileHash: text('file_hash'), - archiveTelegramFileId: text('archive_telegram_file_id'), - archiveStorageMessageId: bigint('archive_storage_message_id', { mode: 'number' }), - archiveFileName: text('archive_file_name'), - archiveEntryName: text('archive_entry_name'), - archiveMimeType: text('archive_mime_type'), - archiveSizeBytes: bigint('archive_size_bytes', { mode: 'number' }), - bucketId: text('bucket_id'), - s3Key: text('s3_key'), - storageBackend: text('storage_backend').default('telegram'), - isDeleted: boolean('is_deleted').default(false), - multipartUploadId: text('multipart_upload_id'), - partCount: integer('part_count'), - createdAt: timestamp('created_at').defaultNow().notNull(), - updatedAt: timestamp('updated_at').defaultNow().notNull(), - }, - (table) => ({ - // H2: Prevent TOCTOU race on concurrent PUT — only one active (non-deleted) - // object per (bucket_id, s3_key) pair. - activeObjectIdx: uniqueIndex('active_object_idx') - .on(table.bucketId, table.s3Key) - .where(eq(table.isDeleted, false)), - }), -); - -export const fileParts = pgTable('file_parts', { - id: serial('id').primaryKey(), - fileId: uuid('file_id').notNull(), - partNumber: integer('part_number').notNull(), - telegramFileId: text('telegram_file_id').notNull(), - telegramFileUniqueId: text('telegram_file_unique_id').notNull(), - storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(), - storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(), - sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(), - storedSizeBytes: bigint('stored_size_bytes', { mode: 'number' }).notNull(), - compressionAlgorithm: text('compression_algorithm'), - etag: text('etag').notNull(), - createdAt: timestamp('created_at').defaultNow().notNull(), -}); - -export type File = InferSelectModel; -export type NewFile = InferInsertModel; -export type FilePart = InferSelectModel; -export type NewFilePart = InferInsertModel; diff --git a/src/env.ts b/src/env.ts index 1d2e662..8c82538 100644 --- a/src/env.ts +++ b/src/env.ts @@ -1,4 +1,4 @@ -import logger from './utils/logger'; +import logger from './shared/logger/index'; interface AppConfig { /** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */ diff --git a/src/index.ts b/src/index.ts index b072d14..79dd2a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,7 @@ import { metricsCollector } from './shared/metrics/index'; // ─── Auto-run migration at startup ────────────────────────────────────────── try { - const { runMigration } = await import('./db/migrate'); + const { runMigration } = await import('./infrastructure/persistence/drizzle/migrate'); await runMigration(); } catch { logger.warn('Auto-migration skipped (non-fatal)'); diff --git a/src/infrastructure/di.ts b/src/infrastructure/di.ts new file mode 100644 index 0000000..4216e87 --- /dev/null +++ b/src/infrastructure/di.ts @@ -0,0 +1,51 @@ +/** + * Dependency Injection container. + * + * Wires up singleton instances of all repositories and application services, + * making them available to controllers and other adapters without requiring + * a full DI framework. + * + * @module infrastructure/di + */ + +import type { IBucketRepository } from '../domain/ports/bucket-repository'; +import type { IFilePartRepository } from '../domain/ports/file-part-repository'; +import type { IFileRepository } from '../domain/ports/file-repository'; +import type { IMultipartRepository } from '../domain/ports/multipart-repository'; +import type { ITelegramService } from '../domain/ports/telegram-service'; +import { DrizzleBucketRepository } from './persistence/repositories/bucket-repository'; +import { DrizzleFilePartRepository } from './persistence/repositories/file-part-repository'; +import { DrizzleFileRepository } from './persistence/repositories/file-repository'; +import { DrizzleMultipartRepository } from './persistence/repositories/multipart-repository'; +import { botPool } from './telegram/bot-pool'; +import { ChunkedStorage } from './telegram/chunked-storage'; +import { UploadBatcher } from './telegram/upload-batcher'; + +// ─── Repository Singletons ────────────────────────────────────────── + +/** Singleton IFileRepository instance backed by Drizzle ORM. */ +export const fileRepository: IFileRepository = new DrizzleFileRepository(); + +/** Singleton IBucketRepository instance backed by Drizzle ORM. */ +export const bucketRepository: IBucketRepository = new DrizzleBucketRepository(); + +/** Singleton IFilePartRepository instance backed by Drizzle ORM. */ +export const filePartRepository: IFilePartRepository = new DrizzleFilePartRepository(); + +/** Singleton IMultipartRepository instance backed by Drizzle ORM. */ +export const multipartRepository: IMultipartRepository = new DrizzleMultipartRepository(); + +/** Singleton ITelegramService instance backed by the bot pool. */ +export const telegramService: ITelegramService = botPool; + +// ─── Service Singletons ───────────────────────────────────────────── + +/** Singleton ChunkedStorage for large file chunked uploads. */ +export const chunkedStorage = new ChunkedStorage( + fileRepository, + filePartRepository, + telegramService, +); + +/** Singleton UploadBatcher for batched small-file uploads. */ +export const uploadBatcher = new UploadBatcher(fileRepository, telegramService); diff --git a/src/interfaces/http/controllers/auth-controller.ts b/src/interfaces/http/controllers/auth-controller.ts index 7272767..eed6a1d 100644 --- a/src/interfaces/http/controllers/auth-controller.ts +++ b/src/interfaces/http/controllers/auth-controller.ts @@ -11,7 +11,7 @@ import { createSessionCookie, getAuthSession, isAuthEnabled, -} from '../../../utils/auth'; +} from '../middleware/auth'; /** * Helper that builds a JSON Response with optional extra headers. diff --git a/src/interfaces/http/controllers/file-controller.ts b/src/interfaces/http/controllers/file-controller.ts index f445808..50dc20a 100644 --- a/src/interfaces/http/controllers/file-controller.ts +++ b/src/interfaces/http/controllers/file-controller.ts @@ -2,11 +2,11 @@ import { createReadStream } from 'node:fs'; import { nanoid } from 'nanoid'; import type { TelegramFileInfo } from '../../../domain/ports/telegram-service'; import { fileInfoCache } from '../../../infrastructure/cache/index'; +import { chunkedStorage } from '../../../infrastructure/di'; import { botPool } from '../../../infrastructure/telegram/bot-pool'; import logger from '../../../shared/logger/index'; import { cleanupTempFile, formatCreatedAt, getErrorMessage } from '../../../shared/utils/file'; -import { createChunkedObjectResponse } from '../../../utils/chunked-storage'; -import { locateZipEntry } from '../../../utils/zip'; +import { locateZipEntry } from '../../../shared/utils/zip'; /** * Extended Request type that includes route parameter access. @@ -115,7 +115,7 @@ export const handleFileRedirect = async (req: RequestWithParams): Promise => { - const buckets = await listBuckets(); + const buckets = await bucketRepository.list(); const xml = listBucketsXml(buckets, reqId); return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; @@ -339,7 +325,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise => { - const bucket = await findBucketByName(bucketName); + const bucket = await bucketRepository.findByName(bucketName); if (!bucket) { return s3ErrorResponse( 'NoSuchBucket', @@ -384,7 +370,7 @@ const handleHeadBucket = async (bucketName: string, reqId: string): Promise => { - const bucket = await findBucketByName(bucketName); + const bucket = await bucketRepository.findByName(bucketName); if (!bucket) { return s3ErrorResponse( 'NoSuchBucket', @@ -394,7 +380,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise 0) { return s3ErrorResponse( 'BucketNotEmpty', @@ -404,7 +390,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise => { - const bucket = await findBucketByName(bucketName); + const bucket = await bucketRepository.findByName(bucketName); if (!bucket) { return s3ErrorResponse( 'NoSuchBucket', @@ -456,7 +442,7 @@ const handleGetObject = async ( headers: Record, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -466,7 +452,7 @@ const handleGetObject = async ( reqId, ); - const file = await findFileByBucketAndKey(bucketRecord.id, key); + const file = await fileRepository.findByBucketAndKey(bucketRecord.id, key); if (!file) return s3ErrorResponse( 'NoSuchKey', @@ -551,7 +537,7 @@ const handleGetObject = async ( ); } try { - return await createChunkedObjectResponse({ file, range, reqId }); + return await chunkedStorage.createChunkedObjectResponse({ file, range, reqId }); } catch (error) { logger.warn('Chunked object content fetch failed', { key, error: getErrorMessage(error) }); return s3ErrorResponse( @@ -638,14 +624,14 @@ const handleGetObject = async ( * @returns An S3 response streaming the assembled object content. */ const handleGetMultipartObject = async ( - file: File, + file: FileEntity, bucket: string, key: string, headers: Record, reqId: string, ): Promise => { const uploadId = file.multipartUploadId!; - const parts = await listMultipartParts(uploadId); + const parts = await multipartRepository.listParts(uploadId); if (parts.length === 0) { return s3ErrorResponse( @@ -724,7 +710,7 @@ const handleHeadObject = async ( headers: Record, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -734,7 +720,7 @@ const handleHeadObject = async ( reqId, ); - const file = await findFileByBucketAndKey(bucketRecord.id, key); + const file = await fileRepository.findByBucketAndKey(bucketRecord.id, key); if (!file) return s3ErrorResponse( 'NoSuchKey', @@ -930,7 +916,7 @@ const handlePutObject = async ( req: Request, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -1011,7 +997,7 @@ const handlePutObject = async ( // Idempotent PUT: if the object already exists, skip upload try { - const existing = await findFileByBucketAndKey(bucketRecord.id, key); + const existing = await fileRepository.findByBucketAndKey(bucketRecord.id, key); if (existing) { await cleanupTempFile(streamed.tempPath); return s3Response(null, 200, reqId, { etag: `"${streamed.fileHash}"` }); @@ -1057,7 +1043,7 @@ const storeFileFromTemp = async ( const partFileNamePrefix = `s3-${bucketRecord.name}-${key.replace(/\//g, '_')}`; if (streamed.sizeBytes > config.telegramChunkSizeBytes) { - const file = await storeFileInTelegramChunks({ + const file = await chunkedStorage.storeFileInTelegramChunks({ tempPath: streamed.tempPath, partFileNamePrefix, fileName: finalFileName, @@ -1138,7 +1124,7 @@ const handleCopyObject = async ( const sourceBucket = parts[0]; const sourceKey = parts.slice(1).join('/'); - const sourceBucketRecord = await findBucketByName(sourceBucket); + const sourceBucketRecord = await bucketRepository.findByName(sourceBucket); if (!sourceBucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -1148,7 +1134,7 @@ const handleCopyObject = async ( reqId, ); - const sourceFile = await findFileByBucketAndKey(sourceBucketRecord.id, sourceKey); + const sourceFile = await fileRepository.findByBucketAndKey(sourceBucketRecord.id, sourceKey); if (!sourceFile) return s3ErrorResponse( 'NoSuchKey', @@ -1232,7 +1218,7 @@ const handleDeleteObject = async ( key: string, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -1242,7 +1228,7 @@ const handleDeleteObject = async ( reqId, ); - await softDeleteFile(bucketRecord.id, key); + await fileRepository.softDelete(bucketRecord.id, key); return s3Response(null, 204, reqId); }; @@ -1262,7 +1248,7 @@ const handleDeleteObjects = async ( body: string, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -1288,7 +1274,7 @@ const handleDeleteObjects = async ( const deletedKeys: string[] = []; const errors: Array<{ key: string; code: string; message: string }> = []; for (const key of keys) { - const ok = await softDeleteFile(bucketRecord.id, key); + const ok = await fileRepository.softDelete(bucketRecord.id, key); if (ok) { deletedKeys.push(key); } else { @@ -1316,7 +1302,7 @@ const handleListObjectsV1 = async ( searchParams: URLSearchParams, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -1335,7 +1321,7 @@ const handleListObjectsV1 = async ( const marker = searchParams.get('marker') || null; const encodingType = searchParams.get('encoding-type') || null; - const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix( + const { objects, prefixes: commonPrefixes } = await fileRepository.listByPrefix( bucketRecord.id, prefix, delimiter, @@ -1386,7 +1372,7 @@ const handleListObjectsV2 = async ( searchParams: URLSearchParams, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -1403,7 +1389,7 @@ const handleListObjectsV2 = async ( const startAfter = searchParams.get('start-after') || null; const encodingType = searchParams.get('encoding-type') || null; - const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix( + const { objects, prefixes: commonPrefixes } = await fileRepository.listByPrefix( bucketRecord.id, prefix, delimiter, @@ -1459,7 +1445,7 @@ const handleCreateMultipartUpload = async ( headers: Record, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -1470,7 +1456,7 @@ const handleCreateMultipartUpload = async ( ); const contentType = headers['content-type'] || null; - const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3', contentType); + const uploadId = await multipartRepository.create(bucketRecord.id, key, 's3', contentType); const xml = initiateMultipartUploadXml(bucket, key, uploadId); return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); @@ -1514,7 +1500,7 @@ const handleUploadPart = async ( ); } - const multipart = await findMultipartUpload(uploadId); + const multipart = await multipartRepository.findById(uploadId); if (!multipart || multipart.s3Key !== key) { return s3ErrorResponse( 'NoSuchUpload', @@ -1583,7 +1569,7 @@ const handleUploadPart = async ( await cleanupTempFile(tempPath); const etag = hasher.digest('hex'); - await insertMultipartPart({ + await multipartRepository.insertPart({ uploadId, partNumber, telegramFileId: forwardResult.telegramFileId, @@ -1617,7 +1603,7 @@ const handleCompleteMultipartUpload = async ( reqId: string, ): Promise => { const uploadId = searchParams.get('uploadId')!; - const multipart = await findMultipartUpload(uploadId); + const multipart = await multipartRepository.findById(uploadId); // H5: Verify both upload exists AND key matches (consistent with handleUploadPart) if (!multipart || multipart.s3Key !== key) { return s3ErrorResponse( @@ -1630,7 +1616,7 @@ const handleCompleteMultipartUpload = async ( } const parts = parseCompleteMultipartBody(body); - const storedParts = await listMultipartParts(uploadId); + const storedParts = await multipartRepository.listParts(uploadId); // Validate ascending part order const partNumbers = parts.map((p) => p.partNumber); @@ -1702,7 +1688,7 @@ const handleCompleteMultipartUpload = async ( updatedAt: new Date(), }); - await completeMultipartUpload(uploadId); + await multipartRepository.complete(uploadId); const location = `${config.baseUrl}/${bucket}/${key}`; const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location); @@ -1723,7 +1709,7 @@ const handleListMultipartUploads = async ( searchParams: URLSearchParams, reqId: string, ): Promise => { - const bucketRecord = await findBucketByName(bucket); + const bucketRecord = await bucketRepository.findByName(bucket); if (!bucketRecord) return s3ErrorResponse( 'NoSuchBucket', @@ -1735,7 +1721,7 @@ const handleListMultipartUploads = async ( const maxUploads = Math.min(Number.parseInt(searchParams.get('max-uploads') || '1000', 10), 1000); const keyMarker = searchParams.get('key-marker') || null; - const { uploads, isTruncated, nextKeyMarker } = await listMultipartUploadsByBucket( + const { uploads, isTruncated, nextKeyMarker } = await multipartRepository.listByBucket( bucketRecord.id, maxUploads, keyMarker, @@ -1774,7 +1760,7 @@ const handleAbortMultipartUpload = async ( reqId: string, ): Promise => { const uploadId = searchParams.get('uploadId')!; - const multipart = await findMultipartUpload(uploadId); + const multipart = await multipartRepository.findById(uploadId); if (!multipart) { return s3ErrorResponse( 'NoSuchUpload', @@ -1785,7 +1771,7 @@ const handleAbortMultipartUpload = async ( ); } - await abortMultipartUpload(uploadId); + await multipartRepository.abort(uploadId); return s3Response(null, 204, reqId); }; @@ -1807,7 +1793,7 @@ const handleListParts = async ( reqId: string, ): Promise => { const uploadId = searchParams.get('uploadId')!; - const multipart = await findMultipartUpload(uploadId); + const multipart = await multipartRepository.findById(uploadId); if (!multipart) { return s3ErrorResponse( 'NoSuchUpload', @@ -1818,7 +1804,7 @@ const handleListParts = async ( ); } - const parts = await listMultipartParts(uploadId); + const parts = await multipartRepository.listParts(uploadId); const maxParts = Math.min(Number.parseInt(searchParams.get('max-parts') || '1000', 10), 1000); const xml = listPartsXml( diff --git a/src/interfaces/http/controllers/upload-controller.ts b/src/interfaces/http/controllers/upload-controller.ts index 99803e8..d5caae5 100644 --- a/src/interfaces/http/controllers/upload-controller.ts +++ b/src/interfaces/http/controllers/upload-controller.ts @@ -1,7 +1,8 @@ import { createWriteStream } from 'node:fs'; import { nanoid } from 'nanoid'; import { config } from '../../../config/index'; -import { findFileByHash } from '../../../db/files'; +import { chunkedStorage, fileRepository, uploadBatcher } from '../../../infrastructure/di'; +import type { PreparedUpload } from '../../../infrastructure/telegram/upload-batcher'; import logger from '../../../shared/logger/index'; import { metricsCollector } from '../../../shared/metrics/index'; import { @@ -14,8 +15,6 @@ import { getErrorMessage, getFileType, } from '../../../shared/utils/file'; -import { storeFileInTelegramChunks } from '../../../utils/chunked-storage'; -import { enqueuePreparedUpload, type PreparedUpload } from '../../../utils/uploadBatcher'; /** * Maximum allowed size (in bytes) for a base64 JSON upload. @@ -223,7 +222,7 @@ const handleMultipartUpload = async (req: Request): Promise => { const prepared = await streamFileToTemp(file, config.maxRequestBodyBytes); - const existingFile = await findFileByHash(prepared.fileHash); + const existingFile = await fileRepository.findByHash(prepared.fileHash); if (existingFile) { await cleanupTempFile(prepared.tempPath); return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 }); @@ -243,7 +242,7 @@ const handleMultipartUpload = async (req: Request): Promise => { } if (prepared.sizeBytes > config.telegramChunkSizeBytes) { - const uploadedFile = await storeFileInTelegramChunks({ + const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({ tempPath: prepared.tempPath, partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'upload'}`, fileName: finalFileName, @@ -256,7 +255,7 @@ const handleMultipartUpload = async (req: Request): Promise => { return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 }); } - const uploaded = await enqueuePreparedUpload({ + const uploaded = await uploadBatcher.enqueuePreparedUpload({ prepared, fileName: finalFileName, mimeType, @@ -317,7 +316,7 @@ const handleJSONUpload = async (req: Request): Promise => { const fileBytes = Buffer.from(base64Data, 'base64'); const hash = computeHash(fileBytes); - const existingFile = await findFileByHash(hash); + const existingFile = await fileRepository.findByHash(hash); if (existingFile) { return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 }); } @@ -334,7 +333,7 @@ const handleJSONUpload = async (req: Request): Promise => { const prepared = await writeBufferToTemp(fileBytes, hash); if (prepared.sizeBytes > config.telegramChunkSizeBytes) { - const uploadedFile = await storeFileInTelegramChunks({ + const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({ tempPath: prepared.tempPath, partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'json'}`, fileName: finalFileName, @@ -347,7 +346,7 @@ const handleJSONUpload = async (req: Request): Promise => { return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 }); } - const uploaded = await enqueuePreparedUpload({ + const uploaded = await uploadBatcher.enqueuePreparedUpload({ prepared, fileName: finalFileName, mimeType, diff --git a/src/interfaces/http/controllers/web-api-controller.ts b/src/interfaces/http/controllers/web-api-controller.ts index a69824d..4264093 100644 --- a/src/interfaces/http/controllers/web-api-controller.ts +++ b/src/interfaces/http/controllers/web-api-controller.ts @@ -1,21 +1,11 @@ import { createReadStream } from 'node:fs'; import { nanoid } from 'nanoid'; import { config } from '../../../config/index'; -import { createBucket, deleteBucket, findBucketByName, listBuckets } from '../../../db/buckets'; -import { - countBucketObjects, - findFileByBucketAndKey, - listObjectsByPrefix, - softDeleteFile, -} from '../../../db/files-ext'; -import { db, files as fileSchema } from '../../../db/index'; +import { bucketRepository, chunkedStorage, fileRepository } from '../../../infrastructure/di'; +import { db, files as fileSchema } from '../../../infrastructure/persistence/drizzle/index'; import { botPool } from '../../../infrastructure/telegram/bot-pool'; import logger from '../../../shared/logger/index'; import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file'; -import { - createChunkedObjectResponse, - storeFileInTelegramChunks, -} from '../../../utils/chunked-storage'; /** * Route parameters extracted from the URL path. @@ -48,13 +38,13 @@ const jsonError = (error: string, status: number): Response => Response.json({ e * @returns A JSON response with the bucket list. */ export const handleListBucketsV1 = async (): Promise => { - const buckets = await listBuckets(); + const buckets = await bucketRepository.list(); const result = await Promise.all( buckets.map(async (b) => ({ id: b.id, name: b.name, createdAt: b.createdAt.toISOString(), - objectCount: await countBucketObjects(b.id), + objectCount: await fileRepository.countByBucket(b.id), })), ); return json({ buckets: result }); @@ -73,9 +63,9 @@ export const handleCreateBucketV1 = async (req: Request): Promise => { if (!body.name || !/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(body.name)) { return jsonError('Invalid bucket name. Use lowercase, 3-63 chars, no underscore', 400); } - const existing = await findBucketByName(body.name); + const existing = await bucketRepository.findByName(body.name); if (existing) return jsonError('Bucket already exists', 409); - const bucket = await createBucket(body.name); + const bucket = await bucketRepository.create(body.name); return json({ id: bucket.id, name: bucket.name }, 201); }; @@ -92,11 +82,11 @@ export const handleDeleteBucketV1 = async ( _req: Request, params: RouteParams, ): Promise => { - const bucket = await findBucketByName(params.bucket!); + const bucket = await bucketRepository.findByName(params.bucket!); if (!bucket) return jsonError('Bucket not found', 404); - const count = await countBucketObjects(bucket.id); + const count = await fileRepository.countByBucket(bucket.id); if (count > 0) return jsonError('Bucket is not empty', 409); - await deleteBucket(params.bucket!); + await bucketRepository.delete(params.bucket!); return json({ success: true }); }; @@ -110,7 +100,7 @@ export const handleDeleteBucketV1 = async ( * @returns A JSON response with the object list. */ export const handleListObjectsV1 = async (req: Request, params: RouteParams): Promise => { - const bucket = await findBucketByName(params.bucket!); + const bucket = await bucketRepository.findByName(params.bucket!); if (!bucket) return jsonError('Bucket not found', 404); const url = new URL(req.url); @@ -119,7 +109,7 @@ export const handleListObjectsV1 = async (req: Request, params: RouteParams): Pr const maxKeys = Number.parseInt(url.searchParams.get('max-keys') || '1000', 10); const continuationToken = url.searchParams.get('continuation-token') || null; - const { objects, prefixes } = await listObjectsByPrefix( + const { objects, prefixes } = await fileRepository.listByPrefix( bucket.id, prefix, delimiter, @@ -162,7 +152,7 @@ export const handleUploadObjectV1 = async ( req: Request, params: RouteParams, ): Promise => { - const bucket = await findBucketByName(params.bucket!); + const bucket = await bucketRepository.findByName(params.bucket!); if (!bucket) return jsonError('Bucket not found', 404); const formData = await req.formData(); @@ -217,7 +207,7 @@ export const handleUploadObjectV1 = async ( const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`; if (sizeBytes > config.telegramChunkSizeBytes) { - const uploadedFile = await storeFileInTelegramChunks({ + const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({ tempPath, partFileNamePrefix, fileName: finalFileName, @@ -287,9 +277,9 @@ export const handleDeleteObjectV1 = async ( _req: Request, params: RouteParams, ): Promise => { - const bucket = await findBucketByName(params.bucket!); + const bucket = await bucketRepository.findByName(params.bucket!); if (!bucket) return jsonError('Bucket not found', 404); - await softDeleteFile(bucket.id, params.key!); + await fileRepository.softDelete(bucket.id, params.key!); return json({ success: true }); }; @@ -307,15 +297,15 @@ export const handleDownloadObjectV1 = async ( _req: Request, params: RouteParams, ): Promise => { - const bucket = await findBucketByName(params.bucket!); + const bucket = await bucketRepository.findByName(params.bucket!); if (!bucket) return jsonError('Bucket not found', 404); - const file = await findFileByBucketAndKey(bucket.id, params.key!); + const file = await fileRepository.findByBucketAndKey(bucket.id, params.key!); if (!file) return jsonError('Object not found', 404); if (file.storageBackend === 'chunked') { const range = { type: 'none' as const }; - return createChunkedObjectResponse({ file, range, reqId: '' }); + return chunkedStorage.createChunkedObjectResponse({ file, range, reqId: '' }); } const fileInfo = await botPool.getFileInfo(file.telegramFileId); @@ -348,12 +338,12 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro } const destBucketName = body.destBucket || params.bucket!; - const sourceBucket = await findBucketByName(params.bucket!); - const destBucket = await findBucketByName(destBucketName); + const sourceBucket = await bucketRepository.findByName(params.bucket!); + const destBucket = await bucketRepository.findByName(destBucketName); if (!sourceBucket || !destBucket) return jsonError('Bucket not found', 404); - const sourceFile = await findFileByBucketAndKey(sourceBucket.id, body.sourceKey); + const sourceFile = await fileRepository.findByBucketAndKey(sourceBucket.id, body.sourceKey); if (!sourceFile) return jsonError('Source object not found', 404); if (sourceFile.storageBackend === 'chunked') { diff --git a/src/interfaces/http/middleware/rate-limit.ts b/src/interfaces/http/middleware/rate-limit.ts index 2a37719..84332dd 100644 --- a/src/interfaces/http/middleware/rate-limit.ts +++ b/src/interfaces/http/middleware/rate-limit.ts @@ -1,7 +1,89 @@ -export { - checkRateLimit, - cleanupRateLimitCache, - clearRateLimitCache, - getRateLimitStats, - withRateLimit, -} from '../../../utils/rateLimit'; +import { config } from '../../../config/index'; +import logger from '../../../shared/logger/index'; +import { extractClientIp } from '../../../shared/utils/ip'; + +interface RateLimitEntry { + count: number; + resetTime: number; +} + +const rateLimitStore = new Map(); +const MAX_STORE_ENTRIES = 50000; + +const evictExpiredEntries = (now = Date.now()): number => { + let cleaned = 0; + + for (const [key, entry] of rateLimitStore.entries()) { + if (now > entry.resetTime) { + rateLimitStore.delete(key); + cleaned++; + } + } + + return cleaned; +}; + +const ensureStoreCapacity = (now: number): void => { + if (rateLimitStore.size < MAX_STORE_ENTRIES) return; + + evictExpiredEntries(now); + while (rateLimitStore.size >= MAX_STORE_ENTRIES) { + const oldestKey = rateLimitStore.keys().next().value; + if (!oldestKey) break; + rateLimitStore.delete(oldestKey); + } +}; + +export const checkRateLimit = (key: string): boolean => { + const now = Date.now(); + const entry = rateLimitStore.get(key); + + if (!entry || now > entry.resetTime) { + ensureStoreCapacity(now); + rateLimitStore.set(key, { + count: 1, + resetTime: now + config.rateLimitWindowMs, + }); + return true; + } + + if (entry.count >= config.rateLimitMaxRequests) { + logger.warn('Rate limit exceeded', { key, count: entry.count }); + return false; + } + + entry.count++; + return true; +}; + +export const withRateLimit = ( + handler: (req: T) => Promise, +): ((req: T) => Promise) => { + return async (req: T): Promise => { + const ip = extractClientIp(req); + if (!checkRateLimit(ip)) { + return Response.json({ error: 'Rate limit exceeded' }, { status: 429 }); + } + + return handler(req); + }; +}; + +export const cleanupRateLimitCache = (): void => { + const cleaned = evictExpiredEntries(); + + if (cleaned > 0) { + logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size }); + } +}; + +export const getRateLimitStats = () => ({ + trackedIPs: rateLimitStore.size, + windowSize: config.rateLimitWindowMs, + maxRequests: config.rateLimitMaxRequests, + maxTrackedIPs: MAX_STORE_ENTRIES, +}); + +export const clearRateLimitCache = (): void => { + rateLimitStore.clear(); +}; diff --git a/src/interfaces/http/routes/index.ts b/src/interfaces/http/routes/index.ts index f70751e..02f2a2b 100644 --- a/src/interfaces/http/routes/index.ts +++ b/src/interfaces/http/routes/index.ts @@ -1,7 +1,7 @@ import { config } from '../../../config/index'; import { handleSwaggerHtml, handleSwaggerJson } from '../../../routes/swagger'; -import { extractS3BucketFromHost } from '../../../utils/s3/virtual-host'; import { isS3Request } from '../../s3/auth'; +import { extractS3BucketFromHost } from '../../s3/virtual-host'; import { handleLogin, handleLogout, handleMe } from '../controllers/auth-controller'; import { handleFileInfo, handleFileRedirect } from '../controllers/file-controller'; import { handleHealth } from '../controllers/health-controller'; diff --git a/src/interfaces/s3/auth.ts b/src/interfaces/s3/auth.ts index f1dae39..720b3ed 100644 --- a/src/interfaces/s3/auth.ts +++ b/src/interfaces/s3/auth.ts @@ -1,7 +1,463 @@ -export type { SigV4Result, VerifyPresignedUrlInput } from '../../utils/s3/auth'; -export { - buildCanonicalQueryString, - isS3Request, - verifyPresignedUrl, - verifySignature, -} from '../../utils/s3/auth'; +import { timingSafeEqual } from 'node:crypto'; + +/** + * Timing-safe string comparison that prevents timing attacks. + * + * Uses `crypto.timingSafeEqual` which runs in constant time regardless of + * where the strings differ. Returns false for mismatched-length inputs + * to avoid leaking length information via early return. + * + * @param left - The first string to compare. + * @param right - The second string to compare. + * @returns True if both strings are equal. + */ +const timingSafeCompare = (left: string, right: string): boolean => { + const leftBuffer = Buffer.from(left); + const rightBuffer = Buffer.from(right); + + if (leftBuffer.length !== rightBuffer.length) { + return false; + } + + return timingSafeEqual(leftBuffer, rightBuffer); +}; + +export interface SigV4Result { + isValid: boolean; + credential: { + accessKey: string; + date: string; + region: string; + service: string; + } | null; + errorCode?: string; +} + +export interface VerifyPresignedUrlInput { + url: string; + method: string; + headers: Record; + s3AccessKey: string; + s3SecretKey: string; + region: string; + now?: Date; +} + +const SERVICE = 's3'; +const TERMINATION = 'aws4_request'; + +/** + * Maximum acceptable clock skew between client and server for header-based + * SigV4 authentication. AWS allows 15 minutes. + */ +const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => { + if (data instanceof Uint8Array) return data; + if (data instanceof ArrayBuffer) return new Uint8Array(data); + return new TextEncoder().encode(data); +}; + +const sha256Hex = async (data: string | Uint8Array | ArrayBuffer): Promise => { + const hashBuffer = await crypto.subtle.digest('SHA-256', buf(data) as never); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + return hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); +}; + +const hmacSha256 = async (key: Uint8Array, message: string): Promise => { + const cryptoKey = await crypto.subtle.importKey( + 'raw', + key as never, + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'], + ); + const result = await crypto.subtle.sign('HMAC', cryptoKey, buf(message) as never); + return new Uint8Array(result); +}; + +const getSigningKey = async ( + secretKey: string, + dateStamp: string, + region: string, +): Promise => { + let key = await hmacSha256(buf(`AWS4${secretKey}`), dateStamp); + key = await hmacSha256(key, region); + key = await hmacSha256(key, SERVICE); + return await hmacSha256(key, TERMINATION); +}; + +const hmacHex = async (key: Uint8Array, message: string): Promise => { + const result = await hmacSha256(key, message); + return Array.from(result) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); +}; + +const parseAuthorizationHeader = (authHeader: string) => { + const credentialMatch = authHeader.match(/Credential=([^,]+)/); + const signedHeadersMatch = authHeader.match(/SignedHeaders=([^,]+)/); + const signatureMatch = authHeader.match(/Signature=([^,]+)/); + + if (!credentialMatch || !signedHeadersMatch || !signatureMatch) return null; + + const credentialParts = credentialMatch[1].split('/'); + if (credentialParts.length !== 5) return null; + + return { + accessKey: credentialParts[0], + date: credentialParts[1], + region: credentialParts[2], + service: credentialParts[3], + termination: credentialParts[4], + signedHeaders: signedHeadersMatch[1], + signature: signatureMatch[1], + }; +}; + +const buildCanonicalRequest = ( + method: string, + canonicalUri: string, + canonicalQueryString: string, + signedHeaders: string, + headers: Record, + hashedPayload: string, +): string => { + const canonicalHeaders = signedHeaders + .split(';') + .map((h) => { + const value = headers[h.toLowerCase()] || ''; + return `${h.toLowerCase()}:${value.trim()}\n`; + }) + .join(''); + + return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`; +}; + +/** + * Normalizes a URI per AWS SigV4 requirements plus RFC 3986: + * + * 1. Decode percent-encoded characters + * 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4 + * + * @param uri - The raw URI path to normalize. + * @returns The normalized URI path. + */ +const normalizeUri = (uri: string): string => { + if (!uri || uri === '') return '/'; + + // AWS SigV4 requires URI-decoded paths in the canonical request + // Only `.` and `..` segments are removed per RFC 3986 section 5.2.4 + // Empty segments (from `//` or trailing `/`) are preserved — they are + // part of the URI and the SDK signs them. + const decoded = decodeURIComponent(uri); + const segments = decoded.split('/'); + const result: string[] = []; + + for (const segment of segments) { + if (segment === '.') continue; + if (segment === '..') { + result.pop(); + continue; + } + result.push(segment); + } + + // Join preserves empty first segment (from leading /) automatically + return result.join('/') || '/'; +}; + +const awsEncode = (value: string): string => + encodeURIComponent(value).replace( + /[!'()*]/g, + (ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`, + ); + +export const buildCanonicalQueryString = ( + searchParams: URLSearchParams, + excludeKeys: Set = new Set(), +): string => { + const pairs: Array<[string, string]> = []; + for (const [key, value] of searchParams.entries()) { + if (!excludeKeys.has(key)) pairs.push([key, value]); + } + // AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare + pairs.sort(([ak, av], [bk, bv]) => { + const a = `${awsEncode(ak)}=${awsEncode(av)}`; + const b = `${awsEncode(bk)}=${awsEncode(bv)}`; + if (a < b) return -1; + if (a > b) return 1; + return 0; + }); + return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&'); +}; + +const getHashedPayload = async (body: string | null): Promise => { + if (!body || body.length === 0) return await sha256Hex(''); + return await sha256Hex(body); +}; + +/** + * Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date. + * + * @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format. + * @returns The parsed Date, or null if the format is invalid. + */ +const parseAmzDateUtc = (amzDate: string): Date | null => { + const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/); + if (!match) return null; + const [, year, month, day, hour, minute, second] = match; + return new Date( + Date.UTC( + Number.parseInt(year, 10), + Number.parseInt(month, 10) - 1, + Number.parseInt(day, 10), + Number.parseInt(hour, 10), + Number.parseInt(minute, 10), + Number.parseInt(second, 10), + ), + ); +}; + +/** + * Validates that `host` is included in the signed headers list. + * + * AWS SigV4 mandates that `host` is always signed. Reject requests that + * omit it to prevent header injection / replay variants. + * + * @param signedHeaders - The semicolon-separated signed headers string. + * @returns True if `host` is present. + */ +const validateSignedHeaders = (signedHeaders: string): boolean => { + return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host'); +}; + +export const verifySignature = async ( + method: string, + url: string, + headers: Record, + body: string | null, + s3AccessKey: string, + s3SecretKey: string, + region: string, +): Promise => { + const authHeader = headers.authorization; + if (!authHeader?.startsWith('AWS4-HMAC-SHA256')) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + + const parsed = parseAuthorizationHeader(authHeader); + if (!parsed) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + + if (!timingSafeCompare(parsed.accessKey, s3AccessKey)) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + + if (!timingSafeCompare(parsed.region, region)) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + + // Validate service and termination in credential scope (M2) + if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + + // Validate host is in signed headers (LOW/host) + if (!validateSignedHeaders(parsed.signedHeaders)) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + + const parsedUrl = new URL(url, 'http://localhost'); + const canonicalUri = normalizeUri(parsedUrl.pathname); + const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams); + + const contentSha256 = headers['x-amz-content-sha256'] || null; + if (contentSha256?.startsWith('STREAMING-')) { + return { isValid: false, credential: null, errorCode: 'NotImplemented' }; + } + + // CRITICAL: Use the x-amz-content-sha256 header value in the canonical + // request because that's what the client signed. The actual body hash is + // verified by verifyBodyHash() after streaming, ensuring integrity without + // breaking SigV4. + const hashedPayload = contentSha256 || (await getHashedPayload(body)); + + const canonicalRequest = buildCanonicalRequest( + method, + canonicalUri, + canonicalQueryString, + parsed.signedHeaders, + headers, + hashedPayload, + ); + + const hashedCanonicalRequest = await sha256Hex(canonicalRequest); + + // M1: Fall back to Date header if x-amz-date is missing + const amzDate = headers['x-amz-date'] || headers.date || ''; + + // H5: Validate request freshness (clock skew / replay protection) + if (amzDate) { + const requestDate = parseAmzDateUtc(amzDate); + if (requestDate) { + const now = Date.now(); + const skew = Math.abs(now - requestDate.getTime()); + if (skew > MAX_CLOCK_SKEW_MS) { + return { isValid: false, credential: null, errorCode: 'RequestExpired' }; + } + } + } + + const dateStamp = parsed.date; + + // M3: Ensure date in credential scope matches x-amz-date + if (amzDate) { + const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD" + if (amzDateStamp !== dateStamp) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + } + + const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`; + + const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`; + + const signingKey = await getSigningKey(s3SecretKey, dateStamp, region); + const expectedSignature = await hmacHex(signingKey, stringToSign); + + if (!timingSafeCompare(expectedSignature, parsed.signature)) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + + return { + isValid: true, + credential: { + accessKey: parsed.accessKey, + date: parsed.date, + region: parsed.region, + service: parsed.service, + }, + }; +}; + +export const verifyPresignedUrl = async ({ + url, + method, + headers, + s3AccessKey, + s3SecretKey, + region, + now = new Date(), +}: VerifyPresignedUrlInput): Promise => { + const parsedUrl = new URL(url); + const searchParams = parsedUrl.searchParams; + + const algorithm = searchParams.get('X-Amz-Algorithm'); + const credential = searchParams.get('X-Amz-Credential'); + const signedHeaders = searchParams.get('X-Amz-SignedHeaders'); + const signature = searchParams.get('X-Amz-Signature'); + const expiresText = searchParams.get('X-Amz-Expires'); + const amzDate = searchParams.get('X-Amz-Date'); + + if ( + algorithm !== 'AWS4-HMAC-SHA256' || + !credential || + !signedHeaders || + !signature || + !expiresText || + !amzDate + ) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + + const expires = Number.parseInt(expiresText, 10); + const signedAt = parseAmzDateUtc(amzDate); + if (!Number.isFinite(expires) || expires <= 0 || !signedAt) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + // AWS S3 spec limits presigned URLs to 7 days (604800 seconds) + const MAX_PRESIGNED_EXPIRY_SECONDS = 604800; + if ( + now.getTime() > signedAt.getTime() + expires * 1000 || + expires > MAX_PRESIGNED_EXPIRY_SECONDS + ) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + + const credParts = credential.split('/'); + if (credParts.length !== 5) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + const [accessKey, dateStamp, credentialRegion, service, termination] = credParts; + if ( + !timingSafeCompare(accessKey, s3AccessKey) || + !timingSafeCompare(credentialRegion, region) || + service !== SERVICE || + termination !== TERMINATION + ) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + + // Validate host is in signed headers for presigned URLs too + if (!validateSignedHeaders(signedHeaders)) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + + const signedHeaderList = signedHeaders.split(';').filter(Boolean); + const canonicalHeaders = signedHeaderList + .map((headerName) => { + const lower = headerName.toLowerCase(); + const value = lower === 'host' ? headers.host || parsedUrl.host : headers[lower] || ''; + return `${lower}:${value.trim()}\n`; + }) + .join(''); + + const canonicalRequest = `${method}\n${normalizeUri(parsedUrl.pathname)}\n${buildCanonicalQueryString(searchParams, new Set(['X-Amz-Signature']))}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`; + const hashedCanonicalRequest = await sha256Hex(canonicalRequest); + const credentialScope = `${dateStamp}/${region}/${SERVICE}/${TERMINATION}`; + const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`; + const expectedSignature = await hmacHex( + await getSigningKey(s3SecretKey, dateStamp, region), + stringToSign, + ); + + if (!timingSafeCompare(expectedSignature, signature)) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + return { isValid: true, credential: { accessKey, date: dateStamp, region, service } }; +}; + +export const isS3Request = (headers: Record): boolean => { + const auth = headers.authorization || ''; + return auth.startsWith('AWS4-HMAC-SHA256'); +}; + +/** + * Verifies that the actual body SHA-256 matches the `x-amz-content-sha256` + * header from the original request. + * + * This MUST be called AFTER the body has been fully streamed and hashed, + * as a second pass after `verifySignature` (which cannot hash a streaming + * body without consuming it). + * + * @param bodySha256 - The SHA-256 hex digest of the actual body content. + * @param headers - The original request headers. + * @returns An error result on mismatch, or null if the check passes. + */ +export const verifyBodyHash = ( + bodySha256: string, + headers: Record, +): SigV4Result | null => { + const claimedHash = headers['x-amz-content-sha256']; + // If the client sent UNSIGNED-PAYLOAD, skip verification + if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) { + return null; + } + if (claimedHash !== bodySha256) { + return { isValid: false, credential: null, errorCode: 'BadDigest' }; + } + return null; +}; diff --git a/src/interfaces/s3/headers.ts b/src/interfaces/s3/headers.ts index 7b18674..bd53bac 100644 --- a/src/interfaces/s3/headers.ts +++ b/src/interfaces/s3/headers.ts @@ -1,10 +1,52 @@ -/** - * Re-export from the canonical headers implementation. - * - * @module - */ -export { - applyS3Headers, - S3_CORS_HEADERS, - s3Headers, -} from '../../utils/s3/headers'; +import { nanoid } from 'nanoid'; + +export const S3_CORS_HEADERS: Record = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS', + 'access-control-allow-headers': [ + 'Authorization', + 'Content-Type', + 'Content-MD5', + 'Range', + 'If-Match', + 'If-None-Match', + 'If-Modified-Since', + 'If-Unmodified-Since', + 'X-Amz-*', + 'x-amz-*', + ].join(', '), + 'access-control-expose-headers': [ + 'Accept-Ranges', + 'Content-Length', + 'Content-Range', + 'Content-Type', + 'ETag', + 'Last-Modified', + 'x-amz-id-2', + 'x-amz-request-id', + ].join(', '), + 'access-control-max-age': '86400', +}; + +export const s3Headers = ( + requestId: string, + extraHeaders: Record = {}, +): Record => ({ + ...S3_CORS_HEADERS, + server: 'AmazonS3', + ...(requestId + ? { + 'x-amz-request-id': requestId, + 'x-amz-id-2': `${requestId}+${nanoid(16)}`, + } + : {}), + ...extraHeaders, +}); + +export const applyS3Headers = (headers: Headers, requestId: string): Headers => { + const result = new Headers(headers); + for (const [key, value] of Object.entries(s3Headers(requestId))) { + result.set(key, value); + } + return result; +}; diff --git a/src/interfaces/s3/virtual-host.ts b/src/interfaces/s3/virtual-host.ts index ee8cb2a..4cf6c76 100644 --- a/src/interfaces/s3/virtual-host.ts +++ b/src/interfaces/s3/virtual-host.ts @@ -1 +1,27 @@ -export { extractS3BucketFromHost } from '../../utils/s3/virtual-host'; +const stripPort = (host: string): string => { + // Handle IPv6: [::1]:8080 -> [::1] + if (host.startsWith('[')) { + const closeBracket = host.indexOf(']'); + return host.slice(0, closeBracket + 1).toLowerCase(); + } + return host.split(':')[0].toLowerCase().replace(/\.$/, ''); +}; + +const isValidBucketLabel = (bucket: string): boolean => + /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) && + !bucket.includes('..') && + !bucket.includes('.-') && + !bucket.includes('-.'); + +export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => { + const normalizedHost = stripPort(host); + for (const domain of domains) { + const normalizedDomain = stripPort(domain); + if (!normalizedDomain || normalizedHost === normalizedDomain) continue; + if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue; + + const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1)); + return isValidBucketLabel(bucket) ? bucket : null; + } + return null; +}; diff --git a/src/interfaces/s3/xml.ts b/src/interfaces/s3/xml.ts index 346bfe3..cd255a7 100644 --- a/src/interfaces/s3/xml.ts +++ b/src/interfaces/s3/xml.ts @@ -1,22 +1,311 @@ -/** - * Re-export from the canonical XML implementation. - * - * @module - */ -export { - bucketVersioningConfigurationXml, - type CompletePart, - completeMultipartUploadXml, - copyObjectResultXml, - deleteResultXml, - initiateMultipartUploadXml, - listBucketResultXml, - listBucketsXml, - listBucketV2ResultXml, - listMultipartUploadsXml, - listPartsXml, - parseCompleteMultipartBody, - parseDeleteObjectsBody, - s3ErrorResponse, - s3ErrorXml, -} from '../../utils/s3/xml'; +import { s3Headers } from './headers'; + +const escapeXml = (str: string): string => + str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + +const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z'); + +const encodeKey = (value: string, encodingType: string | null = null): string => + encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value); + +// ─────── Bucket operations ─────── + +export const listBucketsXml = ( + buckets: { name: string; createdAt: Date }[], + _requestId: string, +): string => ` + + + ${buckets + .map( + (b) => ` + ${escapeXml(b.name)} + ${isoDate(b.createdAt)} + `, + ) + .join('')} + +`; + +export const bucketVersioningConfigurationXml = + (): string => ` +`; + +// ─────── Object listing ─────── + +export const listBucketResultXml = ( + bucketName: string, + objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[], + prefixes: string[], + isTruncated: boolean, + marker: string | null, + maxKeys: number, + prefix: string, + delimiter: string | null, + nextMarker: string | null, + _requestId: string, + encodingType: string | null = null, +): string => ` + + ${escapeXml(bucketName)} + ${encodeKey(prefix, encodingType)} + ${encodeKey(marker || '', encodingType)} + ${maxKeys} + ${encodeKey(delimiter || '', encodingType)} + ${encodingType ? `${escapeXml(encodingType)}` : ''} + ${isTruncated} + ${objects + .map( + (o) => ` + ${encodeKey(o.key, encodingType)} + ${isoDate(o.lastModified)} + "${o.etag}" + ${o.sizeBytes} + STANDARD + `, + ) + .join('')} + ${prefixes + .map( + (p) => ` + ${encodeKey(p, encodingType)} + `, + ) + .join('')} + ${nextMarker ? `${encodeKey(nextMarker, encodingType)}` : ''} +`; + +export const listBucketV2ResultXml = ( + bucketName: string, + objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[], + prefixes: string[], + isTruncated: boolean, + maxKeys: number, + prefix: string, + delimiter: string | null, + continuationToken: string | null, + nextContinuationToken: string | null, + keyCount: number, + _requestId: string, + encodingType: string | null = null, +): string => ` + + ${escapeXml(bucketName)} + ${encodeKey(prefix, encodingType)} + ${maxKeys} + ${keyCount} + ${delimiter ? `${encodeKey(delimiter, encodingType)}` : ''} + ${encodingType ? `${escapeXml(encodingType)}` : ''} + ${continuationToken ? `${encodeKey(continuationToken, encodingType)}` : ''} + ${isTruncated} + ${objects + .map( + (o) => ` + ${encodeKey(o.key, encodingType)} + ${isoDate(o.lastModified)} + "${o.etag}" + ${o.sizeBytes} + STANDARD + `, + ) + .join('')} + ${prefixes + .map( + (p) => ` + ${encodeKey(p, encodingType)} + `, + ) + .join('')} + ${nextContinuationToken ? `${encodeKey(nextContinuationToken, encodingType)}` : ''} +`; + +// ─────── Multipart ─────── + +export const initiateMultipartUploadXml = ( + bucketName: string, + key: string, + uploadId: string, +): string => ` + + ${escapeXml(bucketName)} + ${escapeXml(key)} + ${uploadId} +`; + +export const listPartsXml = ( + bucketName: string, + key: string, + uploadId: string, + parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[], + maxParts: number, + isTruncated: boolean, + _requestId: string, +): string => ` + + ${escapeXml(bucketName)} + ${escapeXml(key)} + ${uploadId} + ${maxParts} + ${isTruncated} + ${parts + .map( + (p) => ` + ${p.partNumber} + ${isoDate(p.createdAt)} + "${p.etag}" + ${p.sizeBytes} + `, + ) + .join('')} +`; + +export const listMultipartUploadsXml = ( + bucketName: string, + uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[], + maxUploads: number, + isTruncated: boolean, + nextKeyMarker: string | null, + _requestId: string, +): string => ` + + ${escapeXml(bucketName)} + + + ${nextKeyMarker ? `${escapeXml(nextKeyMarker)}` : ''} + ${maxUploads} + ${isTruncated} + ${uploads + .map( + (u) => ` + ${escapeXml(u.key)} + ${u.uploadId} + ${escapeXml(u.initiatedBy || 's3')}${escapeXml(u.initiatedBy || 's3')} + ${escapeXml(u.initiatedBy || 's3')}${escapeXml(u.initiatedBy || 's3')} + STANDARD + ${isoDate(u.initiatedAt)} + `, + ) + .join('')} +`; + +export const completeMultipartUploadXml = ( + bucketName: string, + key: string, + etag: string, + location: string, +): string => ` + + ${escapeXml(location)} + ${escapeXml(bucketName)} + ${escapeXml(key)} + "${etag}" +`; + +// ─────── Delete result ─────── + +export const deleteResultXml = ( + deleted: string[], + errors: { key: string; code: string; message: string }[], +): string => ` + + ${deleted + .map( + (key) => ` + ${escapeXml(key)} + `, + ) + .join('')} + ${errors + .map( + (e) => ` + ${escapeXml(e.key)} + ${e.code} + ${escapeXml(e.message)} + `, + ) + .join('')} +`; + +// ─────── Copy ─────── + +export const copyObjectResultXml = ( + etag: string, + lastModified: Date, +): string => ` + + "${etag}" + ${isoDate(lastModified)} +`; + +// ─────── Error ─────── + +export const s3ErrorXml = ( + code: string, + message: string, + resource: string, + requestId: string, +): string => ` + + ${code} + ${escapeXml(message)} + ${escapeXml(resource)} + ${requestId} + ${requestId} +`; + +export const s3ErrorResponse = ( + code: string, + message: string, + resource: string, + status: number, + requestId: string = '', + extraHeaders: Record = {}, +): Response => + new Response(s3ErrorXml(code, message, resource, requestId), { + status, + headers: s3Headers(requestId, { + 'content-type': 'application/xml', + ...extraHeaders, + }), + }); + +// ─────── DeleteObjects XML parser ─────── + +export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => { + // H9: Use non-greedy match to handle keys containing < character + const keys = Array.from(body.matchAll(/([\s\S]*?)<\/Key>/g), (match) => match[1]); + // Handle whitespace inside element + namespace prefix support + const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body); + return { keys, quiet }; +}; + +// ─────── CompleteMultipartUpload XML parser ─────── + +export interface CompletePart { + partNumber: number; + etag: string; +} + +export const parseCompleteMultipartBody = (body: string): CompletePart[] => { + const parts: CompletePart[] = []; + const partRegex = /[\s\S]*?<\/Part>/g; + const partMatch = body.match(partRegex) || []; + + for (const partXml of partMatch) { + const numMatch = partXml.match(/(\d+)<\/PartNumber>/); + const etagMatch = partXml.match(/"?([^"<\s]+)"?<\/ETag>/); + if (numMatch && etagMatch) { + parts.push({ + partNumber: Number.parseInt(numMatch[1], 10), + etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''), + }); + } + } + + return parts; +}; diff --git a/src/shared/logger/index.ts b/src/shared/logger/index.ts index 829fc61..770c0eb 100644 --- a/src/shared/logger/index.ts +++ b/src/shared/logger/index.ts @@ -1,4 +1,26 @@ -import _logger from '../../utils/logger'; -export default _logger; -export type { Logger } from 'winston'; -export { _logger as logger }; +import winston from 'winston'; + +const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.errors({ stack: true }), + winston.format.json(), + ), + defaultMeta: { service: 'filedrop' }, + transports: [ + // Write all logs including error logs to file + new winston.transports.File({ filename: 'logs/error.log', level: 'error' }), + new winston.transports.File({ filename: 'logs/combined.log' }), + // Console transport for docker logs / CLI visibility + new winston.transports.Console({ + format: + process.env.NODE_ENV !== 'production' + ? winston.format.combine(winston.format.colorize(), winston.format.simple()) + : winston.format.json(), + }), + ], +}); + +export default logger; +export { logger }; diff --git a/src/shared/metrics/index.ts b/src/shared/metrics/index.ts index 20837d7..fc49103 100644 --- a/src/shared/metrics/index.ts +++ b/src/shared/metrics/index.ts @@ -1 +1,111 @@ -export { MetricsCollector, metricsCollector } from '../../utils/metrics'; +// No imports needed — logger used only by setInterval which moved to index.ts + +interface Metric { + name: string; + value: number; + timestamp: number; + tags?: Record; +} + +interface MetricsSnapshot { + uploadLatency: { p50: number; p95: number; p99: number }; + uploadThroughput: number; + queueSize: number; + errorRate: number; + cacheHitRate: number; + botUtilization: number; + timestamp: number; +} + +class MetricsCollector { + private metrics: Metric[] = []; + private uploadTimes: number[] = []; + private errorCount = 0; + private totalRequests = 0; + private cacheHits = 0; + private cacheMisses = 0; + private maxMetricsSize = 10000; + + recordUploadTime(durationMs: number): void { + this.uploadTimes.push(durationMs); + this.totalRequests++; + + // Keep only last 1000 measurements + if (this.uploadTimes.length > 1000) { + this.uploadTimes.shift(); + } + } + + recordError(): void { + this.errorCount++; + } + + recordCacheHit(): void { + this.cacheHits++; + } + + recordCacheMiss(): void { + this.cacheMisses++; + } + + recordMetric(name: string, value: number, tags?: Record): void { + this.metrics.push({ + name, + value, + timestamp: Date.now(), + tags, + }); + + // Keep metrics bounded + if (this.metrics.length > this.maxMetricsSize) { + this.metrics = this.metrics.slice(-this.maxMetricsSize); + } + } + + private calculatePercentile(arr: number[], percentile: number): number { + if (arr.length === 0) return 0; + const sorted = [...arr].sort((a, b) => a - b); + const index = Math.ceil((percentile / 100) * sorted.length) - 1; + return sorted[Math.max(0, index)]; + } + + getSnapshot(): MetricsSnapshot { + const errorRate = this.totalRequests > 0 ? (this.errorCount / this.totalRequests) * 100 : 0; + const cacheHitRate = + this.cacheHits + this.cacheMisses > 0 + ? (this.cacheHits / (this.cacheHits + this.cacheMisses)) * 100 + : 0; + + return { + uploadLatency: { + p50: this.calculatePercentile(this.uploadTimes, 50), + p95: this.calculatePercentile(this.uploadTimes, 95), + p99: this.calculatePercentile(this.uploadTimes, 99), + }, + uploadThroughput: this.totalRequests > 0 ? this.totalRequests / 60 : 0, + queueSize: 0, // Will be updated by queue + errorRate, + cacheHitRate, + botUtilization: 0, // Will be updated by bot tracker + timestamp: Date.now(), + }; + } + + reset(): void { + this.uploadTimes = []; + this.errorCount = 0; + this.totalRequests = 0; + this.cacheHits = 0; + this.cacheMisses = 0; + this.metrics = []; + } + + getMetrics(name?: string): Metric[] { + if (!name) return this.metrics; + return this.metrics.filter((m) => m.name === name); + } +} + +export const metricsCollector = new MetricsCollector(); + +export { MetricsCollector }; diff --git a/src/shared/utils/file.ts b/src/shared/utils/file.ts index e6fb427..60f72da 100644 --- a/src/shared/utils/file.ts +++ b/src/shared/utils/file.ts @@ -1,5 +1,5 @@ import { unlink } from 'node:fs/promises'; -import logger from '../../utils/logger'; +import logger from '../logger/index'; /** * Safely extracts an error message from an unknown value. diff --git a/src/shared/utils/retry.ts b/src/shared/utils/retry.ts index 7810d1a..6d2a024 100644 --- a/src/shared/utils/retry.ts +++ b/src/shared/utils/retry.ts @@ -1,4 +1,4 @@ -import logger from '../../utils/logger'; +import logger from '../logger/index'; /** Configuration options for retry behaviour. */ interface RetryOptions { diff --git a/src/utils/auth.ts b/src/utils/auth.ts deleted file mode 100644 index 529aff0..0000000 --- a/src/utils/auth.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { createHmac, timingSafeEqual } from 'node:crypto'; -import { config } from '../env'; - -const ADMIN_USERNAME = 'admin'; -const SIGNATURE_SEPARATOR = '.'; - -type Handler = (req: Request) => Response | Promise; - -export interface AuthSession { - username: string; - expiresAt: Date | null; - method: 'cookie' | 'bearer'; -} - -interface CookieOptions { - secret?: string; - cookieName?: string; - maxAgeMs?: number; -} - -interface SessionPayload { - u: string; - e: number; -} - -const getSecret = (secret?: string): string => secret ?? config.adminApiToken; -const getCookieName = (cookieName?: string): string => cookieName ?? config.sessionCookieName; -const getMaxAgeMs = (maxAgeMs?: number): number => maxAgeMs ?? config.sessionMaxAgeMs; - -const encodePayload = (value: string): string => Buffer.from(value, 'utf8').toString('base64url'); - -const decodePayload = (value: string): string | null => { - try { - return Buffer.from(value, 'base64url').toString('utf8'); - } catch { - return null; - } -}; - -export const isAuthEnabled = (secret = config.adminApiToken): boolean => secret.length > 0; - -export const timingSafeCompare = (left: string, right: string): boolean => { - const leftBuffer = Buffer.from(left); - const rightBuffer = Buffer.from(right); - - if (leftBuffer.length !== rightBuffer.length) { - return false; - } - - return timingSafeEqual(leftBuffer, rightBuffer); -}; - -export const signCookiePayload = (payload: string, secret: string): string => - createHmac('sha256', secret).update(payload).digest('base64url'); - -export const verifyCookieSignature = (cookieValue: string, secret: string): string | null => { - const separatorIndex = cookieValue.lastIndexOf(SIGNATURE_SEPARATOR); - if (separatorIndex <= 0 || separatorIndex === cookieValue.length - 1) { - return null; - } - - const payload = cookieValue.slice(0, separatorIndex); - const signature = cookieValue.slice(separatorIndex + 1); - const expectedSignature = signCookiePayload(payload, secret); - - if (!timingSafeCompare(signature, expectedSignature)) { - return null; - } - - return payload; -}; - -const cookieAttributes = (maxAgeSeconds: number): string => - [`Max-Age=${maxAgeSeconds}`, 'Path=/', 'HttpOnly', 'SameSite=Lax', 'Secure'].join('; '); - -export const createSessionCookie = ( - username = ADMIN_USERNAME, - options: CookieOptions = {}, -): string => { - const secret = getSecret(options.secret); - const cookieName = getCookieName(options.cookieName); - const maxAgeMs = getMaxAgeMs(options.maxAgeMs); - const expiresAt = Date.now() + maxAgeMs; - const payload = encodePayload( - JSON.stringify({ u: username, e: expiresAt } satisfies SessionPayload), - ); - const signature = signCookiePayload(payload, secret); - const maxAgeSeconds = Math.max(1, Math.floor(maxAgeMs / 1000)); - - return `${cookieName}=${payload}${SIGNATURE_SEPARATOR}${signature}; ${cookieAttributes(maxAgeSeconds)}`; -}; - -export const clearSessionCookie = (cookieName = config.sessionCookieName): string => - `${cookieName}=; ${cookieAttributes(0)}`; - -const findCookieValue = (cookieHeader: string | null, cookieName: string): string | null => { - if (!cookieHeader) return null; - - for (const rawCookie of cookieHeader.split(';')) { - const cookie = rawCookie.trim(); - const equalsIndex = cookie.indexOf('='); - if (equalsIndex <= 0) continue; - - const name = cookie.slice(0, equalsIndex); - if (name === cookieName) { - return cookie.slice(equalsIndex + 1); - } - } - - return null; -}; - -export const parseSessionFromCookie = ( - cookieHeader: string | null, - options: Pick = {}, -): AuthSession | null => { - const secret = getSecret(options.secret); - const cookieName = getCookieName(options.cookieName); - if (!isAuthEnabled(secret)) return null; - - const cookieValue = findCookieValue(cookieHeader, cookieName); - if (!cookieValue) return null; - - const encodedPayload = verifyCookieSignature(cookieValue, secret); - if (!encodedPayload) return null; - - const rawPayload = decodePayload(encodedPayload); - if (!rawPayload) return null; - - try { - const payload = JSON.parse(rawPayload) as Partial; - if (payload.u !== ADMIN_USERNAME || typeof payload.e !== 'number') return null; - if (!Number.isFinite(payload.e) || payload.e <= Date.now()) return null; - - return { - username: payload.u, - expiresAt: new Date(payload.e), - method: 'cookie', - }; - } catch { - return null; - } -}; - -export const checkBearerToken = ( - authorizationHeader: string | null, - secret = config.adminApiToken, -): boolean => { - if (!isAuthEnabled(secret) || !authorizationHeader) return false; - - const [scheme, ...rest] = authorizationHeader.split(' '); - if (scheme !== 'Bearer' || rest.length === 0) return false; - - const token = rest.join(' ').trim(); - return token.length > 0 && timingSafeCompare(token, secret); -}; - -export const getAuthSession = ( - req: Request, - options: Pick = {}, -): AuthSession | null => { - const secret = getSecret(options.secret); - if (!isAuthEnabled(secret)) { - return { - username: ADMIN_USERNAME, - expiresAt: null, - method: 'bearer', - }; - } - - const cookieSession = parseSessionFromCookie(req.headers.get('cookie'), options); - if (cookieSession) return cookieSession; - - if (checkBearerToken(req.headers.get('authorization'), secret)) { - return { - username: ADMIN_USERNAME, - expiresAt: null, - method: 'bearer', - }; - } - - return null; -}; - -export const unauthorizedResponse = (): Response => - Response.json({ error: 'Unauthorized' }, { status: 401 }); - -export const requireAuth = ( - handler: Handler, - options: Pick = {}, -): ((req: Request) => Promise) => { - return async (req: Request): Promise => { - const secret = getSecret(options.secret); - if (!isAuthEnabled(secret)) { - return handler(req); - } - - const session = getAuthSession(req, options); - if (!session) { - return unauthorizedResponse(); - } - - return handler(req); - }; -}; diff --git a/src/utils/cache.ts b/src/utils/cache.ts deleted file mode 100644 index 4dd28ad..0000000 --- a/src/utils/cache.ts +++ /dev/null @@ -1,74 +0,0 @@ -// Simple in-memory cache with TTL support -interface CacheEntry { - value: T; - expiresAt: number; -} - -class Cache { - private store = new Map>(); - private ttlMs: number; - - constructor(ttlSeconds: number = 3600) { - this.ttlMs = ttlSeconds * 1000; - } - - set(key: string, value: T): void { - this.store.set(key, { - value, - expiresAt: Date.now() + this.ttlMs, - }); - } - - get(key: string): T | null { - const entry = this.store.get(key); - if (!entry) return null; - - if (Date.now() > entry.expiresAt) { - this.store.delete(key); - return null; - } - - return entry.value; - } - - has(key: string): boolean { - return this.get(key) !== null; - } - - delete(key: string): void { - this.store.delete(key); - } - - clear(): void { - this.store.clear(); - } - - size(): number { - return this.store.size; - } - - // Cleanup expired entries - cleanup(): number { - let removed = 0; - const now = Date.now(); - - for (const [key, entry] of this.store.entries()) { - if (now > entry.expiresAt) { - this.store.delete(key); - removed++; - } - } - - return removed; - } -} - -// File info cache (1 hour TTL) -export const fileInfoCache = new Cache<{ - file_size: number; - mime_type: string; - file_path: string; - bot_token: string; -}>(3600); - -export { Cache }; diff --git a/src/utils/chunked-storage.ts b/src/utils/chunked-storage.ts deleted file mode 100644 index dce3c74..0000000 --- a/src/utils/chunked-storage.ts +++ /dev/null @@ -1,249 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { createReadStream } from 'node:fs'; -import { gzipSync } from 'node:zlib'; -import { nanoid } from 'nanoid'; -import { db, files as fileSchema } from '../db'; -import { insertFileParts, listFileParts, type NewFilePartInput } from '../db/file-parts'; -import type { File } from '../db/schema'; -import { config } from '../env'; -import { botPool } from '../infrastructure/telegram/bot-pool'; -import { computeHash } from './file'; -import { createGetObjectResponse, type ObjectPartSource } from './s3/object-stream'; -import type { RangeParseResult } from './s3/range'; - -export type ChunkCompressionAlgorithm = 'gzip' | null; - -export interface ChunkedUploadPart { - partNumber: number; - telegramFileId: string; - telegramFileUniqueId: string; - storageMessageId: number; - sizeBytes: number; - storedSizeBytes: number; - compressionAlgorithm: ChunkCompressionAlgorithm; - etag: string; -} - -export interface ChunkedUploadResult { - parts: ChunkedUploadPart[]; - fileHash: string; - totalSizeBytes: number; -} - -export interface ChunkedFileInput { - tempPath: string; - partFileNamePrefix: string; - fileName: string; - mimeType: string; - sizeBytes: number; - fileType: string; - uploaderId: number; - bucketId?: string | null; - s3Key?: string | null; -} - -const asSafeChunkSize = (chunkSizeBytes: number): number => { - if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) { - throw new Error('Invalid Telegram chunk size'); - } - return chunkSizeBytes; -}; - -const maybeCompressChunk = ( - chunk: Buffer, - compress: boolean, - compressionMinSizeBytes: number, -): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => { - if (!compress || chunk.byteLength < compressionMinSizeBytes) { - return { bytes: chunk, compressionAlgorithm: null }; - } - - const gzipped = gzipSync(chunk); - if (gzipped.byteLength >= chunk.byteLength) { - return { bytes: chunk, compressionAlgorithm: null }; - } - - return { bytes: gzipped, compressionAlgorithm: 'gzip' }; -}; - -export const uploadFileInTelegramChunks = async (input: { - tempPath: string; - partFileNamePrefix: string; - chunkSizeBytes: number; - compress: boolean; - compressionMinSizeBytes: number; -}): Promise => { - const chunkSizeBytes = asSafeChunkSize(input.chunkSizeBytes); - const hasher = new Bun.CryptoHasher('sha256'); - const parts: ChunkedUploadPart[] = []; - let totalSizeBytes = 0; - let partNumber = 0; - - const stream = createReadStream(input.tempPath, { highWaterMark: chunkSizeBytes }); - - // Concurrent upload set: tracks in-flight uploads and limits how many - // chunks are being uploaded at once from this single file. - const inFlight = new Set>(); - - for await (const data of stream) { - const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data as Uint8Array); - if (chunk.byteLength === 0) continue; - - partNumber += 1; - totalSizeBytes += chunk.byteLength; - hasher.update(chunk); - - const { bytes, compressionAlgorithm } = maybeCompressChunk( - chunk, - input.compress, - input.compressionMinSizeBytes, - ); - const currentPart = partNumber; - - // Fire upload concurrently — don't await inside the read loop - const uploadPromise = botPool - .forwardToStorage(bytes, `${input.partFileNamePrefix}.part-${currentPart}`, 'document') - .then((forwardResult) => { - parts.push({ - partNumber: currentPart, - telegramFileId: forwardResult.telegramFileId, - telegramFileUniqueId: forwardResult.telegramFileUniqueId, - storageMessageId: forwardResult.storageMessageId, - sizeBytes: chunk.byteLength, - storedSizeBytes: bytes.byteLength, - compressionAlgorithm, - etag: computeHash(chunk), - }); - }); - - // Clean up from in-flight set when done (regardless of success/failure) - const trackPromise = uploadPromise.finally(() => { - inFlight.delete(trackPromise); - }); - - inFlight.add(trackPromise); - - // Backpressure: if too many chunks are in-flight, wait for one to - // finish before reading more — prevents unbounded memory growth. - if (inFlight.size >= botPool.getEffectiveConcurrency() * 2) { - await Promise.race(inFlight); - // Yield microtask to let .finally() run and remove from inFlight - await new Promise((resolve) => setTimeout(resolve, 0)); - } - } - - // Wait for all remaining uploads to finish - await Promise.all(inFlight); - - return { - parts, - fileHash: hasher.digest('hex'), - totalSizeBytes, - }; -}; - -export const storeFileInTelegramChunks = async (input: ChunkedFileInput): Promise => { - const upload = await uploadFileInTelegramChunks({ - tempPath: input.tempPath, - partFileNamePrefix: input.partFileNamePrefix, - chunkSizeBytes: config.telegramChunkSizeBytes, - compress: config.compressChunkedUploads, - compressionMinSizeBytes: config.chunkCompressionMinSizeBytes, - }); - - const firstPart = upload.parts[0]; - if (!firstPart) { - throw new Error('Chunked upload produced no parts'); - } - - const now = new Date(); - const fileId = randomUUID(); - const publicId = nanoid(); - const file: File = { - id: fileId, - publicId, - telegramFileId: firstPart.telegramFileId, - telegramFileUniqueId: firstPart.telegramFileUniqueId, - storageChatId: config.storageChatId, - storageMessageId: firstPart.storageMessageId, - fileName: input.fileName, - mimeType: input.mimeType, - sizeBytes: upload.totalSizeBytes, - fileType: input.fileType, - uploaderId: input.uploaderId, - fileHash: upload.fileHash, - archiveTelegramFileId: null, - archiveStorageMessageId: null, - archiveFileName: null, - archiveEntryName: null, - archiveMimeType: null, - archiveSizeBytes: null, - bucketId: input.bucketId ?? null, - s3Key: input.s3Key ?? null, - storageBackend: 'chunked', - isDeleted: false, - multipartUploadId: null, - partCount: upload.parts.length, - createdAt: now, - updatedAt: now, - }; - - await db.insert(fileSchema).values(file); - - const fileParts: NewFilePartInput[] = upload.parts.map((part) => ({ - fileId, - partNumber: part.partNumber, - telegramFileId: part.telegramFileId, - telegramFileUniqueId: part.telegramFileUniqueId, - storageChatId: config.storageChatId, - storageMessageId: part.storageMessageId, - sizeBytes: part.sizeBytes, - storedSizeBytes: part.storedSizeBytes, - compressionAlgorithm: part.compressionAlgorithm, - etag: part.etag, - })); - - await insertFileParts(fileParts); - return file; -}; - -export const buildChunkedObjectSources = async (file: File): Promise => { - const parts = await listFileParts(file.id); - const sources: ObjectPartSource[] = []; - - for (const part of parts) { - const fileInfo = await botPool.getFileInfo(part.telegramFileId); - sources.push({ - telegramFileId: part.telegramFileId, - telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`, - sizeBytes: part.sizeBytes, - storedSizeBytes: part.storedSizeBytes, - compressionAlgorithm: part.compressionAlgorithm, - partNumber: part.partNumber, - }); - } - - return sources; -}; - -export const createChunkedObjectResponse = async (input: { - file: File; - range: RangeParseResult; - reqId: string; -}): Promise => { - const parts = await buildChunkedObjectSources(input.file); - if (parts.length === 0) { - throw new Error('Chunked object has no parts'); - } - - return createGetObjectResponse({ - reqId: input.reqId, - contentType: input.file.mimeType, - etag: input.file.fileHash || parts.map((p) => p.telegramFileId).join('-'), - lastModified: - input.file.createdAt instanceof Date ? input.file.createdAt : new Date(input.file.createdAt), - totalSize: Number(input.file.sizeBytes), - parts, - range: input.range, - }); -}; diff --git a/src/utils/file.ts b/src/utils/file.ts deleted file mode 100644 index b99c4cf..0000000 --- a/src/utils/file.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { unlink } from 'node:fs/promises'; -import logger from './logger'; - -export const getErrorMessage = (error: unknown): string => { - return error instanceof Error ? error.message : String(error); -}; - -export const cleanupTempFile = async (tempPath: string): Promise => { - try { - await unlink(tempPath); - } catch (err) { - logger.warn('Failed to cleanup temp file', { tempPath, error: getErrorMessage(err) }); - } -}; - -interface FileMetadata { - publicId: string; - telegramFileId: string; - telegramFileUniqueId: string; - storageChatId: number; - storageMessageId: number; - fileName: string; - mimeType: string; - sizeBytes: number; - fileType: string; - uploaderId: number; - createdAt: Date | string | number; -} - -const FILE_TYPES: Record = { - document: 2 * 1024 * 1024 * 1024, // 2GB - photo: 10 * 1024 * 1024, // 10MB - video: 2 * 1024 * 1024 * 1024, // 2GB - audio: 200 * 1024 * 1024, // 200MB - voice: 200 * 1024 * 1024, // 200MB - animation: 2 * 1024 * 1024 * 1024, // 2GB - sticker: 10 * 1024 * 1024, // 10MB - video_note: 2 * 1024 * 1024 * 1024, // 2GB -}; - -export const getFileType = (mime: string | null, caption?: string): string => { - const mimeUpper = mime?.split('/')[0]?.toLowerCase(); - const captionLower = caption?.toLowerCase(); - - if (mime?.toLowerCase() === 'image/webp' || captionLower?.includes('sticker')) return 'sticker'; - if (captionLower?.includes('video_note')) return 'video_note'; - if (mimeUpper === 'video') return 'video'; - if (mimeUpper === 'audio') return 'audio'; - if (mimeUpper === 'document') return 'document'; - if (mimeUpper === 'image') return captionLower?.includes('gif') ? 'animation' : 'photo'; - if (captionLower?.includes('voice')) return 'voice'; - if (captionLower?.includes('animation')) return 'animation'; - - return mimeUpper === 'application' ? 'application' : 'document'; -}; - -export const checkFileSize = (sizeBytes: number, fileType: string): boolean => { - const limit = FILE_TYPES[fileType] || FILE_TYPES.document; - return sizeBytes <= limit; -}; - -export const ensureExtension = ( - fileName: string, - buffer: Buffer, - detectedMime?: string, -): { fileName: string; mimeType: string } => { - const mimeMap: Record = { - 'application/pdf': 'pdf', - 'image/png': 'png', - 'image/jpeg': 'jpg', - 'image/gif': 'gif', - 'text/plain': 'txt', - 'application/zip': 'zip', - }; - - let ext: string | null = null; - if (buffer.subarray(0, 4).toString() === '%PDF') { - ext = 'pdf'; - } else if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) { - ext = 'png'; - } else if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { - ext = 'jpg'; - } else if (buffer.subarray(0, 4).toString() === 'GIF8') { - ext = 'gif'; - } else if (detectedMime) { - ext = mimeMap[detectedMime.toLowerCase()] || null; - } - - let finalFileName = fileName; - const hasExtension = fileName.includes('.') && fileName.split('.').pop()!.length >= 2; - if (!hasExtension && ext) { - finalFileName = `${fileName}.${ext}`; - } - - const mimeType = ext - ? Object.keys(mimeMap).find((k) => mimeMap[k] === ext) || - detectedMime || - 'application/octet-stream' - : detectedMime || 'application/octet-stream'; - - return { fileName: finalFileName, mimeType }; -}; - -type HeaderMapRequest = { - headers?: - | { - get?: (name: string) => string | null; - } - | Record; -}; - -type FileLike = { - fileName?: string; - mimeType?: string; -}; - -type MessageLike = { - document?: FileLike; - photo?: FileLike[]; - audio?: FileLike; - voice?: FileLike; - animation?: FileLike; -}; - -const getHeader = (request: HeaderMapRequest | null, name: string): string | undefined => { - const headers = request?.headers; - if (!headers) return undefined; - - const get = 'get' in headers ? headers.get : undefined; - if (typeof get === 'function') return get(name) || undefined; - - return (headers as Record)[name]; -}; - -export const extractFileName = (msg: MessageLike, request: HeaderMapRequest | null): string => { - const headerFileName = getHeader(request, 'x-file-name'); - if (headerFileName) return headerFileName; - - return ( - msg.document?.fileName || - msg.photo?.slice(-1)[0]?.fileName || - msg.audio?.fileName || - msg.voice?.fileName || - msg.animation?.fileName || - 'file' - ); -}; - -export const extractMimeType = (msg: MessageLike, request: HeaderMapRequest | null): string => { - const headerMimeType = getHeader(request, 'x-mime-type'); - if (headerMimeType) return headerMimeType; - - return ( - msg.document?.mimeType || - msg.photo?.slice(-1)[0]?.mimeType || - msg.audio?.mimeType || - msg.voice?.mimeType || - msg.animation?.mimeType || - 'application/octet-stream' - ); -}; - -export const computeHash = (buffer: Buffer): string => { - const hasher = new Bun.CryptoHasher('sha256'); - hasher.update(buffer); - return hasher.digest('hex'); -}; - -export interface TelegramMessageFile { - file_id: string; - file_unique_id: string; - file_size?: number; - mime_type?: string; - file_name?: string; -} - -export interface TelegramMediaMessage { - message_id: number; - document?: TelegramMessageFile; - photo?: TelegramMessageFile[]; - video?: TelegramMessageFile; - audio?: TelegramMessageFile; - voice?: TelegramMessageFile; - animation?: TelegramMessageFile; - sticker?: TelegramMessageFile; - video_note?: TelegramMessageFile; -} - -export const extractFileFromMessage = ( - msg: TelegramMediaMessage, - fileType: string, -): TelegramMessageFile => { - if (fileType === 'photo') return msg.photo?.slice(-1)[0] as TelegramMessageFile; - if (fileType === 'sticker') return msg.sticker as TelegramMessageFile; - return msg[fileType as keyof TelegramMediaMessage] as TelegramMessageFile; -}; - -export const detectFileType = (msg: TelegramMediaMessage): string => { - if (msg.document) return 'document'; - if (msg.photo) return 'photo'; - if (msg.video) return 'video'; - if (msg.audio) return 'audio'; - if (msg.voice) return 'voice'; - if (msg.animation) return 'animation'; - if (msg.sticker) return 'sticker'; - if (msg.video_note) return 'video_note'; - return 'document'; -}; - -export const getFileSizeLimit = (fileType: string): number => - FILE_TYPES[fileType] || FILE_TYPES.document; - -export const formatCreatedAt = (createdAt: Date | string | number): string => { - return createdAt instanceof Date ? createdAt.toISOString() : new Date(createdAt).toISOString(); -}; - -export interface UploadResponse { - public_id: string; - file_name: string; - mime_type: string; - size_bytes: number; - file_type: string; - created_at: string; - download_url: string; -} - -export const buildUploadResponse = (file: FileMetadata, baseUrl: string): UploadResponse => { - return { - public_id: file.publicId, - file_name: file.fileName, - mime_type: file.mimeType, - size_bytes: file.sizeBytes, - file_type: file.fileType, - created_at: formatCreatedAt(file.createdAt), - download_url: `${baseUrl}/f/${file.publicId}`, - }; -}; diff --git a/src/utils/ip.ts b/src/utils/ip.ts deleted file mode 100644 index 4d96f35..0000000 --- a/src/utils/ip.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { config } from '../env'; - -export const extractClientIp = (req: Request): string => { - if (!config.trustProxy) return '127.0.0.1'; - - const forwardedFor = req.headers.get('x-forwarded-for'); - if (forwardedFor) { - const firstIp = forwardedFor.split(',')[0]?.trim(); - if (firstIp) return firstIp; - } - - const realIp = req.headers.get('x-real-ip')?.trim(); - if (realIp) return realIp; - - return '127.0.0.1'; -}; diff --git a/src/utils/logger.ts b/src/utils/logger.ts deleted file mode 100644 index 5767e7c..0000000 --- a/src/utils/logger.ts +++ /dev/null @@ -1,25 +0,0 @@ -import winston from 'winston'; - -const logger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.timestamp(), - winston.format.errors({ stack: true }), - winston.format.json(), - ), - defaultMeta: { service: 'filedrop' }, - transports: [ - // Write all logs including error logs to file - new winston.transports.File({ filename: 'logs/error.log', level: 'error' }), - new winston.transports.File({ filename: 'logs/combined.log' }), - // Console transport for docker logs / CLI visibility - new winston.transports.Console({ - format: - process.env.NODE_ENV !== 'production' - ? winston.format.combine(winston.format.colorize(), winston.format.simple()) - : winston.format.json(), - }), - ], -}); - -export default logger; diff --git a/src/utils/metrics.ts b/src/utils/metrics.ts deleted file mode 100644 index fc49103..0000000 --- a/src/utils/metrics.ts +++ /dev/null @@ -1,111 +0,0 @@ -// No imports needed — logger used only by setInterval which moved to index.ts - -interface Metric { - name: string; - value: number; - timestamp: number; - tags?: Record; -} - -interface MetricsSnapshot { - uploadLatency: { p50: number; p95: number; p99: number }; - uploadThroughput: number; - queueSize: number; - errorRate: number; - cacheHitRate: number; - botUtilization: number; - timestamp: number; -} - -class MetricsCollector { - private metrics: Metric[] = []; - private uploadTimes: number[] = []; - private errorCount = 0; - private totalRequests = 0; - private cacheHits = 0; - private cacheMisses = 0; - private maxMetricsSize = 10000; - - recordUploadTime(durationMs: number): void { - this.uploadTimes.push(durationMs); - this.totalRequests++; - - // Keep only last 1000 measurements - if (this.uploadTimes.length > 1000) { - this.uploadTimes.shift(); - } - } - - recordError(): void { - this.errorCount++; - } - - recordCacheHit(): void { - this.cacheHits++; - } - - recordCacheMiss(): void { - this.cacheMisses++; - } - - recordMetric(name: string, value: number, tags?: Record): void { - this.metrics.push({ - name, - value, - timestamp: Date.now(), - tags, - }); - - // Keep metrics bounded - if (this.metrics.length > this.maxMetricsSize) { - this.metrics = this.metrics.slice(-this.maxMetricsSize); - } - } - - private calculatePercentile(arr: number[], percentile: number): number { - if (arr.length === 0) return 0; - const sorted = [...arr].sort((a, b) => a - b); - const index = Math.ceil((percentile / 100) * sorted.length) - 1; - return sorted[Math.max(0, index)]; - } - - getSnapshot(): MetricsSnapshot { - const errorRate = this.totalRequests > 0 ? (this.errorCount / this.totalRequests) * 100 : 0; - const cacheHitRate = - this.cacheHits + this.cacheMisses > 0 - ? (this.cacheHits / (this.cacheHits + this.cacheMisses)) * 100 - : 0; - - return { - uploadLatency: { - p50: this.calculatePercentile(this.uploadTimes, 50), - p95: this.calculatePercentile(this.uploadTimes, 95), - p99: this.calculatePercentile(this.uploadTimes, 99), - }, - uploadThroughput: this.totalRequests > 0 ? this.totalRequests / 60 : 0, - queueSize: 0, // Will be updated by queue - errorRate, - cacheHitRate, - botUtilization: 0, // Will be updated by bot tracker - timestamp: Date.now(), - }; - } - - reset(): void { - this.uploadTimes = []; - this.errorCount = 0; - this.totalRequests = 0; - this.cacheHits = 0; - this.cacheMisses = 0; - this.metrics = []; - } - - getMetrics(name?: string): Metric[] { - if (!name) return this.metrics; - return this.metrics.filter((m) => m.name === name); - } -} - -export const metricsCollector = new MetricsCollector(); - -export { MetricsCollector }; diff --git a/src/utils/rateLimit.ts b/src/utils/rateLimit.ts deleted file mode 100644 index 6843957..0000000 --- a/src/utils/rateLimit.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { config } from '../env'; -import { extractClientIp } from './ip'; -import logger from './logger'; - -interface RateLimitEntry { - count: number; - resetTime: number; -} - -const rateLimitStore = new Map(); -const MAX_STORE_ENTRIES = 50000; - -const evictExpiredEntries = (now = Date.now()): number => { - let cleaned = 0; - - for (const [key, entry] of rateLimitStore.entries()) { - if (now > entry.resetTime) { - rateLimitStore.delete(key); - cleaned++; - } - } - - return cleaned; -}; - -const ensureStoreCapacity = (now: number): void => { - if (rateLimitStore.size < MAX_STORE_ENTRIES) return; - - evictExpiredEntries(now); - while (rateLimitStore.size >= MAX_STORE_ENTRIES) { - const oldestKey = rateLimitStore.keys().next().value; - if (!oldestKey) break; - rateLimitStore.delete(oldestKey); - } -}; - -export const checkRateLimit = (key: string): boolean => { - const now = Date.now(); - const entry = rateLimitStore.get(key); - - if (!entry || now > entry.resetTime) { - ensureStoreCapacity(now); - rateLimitStore.set(key, { - count: 1, - resetTime: now + config.rateLimitWindowMs, - }); - return true; - } - - if (entry.count >= config.rateLimitMaxRequests) { - logger.warn('Rate limit exceeded', { key, count: entry.count }); - return false; - } - - entry.count++; - return true; -}; - -export const withRateLimit = ( - handler: (req: T) => Promise, -): ((req: T) => Promise) => { - return async (req: T): Promise => { - const ip = extractClientIp(req); - if (!checkRateLimit(ip)) { - return Response.json({ error: 'Rate limit exceeded' }, { status: 429 }); - } - - return handler(req); - }; -}; - -export const cleanupRateLimitCache = (): void => { - const cleaned = evictExpiredEntries(); - - if (cleaned > 0) { - logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size }); - } -}; - -export const getRateLimitStats = () => ({ - trackedIPs: rateLimitStore.size, - windowSize: config.rateLimitWindowMs, - maxRequests: config.rateLimitMaxRequests, - maxTrackedIPs: MAX_STORE_ENTRIES, -}); - -export const clearRateLimitCache = (): void => { - rateLimitStore.clear(); -}; diff --git a/src/utils/retry.ts b/src/utils/retry.ts deleted file mode 100644 index d3af93e..0000000 --- a/src/utils/retry.ts +++ /dev/null @@ -1,91 +0,0 @@ -import logger from './logger'; - -interface RetryOptions { - maxRetries?: number; - initialDelayMs?: number; - maxDelayMs?: number; - backoffMultiplier?: number; - shouldRetry?: (error: unknown) => boolean; -} - -const DEFAULT_OPTIONS: Required = { - maxRetries: 3, - initialDelayMs: 100, - maxDelayMs: 5000, - backoffMultiplier: 2, - shouldRetry: (error: unknown) => { - const errorStr = error instanceof Error ? error.message : String(error); - // Retry on transient errors - return ( - errorStr.includes('ECONNREFUSED') || - errorStr.includes('ETIMEDOUT') || - errorStr.includes('ENOTFOUND') || - errorStr.includes('429') || - errorStr.includes('timeout') - ); - }, -}; - -export const withRetry = async ( - fn: () => Promise, - options: RetryOptions = {}, -): Promise => { - const opts = { ...DEFAULT_OPTIONS, ...options }; - let lastError: unknown; - let delay = opts.initialDelayMs; - - for (let attempt = 0; attempt <= opts.maxRetries; attempt++) { - try { - return await fn(); - } catch (error: unknown) { - lastError = error; - const errorStr = error instanceof Error ? error.message : String(error); - - if (attempt === opts.maxRetries || !opts.shouldRetry(error)) { - logger.error('Retry exhausted', { - attempt, - maxRetries: opts.maxRetries, - error: errorStr, - }); - throw error; - } - - logger.warn('Retrying after error', { - attempt, - delay, - error: errorStr, - }); - - await new Promise((resolve) => setTimeout(resolve, delay)); - delay = Math.min(delay * opts.backoffMultiplier, opts.maxDelayMs); - } - } - - throw lastError; -}; - -export const withTimeout = async ( - fn: () => Promise, - timeoutMs: number = 30000, -): Promise => { - return Promise.race([ - fn(), - new Promise((_, reject) => - setTimeout(() => reject(new Error(`Operation timeout after ${timeoutMs}ms`)), timeoutMs), - ), - ]); -}; - -export const withFallback = async ( - primary: () => Promise, - fallback: () => Promise, -): Promise => { - try { - return await primary(); - } catch (error: unknown) { - logger.warn('Primary operation failed, using fallback', { - error: error instanceof Error ? error.message : String(error), - }); - return fallback(); - } -}; diff --git a/src/utils/s3/auth.ts b/src/utils/s3/auth.ts deleted file mode 100644 index 720b3ed..0000000 --- a/src/utils/s3/auth.ts +++ /dev/null @@ -1,463 +0,0 @@ -import { timingSafeEqual } from 'node:crypto'; - -/** - * Timing-safe string comparison that prevents timing attacks. - * - * Uses `crypto.timingSafeEqual` which runs in constant time regardless of - * where the strings differ. Returns false for mismatched-length inputs - * to avoid leaking length information via early return. - * - * @param left - The first string to compare. - * @param right - The second string to compare. - * @returns True if both strings are equal. - */ -const timingSafeCompare = (left: string, right: string): boolean => { - const leftBuffer = Buffer.from(left); - const rightBuffer = Buffer.from(right); - - if (leftBuffer.length !== rightBuffer.length) { - return false; - } - - return timingSafeEqual(leftBuffer, rightBuffer); -}; - -export interface SigV4Result { - isValid: boolean; - credential: { - accessKey: string; - date: string; - region: string; - service: string; - } | null; - errorCode?: string; -} - -export interface VerifyPresignedUrlInput { - url: string; - method: string; - headers: Record; - s3AccessKey: string; - s3SecretKey: string; - region: string; - now?: Date; -} - -const SERVICE = 's3'; -const TERMINATION = 'aws4_request'; - -/** - * Maximum acceptable clock skew between client and server for header-based - * SigV4 authentication. AWS allows 15 minutes. - */ -const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => { - if (data instanceof Uint8Array) return data; - if (data instanceof ArrayBuffer) return new Uint8Array(data); - return new TextEncoder().encode(data); -}; - -const sha256Hex = async (data: string | Uint8Array | ArrayBuffer): Promise => { - const hashBuffer = await crypto.subtle.digest('SHA-256', buf(data) as never); - const hashArray = Array.from(new Uint8Array(hashBuffer)); - return hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); -}; - -const hmacSha256 = async (key: Uint8Array, message: string): Promise => { - const cryptoKey = await crypto.subtle.importKey( - 'raw', - key as never, - { name: 'HMAC', hash: 'SHA-256' }, - false, - ['sign'], - ); - const result = await crypto.subtle.sign('HMAC', cryptoKey, buf(message) as never); - return new Uint8Array(result); -}; - -const getSigningKey = async ( - secretKey: string, - dateStamp: string, - region: string, -): Promise => { - let key = await hmacSha256(buf(`AWS4${secretKey}`), dateStamp); - key = await hmacSha256(key, region); - key = await hmacSha256(key, SERVICE); - return await hmacSha256(key, TERMINATION); -}; - -const hmacHex = async (key: Uint8Array, message: string): Promise => { - const result = await hmacSha256(key, message); - return Array.from(result) - .map((b) => b.toString(16).padStart(2, '0')) - .join(''); -}; - -const parseAuthorizationHeader = (authHeader: string) => { - const credentialMatch = authHeader.match(/Credential=([^,]+)/); - const signedHeadersMatch = authHeader.match(/SignedHeaders=([^,]+)/); - const signatureMatch = authHeader.match(/Signature=([^,]+)/); - - if (!credentialMatch || !signedHeadersMatch || !signatureMatch) return null; - - const credentialParts = credentialMatch[1].split('/'); - if (credentialParts.length !== 5) return null; - - return { - accessKey: credentialParts[0], - date: credentialParts[1], - region: credentialParts[2], - service: credentialParts[3], - termination: credentialParts[4], - signedHeaders: signedHeadersMatch[1], - signature: signatureMatch[1], - }; -}; - -const buildCanonicalRequest = ( - method: string, - canonicalUri: string, - canonicalQueryString: string, - signedHeaders: string, - headers: Record, - hashedPayload: string, -): string => { - const canonicalHeaders = signedHeaders - .split(';') - .map((h) => { - const value = headers[h.toLowerCase()] || ''; - return `${h.toLowerCase()}:${value.trim()}\n`; - }) - .join(''); - - return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`; -}; - -/** - * Normalizes a URI per AWS SigV4 requirements plus RFC 3986: - * - * 1. Decode percent-encoded characters - * 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4 - * - * @param uri - The raw URI path to normalize. - * @returns The normalized URI path. - */ -const normalizeUri = (uri: string): string => { - if (!uri || uri === '') return '/'; - - // AWS SigV4 requires URI-decoded paths in the canonical request - // Only `.` and `..` segments are removed per RFC 3986 section 5.2.4 - // Empty segments (from `//` or trailing `/`) are preserved — they are - // part of the URI and the SDK signs them. - const decoded = decodeURIComponent(uri); - const segments = decoded.split('/'); - const result: string[] = []; - - for (const segment of segments) { - if (segment === '.') continue; - if (segment === '..') { - result.pop(); - continue; - } - result.push(segment); - } - - // Join preserves empty first segment (from leading /) automatically - return result.join('/') || '/'; -}; - -const awsEncode = (value: string): string => - encodeURIComponent(value).replace( - /[!'()*]/g, - (ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`, - ); - -export const buildCanonicalQueryString = ( - searchParams: URLSearchParams, - excludeKeys: Set = new Set(), -): string => { - const pairs: Array<[string, string]> = []; - for (const [key, value] of searchParams.entries()) { - if (!excludeKeys.has(key)) pairs.push([key, value]); - } - // AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare - pairs.sort(([ak, av], [bk, bv]) => { - const a = `${awsEncode(ak)}=${awsEncode(av)}`; - const b = `${awsEncode(bk)}=${awsEncode(bv)}`; - if (a < b) return -1; - if (a > b) return 1; - return 0; - }); - return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&'); -}; - -const getHashedPayload = async (body: string | null): Promise => { - if (!body || body.length === 0) return await sha256Hex(''); - return await sha256Hex(body); -}; - -/** - * Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date. - * - * @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format. - * @returns The parsed Date, or null if the format is invalid. - */ -const parseAmzDateUtc = (amzDate: string): Date | null => { - const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/); - if (!match) return null; - const [, year, month, day, hour, minute, second] = match; - return new Date( - Date.UTC( - Number.parseInt(year, 10), - Number.parseInt(month, 10) - 1, - Number.parseInt(day, 10), - Number.parseInt(hour, 10), - Number.parseInt(minute, 10), - Number.parseInt(second, 10), - ), - ); -}; - -/** - * Validates that `host` is included in the signed headers list. - * - * AWS SigV4 mandates that `host` is always signed. Reject requests that - * omit it to prevent header injection / replay variants. - * - * @param signedHeaders - The semicolon-separated signed headers string. - * @returns True if `host` is present. - */ -const validateSignedHeaders = (signedHeaders: string): boolean => { - return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host'); -}; - -export const verifySignature = async ( - method: string, - url: string, - headers: Record, - body: string | null, - s3AccessKey: string, - s3SecretKey: string, - region: string, -): Promise => { - const authHeader = headers.authorization; - if (!authHeader?.startsWith('AWS4-HMAC-SHA256')) { - return { isValid: false, credential: null, errorCode: 'AccessDenied' }; - } - - const parsed = parseAuthorizationHeader(authHeader); - if (!parsed) { - return { isValid: false, credential: null, errorCode: 'AccessDenied' }; - } - - if (!timingSafeCompare(parsed.accessKey, s3AccessKey)) { - return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; - } - - if (!timingSafeCompare(parsed.region, region)) { - return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; - } - - // Validate service and termination in credential scope (M2) - if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) { - return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; - } - - // Validate host is in signed headers (LOW/host) - if (!validateSignedHeaders(parsed.signedHeaders)) { - return { isValid: false, credential: null, errorCode: 'AccessDenied' }; - } - - const parsedUrl = new URL(url, 'http://localhost'); - const canonicalUri = normalizeUri(parsedUrl.pathname); - const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams); - - const contentSha256 = headers['x-amz-content-sha256'] || null; - if (contentSha256?.startsWith('STREAMING-')) { - return { isValid: false, credential: null, errorCode: 'NotImplemented' }; - } - - // CRITICAL: Use the x-amz-content-sha256 header value in the canonical - // request because that's what the client signed. The actual body hash is - // verified by verifyBodyHash() after streaming, ensuring integrity without - // breaking SigV4. - const hashedPayload = contentSha256 || (await getHashedPayload(body)); - - const canonicalRequest = buildCanonicalRequest( - method, - canonicalUri, - canonicalQueryString, - parsed.signedHeaders, - headers, - hashedPayload, - ); - - const hashedCanonicalRequest = await sha256Hex(canonicalRequest); - - // M1: Fall back to Date header if x-amz-date is missing - const amzDate = headers['x-amz-date'] || headers.date || ''; - - // H5: Validate request freshness (clock skew / replay protection) - if (amzDate) { - const requestDate = parseAmzDateUtc(amzDate); - if (requestDate) { - const now = Date.now(); - const skew = Math.abs(now - requestDate.getTime()); - if (skew > MAX_CLOCK_SKEW_MS) { - return { isValid: false, credential: null, errorCode: 'RequestExpired' }; - } - } - } - - const dateStamp = parsed.date; - - // M3: Ensure date in credential scope matches x-amz-date - if (amzDate) { - const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD" - if (amzDateStamp !== dateStamp) { - return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; - } - } - - const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`; - - const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`; - - const signingKey = await getSigningKey(s3SecretKey, dateStamp, region); - const expectedSignature = await hmacHex(signingKey, stringToSign); - - if (!timingSafeCompare(expectedSignature, parsed.signature)) { - return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; - } - - return { - isValid: true, - credential: { - accessKey: parsed.accessKey, - date: parsed.date, - region: parsed.region, - service: parsed.service, - }, - }; -}; - -export const verifyPresignedUrl = async ({ - url, - method, - headers, - s3AccessKey, - s3SecretKey, - region, - now = new Date(), -}: VerifyPresignedUrlInput): Promise => { - const parsedUrl = new URL(url); - const searchParams = parsedUrl.searchParams; - - const algorithm = searchParams.get('X-Amz-Algorithm'); - const credential = searchParams.get('X-Amz-Credential'); - const signedHeaders = searchParams.get('X-Amz-SignedHeaders'); - const signature = searchParams.get('X-Amz-Signature'); - const expiresText = searchParams.get('X-Amz-Expires'); - const amzDate = searchParams.get('X-Amz-Date'); - - if ( - algorithm !== 'AWS4-HMAC-SHA256' || - !credential || - !signedHeaders || - !signature || - !expiresText || - !amzDate - ) { - return { isValid: false, credential: null, errorCode: 'AccessDenied' }; - } - - const expires = Number.parseInt(expiresText, 10); - const signedAt = parseAmzDateUtc(amzDate); - if (!Number.isFinite(expires) || expires <= 0 || !signedAt) { - return { isValid: false, credential: null, errorCode: 'AccessDenied' }; - } - // AWS S3 spec limits presigned URLs to 7 days (604800 seconds) - const MAX_PRESIGNED_EXPIRY_SECONDS = 604800; - if ( - now.getTime() > signedAt.getTime() + expires * 1000 || - expires > MAX_PRESIGNED_EXPIRY_SECONDS - ) { - return { isValid: false, credential: null, errorCode: 'AccessDenied' }; - } - - const credParts = credential.split('/'); - if (credParts.length !== 5) { - return { isValid: false, credential: null, errorCode: 'AccessDenied' }; - } - const [accessKey, dateStamp, credentialRegion, service, termination] = credParts; - if ( - !timingSafeCompare(accessKey, s3AccessKey) || - !timingSafeCompare(credentialRegion, region) || - service !== SERVICE || - termination !== TERMINATION - ) { - return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; - } - - // Validate host is in signed headers for presigned URLs too - if (!validateSignedHeaders(signedHeaders)) { - return { isValid: false, credential: null, errorCode: 'AccessDenied' }; - } - - const signedHeaderList = signedHeaders.split(';').filter(Boolean); - const canonicalHeaders = signedHeaderList - .map((headerName) => { - const lower = headerName.toLowerCase(); - const value = lower === 'host' ? headers.host || parsedUrl.host : headers[lower] || ''; - return `${lower}:${value.trim()}\n`; - }) - .join(''); - - const canonicalRequest = `${method}\n${normalizeUri(parsedUrl.pathname)}\n${buildCanonicalQueryString(searchParams, new Set(['X-Amz-Signature']))}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`; - const hashedCanonicalRequest = await sha256Hex(canonicalRequest); - const credentialScope = `${dateStamp}/${region}/${SERVICE}/${TERMINATION}`; - const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`; - const expectedSignature = await hmacHex( - await getSigningKey(s3SecretKey, dateStamp, region), - stringToSign, - ); - - if (!timingSafeCompare(expectedSignature, signature)) { - return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; - } - return { isValid: true, credential: { accessKey, date: dateStamp, region, service } }; -}; - -export const isS3Request = (headers: Record): boolean => { - const auth = headers.authorization || ''; - return auth.startsWith('AWS4-HMAC-SHA256'); -}; - -/** - * Verifies that the actual body SHA-256 matches the `x-amz-content-sha256` - * header from the original request. - * - * This MUST be called AFTER the body has been fully streamed and hashed, - * as a second pass after `verifySignature` (which cannot hash a streaming - * body without consuming it). - * - * @param bodySha256 - The SHA-256 hex digest of the actual body content. - * @param headers - The original request headers. - * @returns An error result on mismatch, or null if the check passes. - */ -export const verifyBodyHash = ( - bodySha256: string, - headers: Record, -): SigV4Result | null => { - const claimedHash = headers['x-amz-content-sha256']; - // If the client sent UNSIGNED-PAYLOAD, skip verification - if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) { - return null; - } - if (claimedHash !== bodySha256) { - return { isValid: false, credential: null, errorCode: 'BadDigest' }; - } - return null; -}; diff --git a/src/utils/s3/headers.ts b/src/utils/s3/headers.ts deleted file mode 100644 index bd53bac..0000000 --- a/src/utils/s3/headers.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { nanoid } from 'nanoid'; - -export const S3_CORS_HEADERS: Record = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS', - 'access-control-allow-headers': [ - 'Authorization', - 'Content-Type', - 'Content-MD5', - 'Range', - 'If-Match', - 'If-None-Match', - 'If-Modified-Since', - 'If-Unmodified-Since', - 'X-Amz-*', - 'x-amz-*', - ].join(', '), - 'access-control-expose-headers': [ - 'Accept-Ranges', - 'Content-Length', - 'Content-Range', - 'Content-Type', - 'ETag', - 'Last-Modified', - 'x-amz-id-2', - 'x-amz-request-id', - ].join(', '), - 'access-control-max-age': '86400', -}; - -export const s3Headers = ( - requestId: string, - extraHeaders: Record = {}, -): Record => ({ - ...S3_CORS_HEADERS, - server: 'AmazonS3', - ...(requestId - ? { - 'x-amz-request-id': requestId, - 'x-amz-id-2': `${requestId}+${nanoid(16)}`, - } - : {}), - ...extraHeaders, -}); - -export const applyS3Headers = (headers: Headers, requestId: string): Headers => { - const result = new Headers(headers); - for (const [key, value] of Object.entries(s3Headers(requestId))) { - result.set(key, value); - } - return result; -}; diff --git a/src/utils/s3/object-stream.ts b/src/utils/s3/object-stream.ts deleted file mode 100644 index 6d0e71f..0000000 --- a/src/utils/s3/object-stream.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { gunzipSync } from 'node:zlib'; -import { applyS3Headers } from './headers'; -import { contentRange, type RangeParseResult } from './range'; - -export interface ObjectPartSource { - telegramFileId: string; - telegramUrl: string; - sizeBytes: number; - partNumber: number; - storedSizeBytes?: number; - compressionAlgorithm?: 'gzip' | null; -} - -export interface ObjectResponseInput { - reqId: string; - contentType: string; - etag: string; - lastModified: Date; - totalSize: number; - parts: ObjectPartSource[]; - range: RangeParseResult; -} - -interface PlannedPart { - part: ObjectPartSource; - relativeStart: number; - relativeEnd: number; -} - -const baseHeaders = (input: ObjectResponseInput, contentLength: number): Headers => { - const headers = new Headers({ - 'content-type': input.contentType, - 'content-length': String(contentLength), - etag: `"${input.etag}"`, - 'last-modified': input.lastModified.toUTCString(), - 'x-amz-request-id': input.reqId, - 'accept-ranges': 'bytes', - 'cache-control': 'public, max-age=31536000', - }); - return headers; -}; - -const planParts = (parts: ObjectPartSource[], start: number, end: number): PlannedPart[] => { - const planned: PlannedPart[] = []; - let offset = 0; - for (const part of parts) { - const partStart = offset; - const partEnd = offset + part.sizeBytes - 1; - offset += part.sizeBytes; - if (end < partStart || start > partEnd) continue; - planned.push({ - part, - relativeStart: Math.max(start, partStart) - partStart, - relativeEnd: Math.min(end, partEnd) - partStart, - }); - } - return planned; -}; - -const streamFromBytes = (bytes: Uint8Array): ReadableStream => - new Response(bytes).body!; - -const TELEGRAM_FETCH_TIMEOUT_MS = 30_000; - -const fetchWholePartBytes = async (telegramUrl: string): Promise => { - const res = await fetch(telegramUrl, { signal: AbortSignal.timeout(TELEGRAM_FETCH_TIMEOUT_MS) }); - if (!res.ok) throw new Error(`Telegram fetch failed: ${res.status}`); - return new Uint8Array(await res.arrayBuffer()); -}; - -const fetchPartBody = async (planned: PlannedPart): Promise> => { - const wantsWholePart = - planned.relativeStart === 0 && planned.relativeEnd === planned.part.sizeBytes - 1; - - if (planned.part.compressionAlgorithm === 'gzip') { - const storedBytes = await fetchWholePartBytes(planned.part.telegramUrl); - const bytes = gunzipSync(storedBytes); - return streamFromBytes(bytes.subarray(planned.relativeStart, planned.relativeEnd + 1)); - } - - const rangeHeader = `bytes=${planned.relativeStart}-${planned.relativeEnd}`; - const fetchOpts: RequestInit = { signal: AbortSignal.timeout(TELEGRAM_FETCH_TIMEOUT_MS) }; - if (!wantsWholePart) { - fetchOpts.headers = { range: rangeHeader }; - } - const res = await fetch(planned.part.telegramUrl, fetchOpts); - if (!res.ok) throw new Error(`Telegram fetch failed: ${res.status}`); - if (wantsWholePart || res.status === 206) return res.body!; - - const bytes = new Uint8Array(await res.arrayBuffer()); - return streamFromBytes(bytes.slice(planned.relativeStart, planned.relativeEnd + 1)); -}; - -const concatPartStreams = (plannedParts: PlannedPart[]): ReadableStream => - new ReadableStream({ - async start(controller) { - try { - for (const planned of plannedParts) { - const stream = await fetchPartBody(planned); - const reader = stream.getReader(); - while (true) { - const { value, done } = await reader.read(); - if (done) break; - if (value) controller.enqueue(value); - } - } - controller.close(); - } catch (error) { - controller.error(error); - } - }, - }); - -export const createGetObjectResponse = async (input: ObjectResponseInput): Promise => { - if (input.range.type === 'invalid') { - throw new Error('createGetObjectResponse received invalid range'); - } - - const start = input.range.type === 'valid' ? input.range.start : 0; - const end = input.range.type === 'valid' ? input.range.end : input.totalSize - 1; - const plannedParts = planParts(input.parts, start, end); - const contentLength = end >= start ? end - start + 1 : 0; - const headers = applyS3Headers(baseHeaders(input, contentLength), input.reqId); - - if (input.range.type === 'valid') { - headers.set('content-range', contentRange(start, end, input.totalSize)); - } - - return new Response(concatPartStreams(plannedParts), { - status: input.range.type === 'valid' ? 206 : 200, - headers, - }); -}; diff --git a/src/utils/s3/range.ts b/src/utils/s3/range.ts deleted file mode 100644 index 3a1f274..0000000 --- a/src/utils/s3/range.ts +++ /dev/null @@ -1,46 +0,0 @@ -export type RangeParseResult = - | { type: 'none' } - | { type: 'valid'; start: number; end: number } - | { type: 'invalid' }; - -const DECIMAL = /^\d+$/; - -export const parseRangeHeader = (rangeHeader: string | null, size: number): RangeParseResult => { - if (!rangeHeader) return { type: 'none' }; - if (!Number.isSafeInteger(size) || size < 0) return { type: 'invalid' }; - if (!rangeHeader.startsWith('bytes=')) return { type: 'invalid' }; - - const spec = rangeHeader.slice('bytes='.length).trim(); - if (spec.includes(',')) return { type: 'invalid' }; - - const dash = spec.indexOf('-'); - if (dash === -1) return { type: 'invalid' }; - - const startText = spec.slice(0, dash).trim(); - const endText = spec.slice(dash + 1).trim(); - if (!startText && !endText) return { type: 'invalid' }; - if (size === 0) return { type: 'invalid' }; - - if (!startText) { - if (!DECIMAL.test(endText)) return { type: 'invalid' }; - const suffixLength = Number.parseInt(endText, 10); - if (suffixLength <= 0) return { type: 'invalid' }; - return { type: 'valid', start: Math.max(size - suffixLength, 0), end: size - 1 }; - } - - if (!DECIMAL.test(startText)) return { type: 'invalid' }; - const start = Number.parseInt(startText, 10); - if (start >= size) return { type: 'invalid' }; - - if (!endText) return { type: 'valid', start, end: size - 1 }; - if (!DECIMAL.test(endText)) return { type: 'invalid' }; - - const requestedEnd = Number.parseInt(endText, 10); - if (requestedEnd < start) return { type: 'invalid' }; - return { type: 'valid', start, end: Math.min(requestedEnd, size - 1) }; -}; - -export const contentRange = (start: number, end: number, size: number): string => - `bytes ${start}-${end}/${size}`; - -export const unsatisfiedContentRange = (size: number): string => `bytes */${size}`; diff --git a/src/utils/s3/virtual-host.ts b/src/utils/s3/virtual-host.ts deleted file mode 100644 index 4cf6c76..0000000 --- a/src/utils/s3/virtual-host.ts +++ /dev/null @@ -1,27 +0,0 @@ -const stripPort = (host: string): string => { - // Handle IPv6: [::1]:8080 -> [::1] - if (host.startsWith('[')) { - const closeBracket = host.indexOf(']'); - return host.slice(0, closeBracket + 1).toLowerCase(); - } - return host.split(':')[0].toLowerCase().replace(/\.$/, ''); -}; - -const isValidBucketLabel = (bucket: string): boolean => - /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) && - !bucket.includes('..') && - !bucket.includes('.-') && - !bucket.includes('-.'); - -export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => { - const normalizedHost = stripPort(host); - for (const domain of domains) { - const normalizedDomain = stripPort(domain); - if (!normalizedDomain || normalizedHost === normalizedDomain) continue; - if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue; - - const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1)); - return isValidBucketLabel(bucket) ? bucket : null; - } - return null; -}; diff --git a/src/utils/s3/xml.ts b/src/utils/s3/xml.ts deleted file mode 100644 index cd255a7..0000000 --- a/src/utils/s3/xml.ts +++ /dev/null @@ -1,311 +0,0 @@ -import { s3Headers } from './headers'; - -const escapeXml = (str: string): string => - str - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); - -const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z'); - -const encodeKey = (value: string, encodingType: string | null = null): string => - encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value); - -// ─────── Bucket operations ─────── - -export const listBucketsXml = ( - buckets: { name: string; createdAt: Date }[], - _requestId: string, -): string => ` - - - ${buckets - .map( - (b) => ` - ${escapeXml(b.name)} - ${isoDate(b.createdAt)} - `, - ) - .join('')} - -`; - -export const bucketVersioningConfigurationXml = - (): string => ` -`; - -// ─────── Object listing ─────── - -export const listBucketResultXml = ( - bucketName: string, - objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[], - prefixes: string[], - isTruncated: boolean, - marker: string | null, - maxKeys: number, - prefix: string, - delimiter: string | null, - nextMarker: string | null, - _requestId: string, - encodingType: string | null = null, -): string => ` - - ${escapeXml(bucketName)} - ${encodeKey(prefix, encodingType)} - ${encodeKey(marker || '', encodingType)} - ${maxKeys} - ${encodeKey(delimiter || '', encodingType)} - ${encodingType ? `${escapeXml(encodingType)}` : ''} - ${isTruncated} - ${objects - .map( - (o) => ` - ${encodeKey(o.key, encodingType)} - ${isoDate(o.lastModified)} - "${o.etag}" - ${o.sizeBytes} - STANDARD - `, - ) - .join('')} - ${prefixes - .map( - (p) => ` - ${encodeKey(p, encodingType)} - `, - ) - .join('')} - ${nextMarker ? `${encodeKey(nextMarker, encodingType)}` : ''} -`; - -export const listBucketV2ResultXml = ( - bucketName: string, - objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[], - prefixes: string[], - isTruncated: boolean, - maxKeys: number, - prefix: string, - delimiter: string | null, - continuationToken: string | null, - nextContinuationToken: string | null, - keyCount: number, - _requestId: string, - encodingType: string | null = null, -): string => ` - - ${escapeXml(bucketName)} - ${encodeKey(prefix, encodingType)} - ${maxKeys} - ${keyCount} - ${delimiter ? `${encodeKey(delimiter, encodingType)}` : ''} - ${encodingType ? `${escapeXml(encodingType)}` : ''} - ${continuationToken ? `${encodeKey(continuationToken, encodingType)}` : ''} - ${isTruncated} - ${objects - .map( - (o) => ` - ${encodeKey(o.key, encodingType)} - ${isoDate(o.lastModified)} - "${o.etag}" - ${o.sizeBytes} - STANDARD - `, - ) - .join('')} - ${prefixes - .map( - (p) => ` - ${encodeKey(p, encodingType)} - `, - ) - .join('')} - ${nextContinuationToken ? `${encodeKey(nextContinuationToken, encodingType)}` : ''} -`; - -// ─────── Multipart ─────── - -export const initiateMultipartUploadXml = ( - bucketName: string, - key: string, - uploadId: string, -): string => ` - - ${escapeXml(bucketName)} - ${escapeXml(key)} - ${uploadId} -`; - -export const listPartsXml = ( - bucketName: string, - key: string, - uploadId: string, - parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[], - maxParts: number, - isTruncated: boolean, - _requestId: string, -): string => ` - - ${escapeXml(bucketName)} - ${escapeXml(key)} - ${uploadId} - ${maxParts} - ${isTruncated} - ${parts - .map( - (p) => ` - ${p.partNumber} - ${isoDate(p.createdAt)} - "${p.etag}" - ${p.sizeBytes} - `, - ) - .join('')} -`; - -export const listMultipartUploadsXml = ( - bucketName: string, - uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[], - maxUploads: number, - isTruncated: boolean, - nextKeyMarker: string | null, - _requestId: string, -): string => ` - - ${escapeXml(bucketName)} - - - ${nextKeyMarker ? `${escapeXml(nextKeyMarker)}` : ''} - ${maxUploads} - ${isTruncated} - ${uploads - .map( - (u) => ` - ${escapeXml(u.key)} - ${u.uploadId} - ${escapeXml(u.initiatedBy || 's3')}${escapeXml(u.initiatedBy || 's3')} - ${escapeXml(u.initiatedBy || 's3')}${escapeXml(u.initiatedBy || 's3')} - STANDARD - ${isoDate(u.initiatedAt)} - `, - ) - .join('')} -`; - -export const completeMultipartUploadXml = ( - bucketName: string, - key: string, - etag: string, - location: string, -): string => ` - - ${escapeXml(location)} - ${escapeXml(bucketName)} - ${escapeXml(key)} - "${etag}" -`; - -// ─────── Delete result ─────── - -export const deleteResultXml = ( - deleted: string[], - errors: { key: string; code: string; message: string }[], -): string => ` - - ${deleted - .map( - (key) => ` - ${escapeXml(key)} - `, - ) - .join('')} - ${errors - .map( - (e) => ` - ${escapeXml(e.key)} - ${e.code} - ${escapeXml(e.message)} - `, - ) - .join('')} -`; - -// ─────── Copy ─────── - -export const copyObjectResultXml = ( - etag: string, - lastModified: Date, -): string => ` - - "${etag}" - ${isoDate(lastModified)} -`; - -// ─────── Error ─────── - -export const s3ErrorXml = ( - code: string, - message: string, - resource: string, - requestId: string, -): string => ` - - ${code} - ${escapeXml(message)} - ${escapeXml(resource)} - ${requestId} - ${requestId} -`; - -export const s3ErrorResponse = ( - code: string, - message: string, - resource: string, - status: number, - requestId: string = '', - extraHeaders: Record = {}, -): Response => - new Response(s3ErrorXml(code, message, resource, requestId), { - status, - headers: s3Headers(requestId, { - 'content-type': 'application/xml', - ...extraHeaders, - }), - }); - -// ─────── DeleteObjects XML parser ─────── - -export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => { - // H9: Use non-greedy match to handle keys containing < character - const keys = Array.from(body.matchAll(/([\s\S]*?)<\/Key>/g), (match) => match[1]); - // Handle whitespace inside element + namespace prefix support - const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body); - return { keys, quiet }; -}; - -// ─────── CompleteMultipartUpload XML parser ─────── - -export interface CompletePart { - partNumber: number; - etag: string; -} - -export const parseCompleteMultipartBody = (body: string): CompletePart[] => { - const parts: CompletePart[] = []; - const partRegex = /[\s\S]*?<\/Part>/g; - const partMatch = body.match(partRegex) || []; - - for (const partXml of partMatch) { - const numMatch = partXml.match(/(\d+)<\/PartNumber>/); - const etagMatch = partXml.match(/"?([^"<\s]+)"?<\/ETag>/); - if (numMatch && etagMatch) { - parts.push({ - partNumber: Number.parseInt(numMatch[1], 10), - etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''), - }); - } - } - - return parts; -}; diff --git a/src/utils/uploadBatcher.ts b/src/utils/uploadBatcher.ts deleted file mode 100644 index 1643e35..0000000 --- a/src/utils/uploadBatcher.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { createReadStream } from 'node:fs'; -import { nanoid } from 'nanoid'; -import { db, files as fileSchema } from '../db'; -import type { NewFile } from '../db/schema'; -import { config } from '../env'; -import { botPool } from '../infrastructure/telegram/bot-pool'; -import { cleanupTempFile } from './file'; -import { createZip, type ZipEntry } from './zip'; - -export type PreparedUpload = { - tempPath: string; - fileHash: string; - sizeBytes: number; - signatureBuffer: Buffer; -}; - -export type UploadedFile = NewFile & { - createdAt: Date; - updatedAt: Date; -}; - -export type BatchUploadItem = { - prepared: PreparedUpload; - fileName: string; - mimeType: string; - fileType: string; -}; - -type PendingUpload = BatchUploadItem & { - resolve: (file: UploadedFile) => void; - reject: (error: unknown) => void; -}; - -const BATCH_WINDOW_MS = 2000; - -let pendingUploads: PendingUpload[] = []; -let flushTimer: ReturnType | null = null; - -const buildUploadedFile = ( - item: BatchUploadItem, - entry: ZipEntry, - archive: { - telegramFileId: string; - telegramFileUniqueId: string; - storageMessageId: number; - fileName: string; - sizeBytes: number; - }, -): UploadedFile => ({ - publicId: nanoid(), - telegramFileId: archive.telegramFileId, - telegramFileUniqueId: archive.telegramFileUniqueId, - storageChatId: config.storageChatId, - storageMessageId: archive.storageMessageId, - fileName: item.fileName, - mimeType: item.mimeType || 'application/octet-stream', - sizeBytes: item.prepared.sizeBytes, - fileType: item.fileType, - uploaderId: 0, - fileHash: item.prepared.fileHash, - archiveTelegramFileId: archive.telegramFileId, - archiveStorageMessageId: archive.storageMessageId, - archiveFileName: archive.fileName, - archiveEntryName: entry.entryName, - archiveMimeType: 'application/zip', - archiveSizeBytes: archive.sizeBytes, - createdAt: new Date(), - updatedAt: new Date(), -}); - -const flushUploads = async (): Promise => { - if (flushTimer) { - clearTimeout(flushTimer); - flushTimer = null; - } - - const batch = pendingUploads; - pendingUploads = []; - if (batch.length === 0) return; - - let zipTempPath: string | null = null; - - try { - const zip = await createZip( - batch.map((item) => ({ tempPath: item.prepared.tempPath, fileName: item.fileName })), - ); - zipTempPath = zip.tempPath; - const archiveFileName = `filedrop-${nanoid()}.zip`; - const archiveResult = await botPool.forwardToStorage( - createReadStream(zip.tempPath), - archiveFileName, - 'document', - ); - - const uploadedFiles = batch.map((item, index) => - buildUploadedFile(item, zip.entries[index], { - telegramFileId: archiveResult.telegramFileId, - telegramFileUniqueId: archiveResult.telegramFileUniqueId, - storageMessageId: archiveResult.storageMessageId, - fileName: archiveFileName, - sizeBytes: zip.sizeBytes, - }), - ); - - await db.insert(fileSchema).values(uploadedFiles); - - for (let i = 0; i < batch.length; i++) { - batch[i].resolve(uploadedFiles[i]); - } - } catch (error) { - for (const item of batch) { - item.reject(error); - } - } finally { - await Promise.all(batch.map((item) => cleanupTempFile(item.prepared.tempPath))); - if (zipTempPath) await cleanupTempFile(zipTempPath); - // Reschedule timer if new items arrived during async processing - if (pendingUploads.length > 0 && !flushTimer) { - flushTimer = setTimeout(() => { - void flushUploads(); - }, BATCH_WINDOW_MS); - } - } -}; - -const getPendingSize = (): number => - pendingUploads.reduce((total, item) => total + item.prepared.sizeBytes, 0); - -export const enqueuePreparedUpload = (item: BatchUploadItem): Promise => { - return new Promise((resolve, reject) => { - pendingUploads.push({ ...item, resolve, reject }); - - if (!flushTimer) { - flushTimer = setTimeout(() => { - void flushUploads(); - }, BATCH_WINDOW_MS); - } - - if ( - pendingUploads.length >= config.batchMaxItems || - getPendingSize() >= config.batchMaxSizeBytes - ) { - void flushUploads(); - } - }); -}; - -export const flushPendingUploads = async (): Promise => { - await flushUploads(); -}; - -export const getPendingUploadCount = (): number => pendingUploads.length; diff --git a/src/utils/zip.ts b/src/utils/zip.ts deleted file mode 100644 index ac3937e..0000000 --- a/src/utils/zip.ts +++ /dev/null @@ -1,292 +0,0 @@ -import { once } from 'node:events'; -import { createReadStream, createWriteStream } from 'node:fs'; -import { open, stat, unlink } from 'node:fs/promises'; -import { basename } from 'node:path'; -import { finished } from 'node:stream/promises'; -import { nanoid } from 'nanoid'; - -export type ZipInputFile = { - tempPath: string; - fileName: string; -}; - -export type ZipEntry = { - fileName: string; - entryName: string; - crc32: number; - compressedSize: number; - uncompressedSize: number; - localHeaderOffset: number; -}; - -export type CreatedZip = { - tempPath: string; - sizeBytes: number; - fileHash: string; - entries: ZipEntry[]; -}; - -const CRC32_TABLE = new Uint32Array(256).map((_, index) => { - let value = index; - for (let bit = 0; bit < 8; bit++) { - value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1; - } - return value >>> 0; -}); - -const updateCrc32 = (crc: number, chunk: Buffer): number => { - let value = crc; - for (const byte of chunk) { - value = CRC32_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8); - } - return value >>> 0; -}; - -const dosDateTime = (date = new Date()): { date: number; time: number } => { - const year = Math.max(date.getFullYear(), 1980); - return { - time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2), - date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(), - }; -}; - -const writeUInt16 = (value: number): Buffer => { - const buffer = Buffer.allocUnsafe(2); - buffer.writeUInt16LE(value & 0xffff, 0); - return buffer; -}; - -const writeUInt32 = (value: number): Buffer => { - const buffer = Buffer.allocUnsafe(4); - buffer.writeUInt32LE(value >>> 0, 0); - return buffer; -}; - -const writeChunk = async ( - writer: ReturnType, - chunk: Buffer, -): Promise => { - if (!writer.write(chunk)) { - await once(writer, 'drain'); - } -}; - -const finishWriter = async (writer: ReturnType): Promise => { - writer.end(); - await finished(writer); -}; - -export const sanitizeZipEntryName = (fileName: string, usedNames = new Set()): string => { - const cleaned = basename(fileName) - .replace(/[\\/]+/g, '_') - .replace(/\.\.+/g, '.') - .trim(); - const fallback = cleaned && cleaned !== '.' && cleaned !== '..' ? cleaned : 'file'; - const dotIndex = fallback.lastIndexOf('.'); - const baseName = dotIndex > 0 ? fallback.slice(0, dotIndex) : fallback; - const extension = dotIndex > 0 ? fallback.slice(dotIndex) : ''; - let candidate = fallback; - let counter = 1; - - while (usedNames.has(candidate)) { - candidate = `${baseName}-${counter}${extension}`; - counter++; - } - - usedNames.add(candidate); - return candidate; -}; - -const calculateFileCrc32 = async (tempPath: string): Promise => { - let crc = 0xffffffff; - const reader = createReadStream(tempPath); - for await (const chunk of reader) { - crc = updateCrc32(crc, chunk as Buffer); - } - return (crc ^ 0xffffffff) >>> 0; -}; - -export const createZip = async (files: ZipInputFile[]): Promise => { - const tempPath = `/tmp/filedrop-${nanoid()}.zip`; - const writer = createWriteStream(tempPath); - const hasher = new Bun.CryptoHasher('sha256'); - const entries: ZipEntry[] = []; - const usedNames = new Set(); - let offset = 0; - - const writeHashed = async (chunk: Buffer): Promise => { - hasher.update(chunk); - await writeChunk(writer, chunk); - offset += chunk.byteLength; - }; - - try { - for (const file of files) { - const entryName = sanitizeZipEntryName(file.fileName, usedNames); - const nameBuffer = Buffer.from(entryName); - const fileStats = await stat(file.tempPath); - const { date, time } = dosDateTime(); - const localHeaderOffset = offset; - const crc32 = await calculateFileCrc32(file.tempPath); - - const localHeader = Buffer.concat([ - writeUInt32(0x04034b50), - writeUInt16(20), - writeUInt16(0), - writeUInt16(0), - writeUInt16(time), - writeUInt16(date), - writeUInt32(crc32), - writeUInt32(fileStats.size), - writeUInt32(fileStats.size), - writeUInt16(nameBuffer.byteLength), - writeUInt16(0), - nameBuffer, - ]); - - await writeHashed(localHeader); - const reader = createReadStream(file.tempPath); - for await (const chunk of reader) { - await writeHashed(chunk as Buffer); - } - - entries.push({ - fileName: file.fileName, - entryName, - crc32, - compressedSize: fileStats.size, - uncompressedSize: fileStats.size, - localHeaderOffset, - }); - } - - const centralDirectoryOffset = offset; - for (const entry of entries) { - const nameBuffer = Buffer.from(entry.entryName); - const { date, time } = dosDateTime(); - await writeHashed( - Buffer.concat([ - writeUInt32(0x02014b50), - writeUInt16(20), - writeUInt16(20), - writeUInt16(0), - writeUInt16(0), - writeUInt16(time), - writeUInt16(date), - writeUInt32(entry.crc32), - writeUInt32(entry.compressedSize), - writeUInt32(entry.uncompressedSize), - writeUInt16(nameBuffer.byteLength), - writeUInt16(0), - writeUInt16(0), - writeUInt16(0), - writeUInt16(0), - writeUInt32(0), - writeUInt32(entry.localHeaderOffset), - nameBuffer, - ]), - ); - } - - const centralDirectorySize = offset - centralDirectoryOffset; - await writeHashed( - Buffer.concat([ - writeUInt32(0x06054b50), - writeUInt16(0), - writeUInt16(0), - writeUInt16(entries.length), - writeUInt16(entries.length), - writeUInt32(centralDirectorySize), - writeUInt32(centralDirectoryOffset), - writeUInt16(0), - ]), - ); - - await finishWriter(writer); - - return { - tempPath, - sizeBytes: offset, - fileHash: hasher.digest('hex'), - entries, - }; - } catch (error) { - writer.destroy(); - await unlink(tempPath).catch(() => {}); - throw error; - } -}; - -export const extractZipEntry = async ( - zipBuffer: Buffer, - entryName: string, -): Promise => { - let offset = 0; - - while (offset + 30 <= zipBuffer.byteLength) { - const signature = zipBuffer.readUInt32LE(offset); - if (signature !== 0x04034b50) break; - - const compressionMethod = zipBuffer.readUInt16LE(offset + 8); - const compressedSize = zipBuffer.readUInt32LE(offset + 18); - const fileNameLength = zipBuffer.readUInt16LE(offset + 26); - const extraLength = zipBuffer.readUInt16LE(offset + 28); - const nameStart = offset + 30; - const nameEnd = nameStart + fileNameLength; - const dataStart = nameEnd + extraLength; - const dataEnd = dataStart + compressedSize; - const currentName = zipBuffer.subarray(nameStart, nameEnd).toString(); - - if (currentName === entryName) { - if (compressionMethod !== 0) return null; - return zipBuffer.subarray(dataStart, dataEnd); - } - - offset = dataEnd; - } - - return null; -}; - -export type LocatedZipEntry = { - start: number; - length: number; -}; - -export const locateZipEntry = async ( - zipPath: string, - entryName: string, -): Promise => { - const handle = await open(zipPath, 'r'); - let offset = 0; - - try { - const header = Buffer.alloc(30); - - while (true) { - const { bytesRead } = await handle.read(header, 0, header.byteLength, offset); - if (bytesRead < header.byteLength) return null; - - const signature = header.readUInt32LE(0); - if (signature !== 0x04034b50) return null; - - const compressionMethod = header.readUInt16LE(8); - const compressedSize = header.readUInt32LE(18); - const fileNameLength = header.readUInt16LE(26); - const extraLength = header.readUInt16LE(28); - const nameBuffer = Buffer.alloc(fileNameLength); - const nameOffset = offset + 30; - await handle.read(nameBuffer, 0, fileNameLength, nameOffset); - - const dataStart = nameOffset + fileNameLength + extraLength; - if (nameBuffer.toString() === entryName) { - if (compressionMethod !== 0) return null; - return { start: dataStart, length: compressedSize }; - } - - offset = dataStart + compressedSize; - } - } finally { - await handle.close(); - } -};