diff --git a/src/db/multipart.ts b/src/db/multipart.ts index d6e3c95..3ed474c 100644 --- a/src/db/multipart.ts +++ b/src/db/multipart.ts @@ -9,6 +9,7 @@ export interface MultipartUpload { initiatedAt: Date; status: string; initiatedBy: string; + contentType: string | null; } export interface MultipartPart { @@ -27,17 +28,20 @@ export const createMultipartUpload = async ( bucketId: string, s3Key: string, initiatedBy: string, + contentType?: string | null, ): Promise => { const uploadId = nanoid(32); await db.execute( - sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy})`, + contentType + ? sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by, content_type) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy}, ${contentType})` + : sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy})`, ); return uploadId; }; export const findMultipartUpload = async (uploadId: string): Promise => { const result = (await db.execute( - sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status FROM multipart_uploads WHERE upload_id = ${uploadId} AND status = 'in_progress'`, + sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, content_type FROM multipart_uploads WHERE upload_id = ${uploadId} AND status = 'in_progress'`, )) as unknown as Record[]; if (result.length === 0) return null; const r = result[0]!; @@ -48,6 +52,7 @@ export const findMultipartUpload = async (uploadId: string): Promise = }; export const abortMultipartUpload = async (uploadId: string): Promise => { + // H7: FK cascade only fires on DELETE, not UPDATE. Delete parts explicitly + // before updating the upload status. + await db.execute(sql`DELETE FROM multipart_parts WHERE upload_id = ${uploadId}`); await db.execute( sql`UPDATE multipart_uploads SET status = 'aborted' WHERE upload_id = ${uploadId}`, ); - // Parts are cascade-deleted by FK }; export const insertMultipartPart = async ( @@ -98,6 +105,7 @@ const mapRowToMultipartUpload = (r: Record): MultipartUpload => initiatedAt: new Date(r.initiated_at as string), status: r.status as string, initiatedBy: (r.initiated_by as string | null) || '', + contentType: (r.content_type as string | null) || null, }); export const listMultipartUploadsByBucket = async ( diff --git a/src/db/schema.ts b/src/db/schema.ts index 275e58a..4e00666 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,4 +1,4 @@ -import type { InferInsertModel, InferSelectModel } from 'drizzle-orm'; +import { eq, type InferInsertModel, type InferSelectModel } from 'drizzle-orm'; import { bigint, boolean, @@ -7,37 +7,48 @@ import { serial, text, timestamp, + uniqueIndex, uuid, } from 'drizzle-orm/pg-core'; -export const files = pgTable('files', { - id: uuid('id').primaryKey().defaultRandom(), - publicId: text('public_id').unique().notNull(), - telegramFileId: text('telegram_file_id').notNull(), - telegramFileUniqueId: text('telegram_file_unique_id').notNull(), - storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(), - storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(), - fileName: text('file_name').notNull(), - mimeType: text('mime_type').notNull(), - sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(), - fileType: text('file_type').notNull(), - uploaderId: bigint('uploader_id', { mode: 'number' }).notNull(), - fileHash: text('file_hash'), - archiveTelegramFileId: text('archive_telegram_file_id'), - archiveStorageMessageId: bigint('archive_storage_message_id', { mode: 'number' }), - archiveFileName: text('archive_file_name'), - archiveEntryName: text('archive_entry_name'), - archiveMimeType: text('archive_mime_type'), - archiveSizeBytes: bigint('archive_size_bytes', { mode: 'number' }), - bucketId: text('bucket_id'), - s3Key: text('s3_key'), - storageBackend: text('storage_backend').default('telegram'), - isDeleted: boolean('is_deleted').default(false), - multipartUploadId: text('multipart_upload_id'), - partCount: integer('part_count'), - createdAt: timestamp('created_at').defaultNow().notNull(), - updatedAt: timestamp('updated_at').defaultNow().notNull(), -}); +export const files = pgTable( + 'files', + { + id: uuid('id').primaryKey().defaultRandom(), + publicId: text('public_id').unique().notNull(), + telegramFileId: text('telegram_file_id').notNull(), + telegramFileUniqueId: text('telegram_file_unique_id').notNull(), + storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(), + storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(), + fileName: text('file_name').notNull(), + mimeType: text('mime_type').notNull(), + sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(), + fileType: text('file_type').notNull(), + uploaderId: bigint('uploader_id', { mode: 'number' }).notNull(), + fileHash: text('file_hash'), + archiveTelegramFileId: text('archive_telegram_file_id'), + archiveStorageMessageId: bigint('archive_storage_message_id', { mode: 'number' }), + archiveFileName: text('archive_file_name'), + archiveEntryName: text('archive_entry_name'), + archiveMimeType: text('archive_mime_type'), + archiveSizeBytes: bigint('archive_size_bytes', { mode: 'number' }), + bucketId: text('bucket_id'), + s3Key: text('s3_key'), + storageBackend: text('storage_backend').default('telegram'), + isDeleted: boolean('is_deleted').default(false), + multipartUploadId: text('multipart_upload_id'), + partCount: integer('part_count'), + createdAt: timestamp('created_at').defaultNow().notNull(), + updatedAt: timestamp('updated_at').defaultNow().notNull(), + }, + (table) => ({ + // H2: Prevent TOCTOU race on concurrent PUT — only one active (non-deleted) + // object per (bucket_id, s3_key) pair. + activeObjectIdx: uniqueIndex('active_object_idx') + .on(table.bucketId, table.s3Key) + .where(eq(table.isDeleted, false)), + }), +); export const fileParts = pgTable('file_parts', { id: serial('id').primaryKey(), diff --git a/src/interfaces/http/controllers/s3-controller.ts b/src/interfaces/http/controllers/s3-controller.ts index 846004a..dd67415 100644 --- a/src/interfaces/http/controllers/s3-controller.ts +++ b/src/interfaces/http/controllers/s3-controller.ts @@ -18,6 +18,7 @@ import { listMultipartUploadsByBucket, } from '../../../db/multipart'; import type { File } from '../../../db/schema'; +import type { ForwardResult } from '../../../domain/ports/telegram-service'; import { botPool } from '../../../infrastructure/telegram/bot-pool'; import logger from '../../../shared/logger/index'; import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file'; @@ -25,7 +26,7 @@ import { createChunkedObjectResponse, storeFileInTelegramChunks, } from '../../../utils/chunked-storage'; -import { verifyPresignedUrl, verifySignature } from '../../../utils/s3/auth'; +import { verifyBodyHash, verifyPresignedUrl, verifySignature } from '../../../utils/s3/auth'; import { S3_CORS_HEADERS, s3Headers } from '../../../utils/s3/headers'; import { createGetObjectResponse, type ObjectPartSource } from '../../../utils/s3/object-stream'; import { parseRangeHeader, unsatisfiedContentRange } from '../../../utils/s3/range'; @@ -71,7 +72,19 @@ const s3Response = ( status: number, reqId: string, extraHeaders: Record = {}, -): Response => new Response(body, { status, headers: s3Headers(reqId, extraHeaders) }); +): Response => { + // Add content-type for empty 200-series responses (not 204 which has no body) + if ( + body === null && + status >= 200 && + status < 300 && + status !== 204 && + !extraHeaders['content-type'] + ) { + extraHeaders['content-type'] = 'application/xml'; + } + return new Response(body, { status, headers: s3Headers(reqId, extraHeaders) }); +}; /** * Builds an S3 OPTIONS preflight response with CORS headers. @@ -93,7 +106,12 @@ const parseS3Path = (pathname: string): { bucket: string | null; key: string | n const parts = pathname.split('/').filter(Boolean); if (parts.length === 0) return { bucket: null, key: null }; if (parts.length === 1) return { bucket: parts[0], key: null }; - return { bucket: parts[0], key: parts.slice(1).join('/') }; + // Decode URI components to match virtual-hosted behavior (H10) + const key = parts + .slice(1) + .map((segment) => decodeURIComponent(segment)) + .join('/'); + return { bucket: parts[0], key }; }; /** @@ -240,7 +258,7 @@ export const handleS3Request = async ( // ── Object-level: Multipart operations ── if (searchParams.has('uploads') && method === 'POST') { - return handleCreateMultipartUpload(bucket, key, searchParams, reqId); + return handleCreateMultipartUpload(bucket, key, searchParams, headers, reqId); } if (searchParams.has('uploadId') && searchParams.has('partNumber') && method === 'PUT') { return handleUploadPart(bucket, key, searchParams, req, reqId); @@ -258,7 +276,7 @@ export const handleS3Request = async ( // ── Standard object operations ── if (method === 'GET') return handleGetObject(bucket, key, searchParams, headers, reqId); - if (method === 'HEAD') return handleHeadObject(bucket, key, reqId); + if (method === 'HEAD') return handleHeadObject(bucket, key, headers, reqId); if (method === 'PUT') return handlePutObject(bucket, key, searchParams, headers, req, reqId); if (method === 'DELETE') return handleDeleteObject(bucket, key, reqId); @@ -305,7 +323,13 @@ const handleListBuckets = async (reqId: string): Promise => { * @returns An S3 XML response indicating success or failure. */ const handleCreateBucket = async (bucketName: string, reqId: string): Promise => { - if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucketName)) { + // M13: Stricter bucket validation — no consecutive dots, no IP format, no xn-- prefix + if ( + !/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucketName) || + bucketName.includes('..') || + /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(bucketName) || + bucketName.startsWith('xn--') + ) { return s3ErrorResponse( 'InvalidBucketName', 'The specified bucket is not valid.', @@ -451,6 +475,46 @@ const handleGetObject = async ( reqId, ); + // H3: Conditional headers — If-Match / If-None-Match + const etag = `"${file.fileHash || nanoid(16)}"`; + const ifMatch = headers['if-match']; + if (ifMatch && ifMatch !== '*' && ifMatch !== etag) { + return s3ErrorResponse( + 'PreconditionFailed', + 'At least one of the pre-conditions you specified did not hold.', + `/${bucket}/${key}`, + 412, + reqId, + ); + } + const ifNoneMatch = headers['if-none-match']; + if (ifNoneMatch && ifNoneMatch === etag) { + return new Response(null, { status: 304 }); + } + + // H3: Conditional headers — If-Modified-Since / If-Unmodified-Since + const lastModified = file.createdAt instanceof Date ? file.createdAt : new Date(file.createdAt); + const ifModifiedSince = headers['if-modified-since']; + if (ifModifiedSince) { + const since = new Date(ifModifiedSince); + if (!Number.isNaN(since.getTime()) && lastModified.getTime() <= since.getTime()) { + return new Response(null, { status: 304 }); + } + } + const ifUnmodifiedSince = headers['if-unmodified-since']; + if (ifUnmodifiedSince) { + const since = new Date(ifUnmodifiedSince); + if (!Number.isNaN(since.getTime()) && lastModified.getTime() > since.getTime()) { + return s3ErrorResponse( + 'PreconditionFailed', + 'At least one of the pre-conditions you specified did not hold.', + `/${bucket}/${key}`, + 412, + reqId, + ); + } + } + // Chunked storage object if (file.storageBackend === 'chunked') { const totalSize = Number(file.sizeBytes); @@ -488,7 +552,7 @@ const handleGetObject = async ( // Regular Telegram object const fileInfo = await botPool.getFileInfo(file.telegramFileId); - const redirectUrl = `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`; + const telegramUrl = `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`; const totalSize = file.sizeBytes; const range = parseRangeHeader(headers.range || null, totalSize); @@ -505,14 +569,13 @@ const handleGetObject = async ( ); } - if (!config.proxyS3Get) { - // Legacy 302 redirect path (when proxy is disabled) - return s3Response(null, 302, reqId, { location: redirectUrl }); - } + // H1: Always proxy S3 GETs to avoid leaking the Telegram bot token + // in redirect URLs. The 302 redirect path is removed because the + // URL contains the bot_token — exposing it to clients is a security risk. const part: ObjectPartSource = { telegramFileId: file.telegramFileId, - telegramUrl: redirectUrl, + telegramUrl, sizeBytes: file.sizeBytes, partNumber: 1, }; @@ -601,9 +664,7 @@ const handleGetMultipartObject = async ( }); } - if (!config.proxyS3Get) { - return s3Response(null, 302, reqId, { location: sources[0]?.telegramUrl ?? '' }); - } + // H1: Always proxy — never expose bot token in redirect URL try { return await createGetObjectResponse({ @@ -638,7 +699,12 @@ const handleGetMultipartObject = async ( * @param reqId - The request identifier for S3 headers. * @returns An S3 response with object metadata headers. */ -const handleHeadObject = async (bucket: string, key: string, reqId: string): Promise => { +const handleHeadObject = async ( + bucket: string, + key: string, + headers: Record, + reqId: string, +): Promise => { const bucketRecord = await findBucketByName(bucket); if (!bucketRecord) return s3ErrorResponse( @@ -659,6 +725,46 @@ const handleHeadObject = async (bucket: string, key: string, reqId: string): Pro reqId, ); + // H3: Conditional headers for HEAD — If-Match / If-None-Match + const etag = `"${file.fileHash || nanoid(16)}"`; + const ifMatch = headers['if-match']; + if (ifMatch && ifMatch !== '*' && ifMatch !== etag) { + return s3ErrorResponse( + 'PreconditionFailed', + 'At least one of the pre-conditions you specified did not hold.', + `/${bucket}/${key}`, + 412, + reqId, + ); + } + const ifNoneMatch = headers['if-none-match']; + if (ifNoneMatch && ifNoneMatch === etag) { + return new Response(null, { status: 304 }); + } + + // H3: Conditional headers for HEAD — If-Modified-Since / If-Unmodified-Since + const lastModified = file.createdAt instanceof Date ? file.createdAt : new Date(file.createdAt); + const ifModifiedSince = headers['if-modified-since']; + if (ifModifiedSince) { + const since = new Date(ifModifiedSince); + if (!Number.isNaN(since.getTime()) && lastModified.getTime() <= since.getTime()) { + return new Response(null, { status: 304 }); + } + } + const ifUnmodifiedSince = headers['if-unmodified-since']; + if (ifUnmodifiedSince) { + const since = new Date(ifUnmodifiedSince); + if (!Number.isNaN(since.getTime()) && lastModified.getTime() > since.getTime()) { + return s3ErrorResponse( + 'PreconditionFailed', + 'At least one of the pre-conditions you specified did not hold.', + `/${bucket}/${key}`, + 412, + reqId, + ); + } + } + return s3Response(null, 200, reqId, { 'content-type': file.mimeType, 'content-length': String(file.sizeBytes), @@ -667,6 +773,7 @@ const handleHeadObject = async (bucket: string, key: string, reqId: string): Pro file.createdAt instanceof Date ? file.createdAt.toUTCString() : new Date().toUTCString(), 'accept-ranges': 'bytes', 'cache-control': 'public, max-age=31536000', + 'x-amz-version-id': 'null', }); }; @@ -790,6 +897,31 @@ const handlePutObject = async ( const contentType = headers['content-type'] || 'application/octet-stream'; const streamed = await streamBodyToTemp(req.body); + // H4: Verify body hash against x-amz-content-sha256 + const bodyHashError = verifyBodyHash(streamed.fileHash, headers); + if (bodyHashError) { + await cleanupTempFile(streamed.tempPath); + return s3ErrorResponse( + bodyHashError.errorCode || 'BadDigest', + 'The Content-MD5 or x-amz-content-sha256 you specified did not match what we received.', + `/${bucket}/${key}`, + 400, + reqId, + ); + } + + // M12: Reject oversized bodies + if (streamed.sizeBytes > config.maxRequestBodyBytes) { + await cleanupTempFile(streamed.tempPath); + return s3ErrorResponse( + 'EntityTooLarge', + 'Your proposed upload exceeds the maximum allowed object size.', + `/${bucket}/${key}`, + 400, + reqId, + ); + } + // Idempotent PUT: if the object already exists, skip upload const existing = await findFileByBucketAndKey(bucketRecord.id, key); if (existing) { @@ -948,9 +1080,11 @@ const handleCopyObject = async ( } // Conditional copy: if-match / if-none-match checks + // M9: Use stable etag (telegramFileId fallback when fileHash is null) + const sourceEtag = sourceFile.fileHash || sourceFile.telegramFileId; const ifMatch = headers['x-amz-copy-source-if-match']; const ifNoneMatch = headers['x-amz-copy-source-if-none-match']; - if (ifMatch && sourceFile.fileHash && ifMatch !== `"${sourceFile.fileHash}"`) { + if (ifMatch && ifMatch !== '*' && ifMatch !== `"${sourceEtag}"`) { return s3ErrorResponse( 'PreconditionFailed', 'The preconditions you specified did not hold.', @@ -959,7 +1093,7 @@ const handleCopyObject = async ( reqId, ); } - if (ifNoneMatch && sourceFile.fileHash && ifNoneMatch === `"${sourceFile.fileHash}"`) { + if (ifNoneMatch && ifNoneMatch === `"${sourceEtag}"`) { return s3ErrorResponse( 'PreconditionFailed', 'The preconditions you specified did not hold.', @@ -1050,12 +1184,30 @@ const handleDeleteObjects = async ( ); const { keys, quiet } = parseDeleteObjectsBody(body); + + // M11: S3 spec limits batch delete to 1000 keys + if (keys.length > 1000) { + return s3ErrorResponse( + 'MalformedXML', + 'The XML you provided was not well-formed or did not validate against our published schema. Max 1000 keys per request.', + `/${bucket}`, + 400, + reqId, + ); + } + const deletedKeys: string[] = []; + const errors: Array<{ key: string; code: string; message: string }> = []; for (const key of keys) { const ok = await softDeleteFile(bucketRecord.id, key); - if (ok) deletedKeys.push(key); + if (ok) { + deletedKeys.push(key); + } else { + // Per S3 spec, deleting a non-existent key is idempotent — report as success + deletedKeys.push(key); + } } - const xml = quiet ? deleteResultXml([], []) : deleteResultXml(deletedKeys, []); + const xml = quiet ? deleteResultXml([], []) : deleteResultXml(deletedKeys, errors); return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; @@ -1212,6 +1364,7 @@ const handleCreateMultipartUpload = async ( bucket: string, key: string, _searchParams: URLSearchParams, + headers: Record, reqId: string, ): Promise => { const bucketRecord = await findBucketByName(bucket); @@ -1224,7 +1377,8 @@ const handleCreateMultipartUpload = async ( reqId, ); - const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3'); + const contentType = headers['content-type'] || null; + const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3', contentType); const xml = initiateMultipartUploadXml(bucket, key, uploadId); return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); @@ -1250,8 +1404,15 @@ const handleUploadPart = async ( reqId: string, ): Promise => { const uploadId = searchParams.get('uploadId')!; - const partNumber = Number.parseInt(searchParams.get('partNumber')!, 10); - if (partNumber < 1 || partNumber > 10000) { + // M14: Validate partNumber is actually an integer, not NaN + const partNumberRaw = searchParams.get('partNumber')!; + const partNumber = Number.parseInt(partNumberRaw, 10); + if ( + !Number.isFinite(partNumber) || + !Number.isInteger(partNumber) || + partNumber < 1 || + partNumber > 10000 + ) { return s3ErrorResponse( 'InvalidArgument', 'Part number must be an integer between 1 and 10000', @@ -1315,12 +1476,18 @@ const handleUploadPart = async ( ); } - const forwardResult = await botPool.forwardToStorage( - createReadStream(tempPath), - `mp-${uploadId}-part-${partNumber}`, - 'document', - ); - + // M4: Ensure temp file cleanup even if forwardToStorage fails + let forwardResult: ForwardResult; + try { + forwardResult = await botPool.forwardToStorage( + createReadStream(tempPath), + `mp-${uploadId}-part-${partNumber}`, + 'document', + ); + } catch (error) { + await cleanupTempFile(tempPath); + throw error; + } await cleanupTempFile(tempPath); const etag = hasher.digest('hex'); @@ -1359,7 +1526,8 @@ const handleCompleteMultipartUpload = async ( ): Promise => { const uploadId = searchParams.get('uploadId')!; const multipart = await findMultipartUpload(uploadId); - if (!multipart) { + // H5: Verify both upload exists AND key matches (consistent with handleUploadPart) + if (!multipart || multipart.s3Key !== key) { return s3ErrorResponse( 'NoSuchUpload', 'The specified upload does not exist.', @@ -1384,6 +1552,7 @@ const handleCompleteMultipartUpload = async ( ); } + // H8: Verify count AND part numbers AND etags match stored parts if (parts.length !== storedParts.length) { return s3ErrorResponse( 'InvalidPart', @@ -1394,11 +1563,34 @@ const handleCompleteMultipartUpload = async ( ); } - const totalSize = storedParts.reduce((sum, p) => sum + p.sizeBytes, 0); + // Build a map for O(1) part number lookup + const storedByNumber = new Map(); + for (const sp of storedParts) { + storedByNumber.set(sp.partNumber, sp); + } + + for (const clientPart of parts) { + const stored = storedByNumber.get(clientPart.partNumber); + if (!stored || stored.etag !== clientPart.etag) { + return s3ErrorResponse( + 'InvalidPart', + 'One or more specified parts could not be found. The etag or part number does not match.', + `/${bucket}/${key}`, + 400, + reqId, + ); + } + } + + const totalSize = storedParts.reduce((sum, p) => sum + Number(p.sizeBytes), 0); + const combinedEtag = storedParts.map((p) => p.etag).join('-'); const publicId = nanoid(); const { db, files: fileSchema } = await import('../../../db/index'); + // M7: Use stored content-type from the multipart record if available + const mimeType = multipart.contentType || 'application/octet-stream'; + await db.insert(fileSchema).values({ publicId, telegramFileId: storedParts[0]!.telegramFileId, @@ -1406,7 +1598,7 @@ const handleCompleteMultipartUpload = async ( storageChatId: config.storageChatId, storageMessageId: storedParts[0]!.storageMessageId, fileName: key.split('/').pop() || 'file', - mimeType: 'application/octet-stream', + mimeType, sizeBytes: totalSize, fileType: 'document', uploaderId: 0, @@ -1422,7 +1614,6 @@ const handleCompleteMultipartUpload = async ( await completeMultipartUpload(uploadId); const location = `${config.baseUrl}/${bucket}/${key}`; - const combinedEtag = storedParts.map((p) => p.etag).join('-'); const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location); return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); diff --git a/src/interfaces/s3/headers.ts b/src/interfaces/s3/headers.ts index 809527d..7b18674 100644 --- a/src/interfaces/s3/headers.ts +++ b/src/interfaces/s3/headers.ts @@ -1,44 +1,10 @@ -export const S3_CORS_HEADERS: Record = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS', - 'access-control-allow-headers': [ - 'Authorization', - 'Content-Type', - 'Content-MD5', - 'Range', - 'If-Match', - 'If-None-Match', - 'If-Modified-Since', - 'If-Unmodified-Since', - 'X-Amz-*', - 'x-amz-*', - ].join(', '), - 'access-control-expose-headers': [ - 'Accept-Ranges', - 'Content-Length', - 'Content-Range', - 'Content-Type', - 'ETag', - 'Last-Modified', - 'x-amz-id-2', - 'x-amz-request-id', - ].join(', '), - 'access-control-max-age': '86400', -}; - -export const s3Headers = ( - requestId: string, - extraHeaders: Record = {}, -): Record => ({ - ...S3_CORS_HEADERS, - ...(requestId ? { 'x-amz-request-id': requestId, 'x-amz-id-2': requestId } : {}), - ...extraHeaders, -}); - -export const applyS3Headers = (headers: Headers, requestId: string): Headers => { - const result = new Headers(headers); - for (const [key, value] of Object.entries(s3Headers(requestId))) { - result.set(key, value); - } - return result; -}; +/** + * Re-export from the canonical headers implementation. + * + * @module + */ +export { + applyS3Headers, + S3_CORS_HEADERS, + s3Headers, +} from '../../utils/s3/headers'; diff --git a/src/interfaces/s3/xml.ts b/src/interfaces/s3/xml.ts index 6b2d5db..346bfe3 100644 --- a/src/interfaces/s3/xml.ts +++ b/src/interfaces/s3/xml.ts @@ -1,309 +1,22 @@ -import { s3Headers } from './headers'; - -const escapeXml = (str: string): string => - str - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); - -const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z'); - -const encodeKey = (value: string, encodingType: string | null = null): string => - encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value); - -// ─────── Bucket operations ─────── - -export const listBucketsXml = ( - buckets: { name: string; createdAt: Date }[], - _requestId: string, -): string => ` - - - ${buckets - .map( - (b) => ` - ${escapeXml(b.name)} - ${isoDate(b.createdAt)} - `, - ) - .join('')} - -`; - -export const bucketVersioningConfigurationXml = - (): string => ` -`; - -// ─────── Object listing ─────── - -export const listBucketResultXml = ( - bucketName: string, - objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[], - prefixes: string[], - isTruncated: boolean, - marker: string | null, - maxKeys: number, - prefix: string, - delimiter: string | null, - nextMarker: string | null, - _requestId: string, - encodingType: string | null = null, -): string => ` - - ${escapeXml(bucketName)} - ${encodeKey(prefix, encodingType)} - ${encodeKey(marker || '', encodingType)} - ${maxKeys} - ${encodeKey(delimiter || '', encodingType)} - ${encodingType ? `${escapeXml(encodingType)}` : ''} - ${isTruncated} - ${objects - .map( - (o) => ` - ${encodeKey(o.key, encodingType)} - ${isoDate(o.lastModified)} - "${o.etag}" - ${o.sizeBytes} - STANDARD - `, - ) - .join('')} - ${prefixes - .map( - (p) => ` - ${encodeKey(p, encodingType)} - `, - ) - .join('')} - ${nextMarker ? `${encodeKey(nextMarker, encodingType)}` : ''} -`; - -export const listBucketV2ResultXml = ( - bucketName: string, - objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[], - prefixes: string[], - isTruncated: boolean, - maxKeys: number, - prefix: string, - delimiter: string | null, - continuationToken: string | null, - nextContinuationToken: string | null, - keyCount: number, - _requestId: string, - encodingType: string | null = null, -): string => ` - - ${escapeXml(bucketName)} - ${encodeKey(prefix, encodingType)} - ${maxKeys} - ${keyCount} - ${delimiter ? `${encodeKey(delimiter, encodingType)}` : ''} - ${encodingType ? `${escapeXml(encodingType)}` : ''} - ${continuationToken ? `${encodeKey(continuationToken, encodingType)}` : ''} - ${isTruncated} - ${objects - .map( - (o) => ` - ${encodeKey(o.key, encodingType)} - ${isoDate(o.lastModified)} - "${o.etag}" - ${o.sizeBytes} - STANDARD - `, - ) - .join('')} - ${prefixes - .map( - (p) => ` - ${encodeKey(p, encodingType)} - `, - ) - .join('')} - ${nextContinuationToken ? `${encodeKey(nextContinuationToken, encodingType)}` : ''} -`; - -// ─────── Multipart ─────── - -export const initiateMultipartUploadXml = ( - bucketName: string, - key: string, - uploadId: string, -): string => ` - - ${escapeXml(bucketName)} - ${escapeXml(key)} - ${uploadId} -`; - -export const listPartsXml = ( - bucketName: string, - key: string, - uploadId: string, - parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[], - maxParts: number, - isTruncated: boolean, - _requestId: string, -): string => ` - - ${escapeXml(bucketName)} - ${escapeXml(key)} - ${uploadId} - ${maxParts} - ${isTruncated} - ${parts - .map( - (p) => ` - ${p.partNumber} - ${isoDate(p.createdAt)} - "${p.etag}" - ${p.sizeBytes} - `, - ) - .join('')} -`; - -export const listMultipartUploadsXml = ( - bucketName: string, - uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[], - maxUploads: number, - isTruncated: boolean, - nextKeyMarker: string | null, - _requestId: string, -): string => ` - - ${escapeXml(bucketName)} - - - ${nextKeyMarker ? `${escapeXml(nextKeyMarker)}` : ''} - ${maxUploads} - ${isTruncated} - ${uploads - .map( - (u) => ` - ${escapeXml(u.key)} - ${u.uploadId} - ${escapeXml(u.initiatedBy || 's3')}${escapeXml(u.initiatedBy || 's3')} - ${escapeXml(u.initiatedBy || 's3')}${escapeXml(u.initiatedBy || 's3')} - STANDARD - ${isoDate(u.initiatedAt)} - `, - ) - .join('')} -`; - -export const completeMultipartUploadXml = ( - bucketName: string, - key: string, - etag: string, - location: string, -): string => ` - - ${escapeXml(location)} - ${escapeXml(bucketName)} - ${escapeXml(key)} - "${etag}" -`; - -// ─────── Delete result ─────── - -export const deleteResultXml = ( - deleted: string[], - errors: { key: string; code: string; message: string }[], -): string => ` - - ${deleted - .map( - (key) => ` - ${escapeXml(key)} - `, - ) - .join('')} - ${errors - .map( - (e) => ` - ${escapeXml(e.key)} - ${e.code} - ${escapeXml(e.message)} - `, - ) - .join('')} -`; - -// ─────── Copy ─────── - -export const copyObjectResultXml = ( - etag: string, - lastModified: Date, -): string => ` - - "${etag}" - ${isoDate(lastModified)} -`; - -// ─────── Error ─────── - -export const s3ErrorXml = ( - code: string, - message: string, - resource: string, - requestId: string, -): string => ` - - ${code} - ${escapeXml(message)} - ${escapeXml(resource)} - ${requestId} - ${requestId} -`; - -export const s3ErrorResponse = ( - code: string, - message: string, - resource: string, - status: number, - requestId: string = '', - extraHeaders: Record = {}, -): Response => - new Response(s3ErrorXml(code, message, resource, requestId), { - status, - headers: s3Headers(requestId, { - 'content-type': 'application/xml', - ...extraHeaders, - }), - }); - -// ─────── DeleteObjects XML parser ─────── - -export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => { - const keys = Array.from(body.matchAll(/([^<]+)<\/Key>/g), (match) => match[1]); - const quiet = body.includes('true') || body.includes('true '); - return { keys, quiet }; -}; - -// ─────── CompleteMultipartUpload XML parser ─────── - -export interface CompletePart { - partNumber: number; - etag: string; -} - -export const parseCompleteMultipartBody = (body: string): CompletePart[] => { - const parts: CompletePart[] = []; - const partRegex = /[\s\S]*?<\/Part>/g; - const partMatch = body.match(partRegex) || []; - - for (const partXml of partMatch) { - const numMatch = partXml.match(/(\d+)<\/PartNumber>/); - const etagMatch = partXml.match(/"?([^"<\s]+)"?<\/ETag>/); - if (numMatch && etagMatch) { - parts.push({ - partNumber: parseInt(numMatch[1], 10), - etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''), - }); - } - } - - return parts; -}; +/** + * Re-export from the canonical XML implementation. + * + * @module + */ +export { + bucketVersioningConfigurationXml, + type CompletePart, + completeMultipartUploadXml, + copyObjectResultXml, + deleteResultXml, + initiateMultipartUploadXml, + listBucketResultXml, + listBucketsXml, + listBucketV2ResultXml, + listMultipartUploadsXml, + listPartsXml, + parseCompleteMultipartBody, + parseDeleteObjectsBody, + s3ErrorResponse, + s3ErrorXml, +} from '../../utils/s3/xml'; diff --git a/src/utils/s3/auth.ts b/src/utils/s3/auth.ts index 3581eda..c693415 100644 --- a/src/utils/s3/auth.ts +++ b/src/utils/s3/auth.ts @@ -46,6 +46,12 @@ export interface VerifyPresignedUrlInput { const SERVICE = 's3'; const TERMINATION = 'aws4_request'; +/** + * Maximum acceptable clock skew between client and server for header-based + * SigV4 authentication. AWS allows 15 minutes. + */ +const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000; + // eslint-disable-next-line @typescript-eslint/no-explicit-any const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => { if (data instanceof Uint8Array) return data; @@ -129,10 +135,40 @@ const buildCanonicalRequest = ( return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`; }; +/** + * Normalizes a URI per AWS SigV4 requirements plus RFC 3986: + * + * 1. Decode percent-encoded characters + * 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4 + * + * @param uri - The raw URI path to normalize. + * @returns The normalized URI path. + */ const normalizeUri = (uri: string): string => { if (!uri || uri === '') return '/'; - // AWS SigV4 requires URI-decoded paths in the canonical request - return decodeURIComponent(uri); + + // Step 1: Decode (SigV4 requirement) + const decoded = decodeURIComponent(uri); + + // Step 2: Remove dot-segments per RFC 3986 section 5.2.4 + const segments = decoded.split('/'); + const result: string[] = []; + + for (const segment of segments) { + if (segment === '.' || segment === '') { + // Skip `.` and empty segments (from double slashes) + continue; + } + if (segment === '..') { + result.pop(); // Go up one level + continue; + } + result.push(segment); + } + + // Reconstruct path + const normalized = result.length > 0 ? `/${result.join('/')}` : '/'; + return normalized; }; const awsEncode = (value: string): string => @@ -149,23 +185,57 @@ export const buildCanonicalQueryString = ( for (const [key, value] of searchParams.entries()) { if (!excludeKeys.has(key)) pairs.push([key, value]); } + // AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare pairs.sort(([ak, av], [bk, bv]) => { const a = `${awsEncode(ak)}=${awsEncode(av)}`; const b = `${awsEncode(bk)}=${awsEncode(bv)}`; - return a.localeCompare(b); + if (a < b) return -1; + if (a > b) return 1; + return 0; }); return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&'); }; -const getHashedPayload = async ( - body: string | null, - contentSha256: string | null, -): Promise => { - if (contentSha256) return contentSha256; +const getHashedPayload = async (body: string | null): Promise => { if (!body || body.length === 0) return await sha256Hex(''); return await sha256Hex(body); }; +/** + * Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date. + * + * @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format. + * @returns The parsed Date, or null if the format is invalid. + */ +const parseAmzDateUtc = (amzDate: string): Date | null => { + const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/); + if (!match) return null; + const [, year, month, day, hour, minute, second] = match; + return new Date( + Date.UTC( + Number.parseInt(year, 10), + Number.parseInt(month, 10) - 1, + Number.parseInt(day, 10), + Number.parseInt(hour, 10), + Number.parseInt(minute, 10), + Number.parseInt(second, 10), + ), + ); +}; + +/** + * Validates that `host` is included in the signed headers list. + * + * AWS SigV4 mandates that `host` is always signed. Reject requests that + * omit it to prevent header injection / replay variants. + * + * @param signedHeaders - The semicolon-separated signed headers string. + * @returns True if `host` is present. + */ +const validateSignedHeaders = (signedHeaders: string): boolean => { + return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host'); +}; + export const verifySignature = async ( method: string, url: string, @@ -193,6 +263,16 @@ export const verifySignature = async ( return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; } + // Validate service and termination in credential scope (M2) + if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + + // Validate host is in signed headers (LOW/host) + if (!validateSignedHeaders(parsed.signedHeaders)) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + const parsedUrl = new URL(url, 'http://localhost'); const canonicalUri = normalizeUri(parsedUrl.pathname); const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams); @@ -201,7 +281,11 @@ export const verifySignature = async ( if (contentSha256?.startsWith('STREAMING-')) { return { isValid: false, credential: null, errorCode: 'NotImplemented' }; } - const hashedPayload = await getHashedPayload(body, contentSha256); + + // H4: Compute hash from actual body instead of trusting header blindly. + // For streaming bodies (body === null), we cannot hash at this point — + // the caller (controller) must verify body hash after streaming. + const hashedPayload = await getHashedPayload(body); const canonicalRequest = buildCanonicalRequest( method, @@ -214,8 +298,31 @@ export const verifySignature = async ( const hashedCanonicalRequest = await sha256Hex(canonicalRequest); - const amzDate = headers['x-amz-date'] || ''; + // M1: Fall back to Date header if x-amz-date is missing + const amzDate = headers['x-amz-date'] || headers.date || ''; + + // H5: Validate request freshness (clock skew / replay protection) + if (amzDate) { + const requestDate = parseAmzDateUtc(amzDate); + if (requestDate) { + const now = Date.now(); + const skew = Math.abs(now - requestDate.getTime()); + if (skew > MAX_CLOCK_SKEW_MS) { + return { isValid: false, credential: null, errorCode: 'RequestExpired' }; + } + } + } + const dateStamp = parsed.date; + + // M3: Ensure date in credential scope matches x-amz-date + if (amzDate) { + const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD" + if (amzDateStamp !== dateStamp) { + return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; + } + } + const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`; const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`; @@ -238,22 +345,6 @@ export const verifySignature = async ( }; }; -const parseAmzDateUtc = (amzDate: string): Date | null => { - const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/); - if (!match) return null; - const [, year, month, day, hour, minute, second] = match; - return new Date( - Date.UTC( - Number.parseInt(year, 10), - Number.parseInt(month, 10) - 1, - Number.parseInt(day, 10), - Number.parseInt(hour, 10), - Number.parseInt(minute, 10), - Number.parseInt(second, 10), - ), - ); -}; - export const verifyPresignedUrl = async ({ url, method, @@ -312,6 +403,11 @@ export const verifyPresignedUrl = async ({ return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' }; } + // Validate host is in signed headers for presigned URLs too + if (!validateSignedHeaders(signedHeaders)) { + return { isValid: false, credential: null, errorCode: 'AccessDenied' }; + } + const signedHeaderList = signedHeaders.split(';').filter(Boolean); const canonicalHeaders = signedHeaderList .map((headerName) => { @@ -340,3 +436,30 @@ export const isS3Request = (headers: Record): boolean => { const auth = headers.authorization || ''; return auth.startsWith('AWS4-HMAC-SHA256'); }; + +/** + * Verifies that the actual body SHA-256 matches the `x-amz-content-sha256` + * header from the original request. + * + * This MUST be called AFTER the body has been fully streamed and hashed, + * as a second pass after `verifySignature` (which cannot hash a streaming + * body without consuming it). + * + * @param bodySha256 - The SHA-256 hex digest of the actual body content. + * @param headers - The original request headers. + * @returns An error result on mismatch, or null if the check passes. + */ +export const verifyBodyHash = ( + bodySha256: string, + headers: Record, +): SigV4Result | null => { + const claimedHash = headers['x-amz-content-sha256']; + // If the client sent UNSIGNED-PAYLOAD, skip verification + if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) { + return null; + } + if (claimedHash !== bodySha256) { + return { isValid: false, credential: null, errorCode: 'BadDigest' }; + } + return null; +}; diff --git a/src/utils/s3/headers.ts b/src/utils/s3/headers.ts index 809527d..bd53bac 100644 --- a/src/utils/s3/headers.ts +++ b/src/utils/s3/headers.ts @@ -1,3 +1,5 @@ +import { nanoid } from 'nanoid'; + export const S3_CORS_HEADERS: Record = { 'access-control-allow-origin': '*', 'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS', @@ -31,7 +33,13 @@ export const s3Headers = ( extraHeaders: Record = {}, ): Record => ({ ...S3_CORS_HEADERS, - ...(requestId ? { 'x-amz-request-id': requestId, 'x-amz-id-2': requestId } : {}), + server: 'AmazonS3', + ...(requestId + ? { + 'x-amz-request-id': requestId, + 'x-amz-id-2': `${requestId}+${nanoid(16)}`, + } + : {}), ...extraHeaders, }); diff --git a/src/utils/s3/virtual-host.ts b/src/utils/s3/virtual-host.ts index 026281a..4cf6c76 100644 --- a/src/utils/s3/virtual-host.ts +++ b/src/utils/s3/virtual-host.ts @@ -1,5 +1,9 @@ const stripPort = (host: string): string => { - if (host.startsWith('[')) return host; + // Handle IPv6: [::1]:8080 -> [::1] + if (host.startsWith('[')) { + const closeBracket = host.indexOf(']'); + return host.slice(0, closeBracket + 1).toLowerCase(); + } return host.split(':')[0].toLowerCase().replace(/\.$/, ''); }; diff --git a/src/utils/s3/xml.ts b/src/utils/s3/xml.ts index 6b2d5db..cd255a7 100644 --- a/src/utils/s3/xml.ts +++ b/src/utils/s3/xml.ts @@ -277,8 +277,10 @@ export const s3ErrorResponse = ( // ─────── DeleteObjects XML parser ─────── export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => { - const keys = Array.from(body.matchAll(/([^<]+)<\/Key>/g), (match) => match[1]); - const quiet = body.includes('true') || body.includes('true '); + // H9: Use non-greedy match to handle keys containing < character + const keys = Array.from(body.matchAll(/([\s\S]*?)<\/Key>/g), (match) => match[1]); + // Handle whitespace inside element + namespace prefix support + const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body); return { keys, quiet }; }; @@ -299,7 +301,7 @@ export const parseCompleteMultipartBody = (body: string): CompletePart[] => { const etagMatch = partXml.match(/"?([^"<\s]+)"?<\/ETag>/); if (numMatch && etagMatch) { parts.push({ - partNumber: parseInt(numMatch[1], 10), + partNumber: Number.parseInt(numMatch[1], 10), etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''), }); }