From 8e6ccf211081ef674b692e6d792c2a7e200f04cc Mon Sep 17 00:00:00 2001 From: asepharyana Date: Tue, 7 Jul 2026 05:39:23 +0700 Subject: [PATCH] =?UTF-8?q?feat:=20S3=20client=20compatibility=20=E2=80=94?= =?UTF-8?q?=20virtual-hosted=20style,=20CORS,=20presigned=20multi-method,?= =?UTF-8?q?=20ListMultipartUploads,=20edge=20case=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Virtual-hosted bucket detection from Host header (extractS3BucketFromHost) - S3 CORS headers + OPTIONS preflight + x-amz-id-2/HostId everywhere - Presigned GET/PUT/HEAD/DELETE via centralized auth (no GET-only restriction) - ListMultipartUploads with DB/xml helpers - UploadPart partNumber range validation (1-10000) - CompleteMultipartUpload ETag matching and ascending order validation - DeleteObjects quiet mode support - CopyObject URL-decode and conditional if-match/if-none-match - encoding-type=url support in ListObjects V1/V2 XML - Safe range-based prefix matching (replaces SQL LIKE) - STREAMING-AWS4-HMAC-SHA256-PAYLOAD → 501 NotImplemented - Traefik wildcard HostRegex for virtual-hosted style - S3_VHOST_DOMAINS config env var --- .env.example | 3 +- docker-compose.yml | 2 +- src/db/files-ext.ts | 4 +- src/db/multipart.ts | 37 ++++ src/env.ts | 12 ++ src/index.ts | 69 +++---- src/routes/s3.ts | 328 +++++++++++++++++++++------------- src/utils/s3/auth.ts | 3 + src/utils/s3/headers.ts | 44 +++++ src/utils/s3/object-stream.ts | 3 +- src/utils/s3/virtual-host.ts | 23 +++ src/utils/s3/xml.ts | 68 +++++-- 12 files changed, 417 insertions(+), 179 deletions(-) create mode 100644 src/utils/s3/headers.ts create mode 100644 src/utils/s3/virtual-host.ts diff --git a/.env.example b/.env.example index c8f0474..890eca1 100644 --- a/.env.example +++ b/.env.example @@ -13,4 +13,5 @@ RATE_LIMIT_MAX_REQUESTS=30 # S3-compatible API credentials # S3_ACCESS_KEY=teleuploader-admin # S3_SECRET_KEY=your-secret-key-here -# S3_DEFAULT_REGION=us-east-1 \ No newline at end of file +# S3_DEFAULT_REGION=us-east-1 +# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1364ed8..af8ad45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: - app-shared-net labels: - "traefik.enable=true" - - "traefik.http.routers.teleuploader.rule=Host(`upload.asepharyana.my.id`) || Host(`upload.asepharyana.web.id`)" + - "traefik.http.routers.teleuploader.rule=Host(`upload.asepharyana.my.id`) || Host(`upload.asepharyana.web.id`) || HostRegexp(`{subhost:[a-z0-9][a-z0-9.-]+}.upload.asepharyana.my.id`) || HostRegexp(`{subhost:[a-z0-9][a-z0-9.-]+}.upload.asepharyana.web.id`)" - "traefik.http.routers.teleuploader.entrypoints=websecure" - "traefik.http.routers.teleuploader.tls=true" - "traefik.http.routers.teleuploader.tls.certresolver=cloudflare" diff --git a/src/db/files-ext.ts b/src/db/files-ext.ts index 876122d..025eb9d 100644 --- a/src/db/files-ext.ts +++ b/src/db/files-ext.ts @@ -62,7 +62,9 @@ export const listObjectsByPrefix = async ( maxKeys: number, startAfter: string | null, ): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => { - let query = sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${prefix}%`}`; + let query = prefix + ? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key >= ${prefix} AND s3_key < ${`${prefix}￿`}` + : sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`; if (startAfter) { query = sql`${query} AND s3_key > ${startAfter}`; diff --git a/src/db/multipart.ts b/src/db/multipart.ts index ab5815a..d6e3c95 100644 --- a/src/db/multipart.ts +++ b/src/db/multipart.ts @@ -90,3 +90,40 @@ export const listMultipartParts = async (uploadId: string): Promise): MultipartUpload => ({ + uploadId: r.upload_id as string, + bucketId: r.bucket_id as string, + s3Key: r.s3_key as string, + initiatedAt: new Date(r.initiated_at as string), + status: r.status as string, + initiatedBy: (r.initiated_by as string | null) || '', +}); + +export const listMultipartUploadsByBucket = async ( + bucketId: string, + maxUploads: number, + keyMarker: string | null, +): Promise<{ uploads: MultipartUpload[]; isTruncated: boolean; nextKeyMarker: string | null }> => { + const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000); + const result = (await db.execute( + keyMarker + ? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by + FROM multipart_uploads + WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker} + ORDER BY s3_key, initiated_at + LIMIT ${limit + 1}` + : sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by + FROM multipart_uploads + WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' + ORDER BY s3_key, initiated_at + LIMIT ${limit + 1}`, + )) as unknown as Record[]; + + const uploads = result.slice(0, limit).map(mapRowToMultipartUpload); + return { + uploads, + isTruncated: result.length > limit, + nextKeyMarker: result.length > limit ? uploads.at(-1)?.s3Key || null : null, + }; +}; diff --git a/src/env.ts b/src/env.ts index 0f3f0de..6ae4b81 100644 --- a/src/env.ts +++ b/src/env.ts @@ -20,6 +20,7 @@ interface AppConfig { s3SecretKey: string; s3DefaultRegion: string; proxyS3Get: boolean; + s3VhostDomains: string[]; } const requiredEnv = { @@ -50,6 +51,14 @@ const parseTokens = (value: string | undefined): string[] => .map((t) => t.trim()) .filter((t) => t !== ''); +const parseDomains = (value: string | undefined): string[] => + parseTokens(value).map((domain) => + domain + .replace(/^https?:\/\//, '') + .split('/')[0] + .toLowerCase(), + ); + const maskSecret = (value: string): string => { if (!value) return ''; if (value.length <= 10) return '***'; @@ -80,6 +89,9 @@ export const config: AppConfig = { s3SecretKey: process.env.S3_SECRET_KEY || '', s3DefaultRegion: process.env.S3_DEFAULT_REGION || 'us-east-1', proxyS3Get: process.env.PROXY_S3_GET !== 'false', + s3VhostDomains: parseDomains( + process.env.S3_VHOST_DOMAINS || 'upload.asepharyana.my.id,upload.asepharyana.web.id', + ), }; logger.info('Environment variables loaded', { diff --git a/src/index.ts b/src/index.ts index 5d98b39..90c9c50 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ import logger from './utils/logger'; import { metricsCollector } from './utils/metrics'; import { cleanupRateLimitCache, withRateLimit } from './utils/rateLimit'; import { isS3Request } from './utils/s3/auth'; +import { extractS3BucketFromHost } from './utils/s3/virtual-host'; // ─── Auto-run migration at startup ────────────────────────────────────────── try { @@ -30,6 +31,29 @@ try { logger.warn('Auto-migration warning (non-fatal)'); } +const getS3RouteBucket = (req: Request): string | null => { + const host = req.headers.get('host') || ''; + return extractS3BucketFromHost(host, config.s3VhostDomains); +}; + +const shouldHandleS3 = (req: Request, headers: Record): boolean => { + const url = new URL(req.url); + return Boolean( + getS3RouteBucket(req) || isS3Request(headers) || url.searchParams.has('X-Amz-Signature'), + ); +}; + +const handleMaybeS3Root = (req: Request): Response | Promise => { + if (req.method === 'OPTIONS') { + return handleS3Request(req, getS3RouteBucket(req)); + } + const headers = Object.fromEntries(req.headers); + if (shouldHandleS3(req, headers)) { + return handleS3Request(req, getS3RouteBucket(req)); + } + return new Response('Not Allowed', { status: 405 }); +}; + const server = serve({ port: config.port, routes: { @@ -54,40 +78,16 @@ const server = serve({ '/': { GET: (req: Request) => { const headers = Object.fromEntries(req.headers); - const url = new URL(req.url); - if (isS3Request(headers) || url.searchParams.has('X-Amz-Signature')) { - return handleS3Request(req); + if (shouldHandleS3(req, headers)) { + return handleS3Request(req, getS3RouteBucket(req)); } return handleHome(); }, - PUT: (req: Request) => { - const headers = Object.fromEntries(req.headers); - if (isS3Request(headers)) { - return handleS3Request(req); - } - return new Response('Not Allowed', { status: 405 }); - }, - HEAD: (req: Request) => { - const headers = Object.fromEntries(req.headers); - if (isS3Request(headers)) { - return handleS3Request(req); - } - return new Response('Not Allowed', { status: 405 }); - }, - DELETE: (req: Request) => { - const headers = Object.fromEntries(req.headers); - if (isS3Request(headers)) { - return handleS3Request(req); - } - return new Response('Not Allowed', { status: 405 }); - }, - POST: (req: Request) => { - const headers = Object.fromEntries(req.headers); - if (isS3Request(headers)) { - return handleS3Request(req); - } - return new Response('Not Allowed', { status: 405 }); - }, + PUT: handleMaybeS3Root, + HEAD: handleMaybeS3Root, + DELETE: handleMaybeS3Root, + POST: handleMaybeS3Root, + OPTIONS: handleMaybeS3Root, }, '/api/v1/*': { GET: handleWebApiV1, @@ -97,9 +97,12 @@ const server = serve({ }, }, fetch: async (req: Request) => { + if (req.method === 'OPTIONS') { + return handleS3Request(req, getS3RouteBucket(req)); + } const headers = Object.fromEntries(req.headers); - if (isS3Request(headers) || new URL(req.url).searchParams.has('X-Amz-Signature')) { - return handleS3Request(req); + if (shouldHandleS3(req, headers)) { + return handleS3Request(req, getS3RouteBucket(req)); } return new Response('Not Found', { status: 404 }); }, diff --git a/src/routes/s3.ts b/src/routes/s3.ts index b286407..22fa736 100644 --- a/src/routes/s3.ts +++ b/src/routes/s3.ts @@ -14,12 +14,14 @@ import { findMultipartUpload, insertMultipartPart, listMultipartParts, + listMultipartUploadsByBucket, } from '../db/multipart'; import type { File } from '../db/schema'; import { config } from '../env'; import { cleanupTempFile, computeHash, ensureExtension, getErrorMessage } from '../utils/file'; import logger from '../utils/logger'; import { verifyPresignedUrl, verifySignature } from '../utils/s3/auth'; +import { S3_CORS_HEADERS, s3Headers } from '../utils/s3/headers'; import { createGetObjectResponse, type ObjectPartSource } from '../utils/s3/object-stream'; import { parseRangeHeader, unsatisfiedContentRange } from '../utils/s3/range'; import { @@ -30,6 +32,7 @@ import { listBucketResultXml, listBucketsXml, listBucketV2ResultXml, + listMultipartUploadsXml, listPartsXml, parseCompleteMultipartBody, parseDeleteObjectsBody, @@ -40,6 +43,16 @@ import { forwardToStorage, getFileInfo } from '../utils/telegram'; const REGION = config.s3DefaultRegion || 'us-east-1'; const REQUEST_ID = () => nanoid(16); +const s3Response = ( + body: string | null, + status: number, + reqId: string, + extraHeaders: Record = {}, +): Response => new Response(body, { status, headers: s3Headers(reqId, extraHeaders) }); + +const s3OptionsResponse = (): Response => + new Response(null, { status: 204, headers: S3_CORS_HEADERS }); + const parseS3Path = (pathname: string): { bucket: string | null; key: string | null } => { const parts = pathname.split('/').filter(Boolean); if (parts.length === 0) return { bucket: null, key: null }; @@ -57,45 +70,62 @@ const headersToRecord = (req: Request): Record => { // ─────── Main Dispatcher ─────── -export const handleS3Request = async (req: Request): Promise => { +export const handleS3Request = async ( + req: Request, + virtualHostBucket: string | null = null, +): Promise => { const method = req.method; const url = new URL(req.url); const pathname = url.pathname; - const { bucket, key } = parseS3Path(pathname); + const { bucket, key } = virtualHostBucket + ? { + bucket: virtualHostBucket, + key: pathname === '/' ? null : decodeURIComponent(pathname.slice(1)), + } + : parseS3Path(pathname); const headers = headersToRecord(req); const searchParams = url.searchParams; const reqId = REQUEST_ID(); - // Presigned URLs: only supported for GET (verified in handleGetObject) - if (searchParams.has('X-Amz-Signature')) { - if (method !== 'GET') { - return s3ErrorResponse( - 'AccessDenied', - 'Presigned URLs only supported for GET', - pathname, - 403, - reqId, + if (method === 'OPTIONS') { + return s3OptionsResponse(); + } + + const isPresigned = searchParams.has('X-Amz-Signature'); + const authResult = isPresigned + ? await verifyPresignedUrl({ + url: req.url, + method, + headers, + s3AccessKey: config.s3AccessKey, + s3SecretKey: config.s3SecretKey, + region: REGION, + }) + : await verifySignature( + method, + req.url, + headers, + null, + config.s3AccessKey, + config.s3SecretKey, + REGION, ); - } - } else { - const authResult = await verifySignature( - method, - req.url, - headers, - null, - config.s3AccessKey, - config.s3SecretKey, - REGION, + + if (!authResult.isValid) { + const status = authResult.errorCode === 'NotImplemented' ? 501 : 403; + const message = + authResult.errorCode === 'NotImplemented' + ? 'aws-chunked streaming payloads are not supported.' + : isPresigned + ? 'Presigned URL verification failed' + : 'Authentication required'; + return s3ErrorResponse( + authResult.errorCode || 'AccessDenied', + message, + pathname, + status, + reqId, ); - if (!authResult.isValid) { - return s3ErrorResponse( - authResult.errorCode || 'AccessDenied', - 'Authentication required', - pathname, - 403, - reqId, - ); - } } try { @@ -116,6 +146,9 @@ export const handleS3Request = async (req: Request): Promise => { // Bucket-level operations if (!key) { if (method === 'GET') { + if (searchParams.has('uploads')) { + return handleListMultipartUploads(bucket, searchParams, reqId); + } const listType = searchParams.get('list-type'); if (listType === '2') { return handleListObjectsV2(bucket, searchParams, reqId); @@ -131,7 +164,7 @@ export const handleS3Request = async (req: Request): Promise => { return handleDeleteObjects(bucket, body, reqId); } if (searchParams.has('tagging')) { - return new Response(null, { status: 204 }); + return s3Response(null, 204, reqId); } } return s3ErrorResponse( @@ -162,8 +195,7 @@ export const handleS3Request = async (req: Request): Promise => { } // Standard object operations - if (method === 'GET') - return handleGetObject(bucket, key, searchParams, headers, req.url, reqId); + if (method === 'GET') return handleGetObject(bucket, key, searchParams, headers, reqId); if (method === 'HEAD') return handleHeadObject(bucket, key, reqId); if (method === 'PUT') return handlePutObject(bucket, key, searchParams, headers, req, reqId); if (method === 'DELETE') return handleDeleteObject(bucket, key, reqId); @@ -192,10 +224,7 @@ export const handleS3Request = async (req: Request): Promise => { const handleListBuckets = async (reqId: string): Promise => { const buckets = await listBuckets(); const xml = listBucketsXml(buckets, reqId); - return new Response(xml, { - status: 200, - headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId }, - }); + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; const handleCreateBucket = async (bucketName: string, reqId: string): Promise => { @@ -219,7 +248,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise => { @@ -233,7 +262,7 @@ const handleHeadBucket = async (bucketName: string, reqId: string): Promise => { @@ -258,7 +287,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise, - requestUrl: string, reqId: string, ): Promise => { - if (searchParams.has('X-Amz-Signature')) { - const presignedResult = await verifyPresignedUrl({ - url: requestUrl, - method: 'GET', - headers, - s3AccessKey: config.s3AccessKey, - s3SecretKey: config.s3SecretKey, - region: REGION, - }); - if (!presignedResult.isValid) { - return s3ErrorResponse( - presignedResult.errorCode || 'AccessDenied', - 'Presigned URL verification failed', - `/${bucket}/${key}`, - 403, - reqId, - ); - } - } - const bucketRecord = await findBucketByName(bucket); if (!bucketRecord) return s3ErrorResponse( @@ -335,10 +343,7 @@ const handleGetObject = async ( if (!config.proxyS3Get) { // Legacy 302 redirect path (when proxy is disabled) - return new Response(null, { - status: 302, - headers: { Location: redirectUrl, 'x-amz-request-id': reqId }, - }); + return s3Response(null, 302, reqId, { location: redirectUrl }); } const part: ObjectPartSource = { @@ -420,10 +425,7 @@ const handleGetMultipartObject = async ( } if (!config.proxyS3Get) { - return new Response(null, { - status: 302, - headers: { Location: sources[0].telegramUrl, 'x-amz-request-id': reqId }, - }); + return s3Response(null, 302, reqId, { location: sources[0].telegramUrl }); } try { @@ -472,16 +474,14 @@ const handleHeadObject = async (bucket: string, key: string, reqId: string): Pro reqId, ); - return new Response(null, { - status: 200, - headers: { - 'content-type': file.mimeType, - 'content-length': String(file.sizeBytes), - etag: `"${file.fileHash || nanoid(16)}"`, - 'last-modified': - file.createdAt instanceof Date ? file.createdAt.toUTCString() : new Date().toUTCString(), - 'x-amz-request-id': reqId, - }, + return s3Response(null, 200, reqId, { + 'content-type': file.mimeType, + 'content-length': String(file.sizeBytes), + etag: `"${file.fileHash || nanoid(16)}"`, + 'last-modified': + file.createdAt instanceof Date ? file.createdAt.toUTCString() : new Date().toUTCString(), + 'accept-ranges': 'bytes', + 'cache-control': 'public, max-age=31536000', }); }; @@ -504,12 +504,12 @@ const handlePutObject = async ( ); if (searchParams.has('tagging')) { - return new Response(null, { status: 204 }); + return s3Response(null, 204, reqId); } const copySource = headers['x-amz-copy-source']; if (copySource) { - return handleCopyObject(bucket, key, copySource, bucketRecord.id, reqId); + return handleCopyObject(bucket, key, copySource, headers, bucketRecord.id, reqId); } // Read raw body — S3 clients send raw binary, not multipart/form-data @@ -520,10 +520,7 @@ const handlePutObject = async ( const existing = await findFileByBucketAndKey(bucketRecord.id, key); if (existing) { - return new Response(null, { - status: 200, - headers: { etag: `"${hash}"`, 'x-amz-request-id': reqId }, - }); + return s3Response(null, 200, reqId, { etag: `"${hash}"` }); } return await storeFileToTelegram(fileBuffer, hash, key, bucketRecord, contentType, reqId); @@ -579,19 +576,18 @@ const storeFileToTelegram = async ( await cleanupTempFile(tempPath); - return new Response(null, { - status: 200, - headers: { etag: `"${hash}"`, 'x-amz-request-id': reqId }, - }); + return s3Response(null, 200, reqId, { etag: `"${hash}"` }); }; const handleCopyObject = async ( _destBucket: string, destKey: string, - copySource: string, + rawCopySource: string, + headers: Record, destBucketId: string, reqId: string, ): Promise => { + const copySource = decodeURIComponent(rawCopySource); const sourcePath = copySource.startsWith('/') ? copySource.slice(1) : copySource; const parts = sourcePath.split('/'); const sourceBucket = parts[0]; @@ -617,6 +613,28 @@ const handleCopyObject = async ( reqId, ); + // Conditional copy: if-match / if-none-match checks + const ifMatch = headers['x-amz-copy-source-if-match']; + const ifNoneMatch = headers['x-amz-copy-source-if-none-match']; + if (ifMatch && sourceFile.fileHash && ifMatch !== `"${sourceFile.fileHash}"`) { + return s3ErrorResponse( + 'PreconditionFailed', + 'The preconditions you specified did not hold.', + copySource, + 412, + reqId, + ); + } + if (ifNoneMatch && sourceFile.fileHash && ifNoneMatch === `"${sourceFile.fileHash}"`) { + return s3ErrorResponse( + 'PreconditionFailed', + 'The preconditions you specified did not hold.', + copySource, + 412, + reqId, + ); + } + const publicId = nanoid(); const { db, files: fileSchema } = await import('../db/index'); @@ -641,10 +659,7 @@ const handleCopyObject = async ( }); const xml = copyObjectResultXml(sourceFile.fileHash || nanoid(16), new Date()); - return new Response(xml, { - status: 200, - headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId }, - }); + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; const handleDeleteObject = async ( @@ -663,7 +678,7 @@ const handleDeleteObject = async ( ); await softDeleteFile(bucketRecord.id, key); - return new Response(null, { status: 204, headers: { 'x-amz-request-id': reqId } }); + return s3Response(null, 204, reqId); }; const handleDeleteObjects = async ( @@ -681,17 +696,14 @@ const handleDeleteObjects = async ( reqId, ); - const { keys } = parseDeleteObjectsBody(body); + const { keys, quiet } = parseDeleteObjectsBody(body); const deletedKeys: string[] = []; for (const key of keys) { const ok = await softDeleteFile(bucketRecord.id, key); if (ok) deletedKeys.push(key); } - const xml = deleteResultXml(deletedKeys, []); - return new Response(xml, { - status: 200, - headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId }, - }); + const xml = quiet ? deleteResultXml([], []) : deleteResultXml(deletedKeys, []); + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; // ─────── Object Listing ─────── @@ -715,6 +727,7 @@ const handleListObjectsV1 = async ( const delimiter = searchParams.get('delimiter') || null; const maxKeys = Math.min(parseInt(searchParams.get('max-keys') || '1000', 10), 1000); const marker = searchParams.get('marker') || null; + const encodingType = searchParams.get('encoding-type') || null; const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix( bucketRecord.id, @@ -747,12 +760,10 @@ const handleListObjectsV1 = async ( delimiter, nextMarker, reqId, + encodingType, ); - return new Response(xml, { - status: 200, - headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId }, - }); + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; const handleListObjectsV2 = async ( @@ -775,6 +786,7 @@ const handleListObjectsV2 = async ( const maxKeys = Math.min(parseInt(searchParams.get('max-keys') || '1000', 10), 1000); const continuationToken = searchParams.get('continuation-token') || null; const startAfter = searchParams.get('start-after') || null; + const encodingType = searchParams.get('encoding-type') || null; const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix( bucketRecord.id, @@ -808,12 +820,10 @@ const handleListObjectsV2 = async ( nextContinuationToken, displayObjects.length, reqId, + encodingType, ); - return new Response(xml, { - status: 200, - headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId }, - }); + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; // ─────── Multipart Upload ─────── @@ -837,10 +847,7 @@ const handleCreateMultipartUpload = async ( const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3'); const xml = initiateMultipartUploadXml(bucket, key, uploadId); - return new Response(xml, { - status: 200, - headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId }, - }); + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; const handleUploadPart = async ( @@ -852,6 +859,15 @@ const handleUploadPart = async ( ): Promise => { const uploadId = searchParams.get('uploadId')!; const partNumber = parseInt(searchParams.get('partNumber')!, 10); + if (partNumber < 1 || partNumber > 10000) { + return s3ErrorResponse( + 'InvalidArgument', + 'Part number must be an integer between 1 and 10000', + `/${bucket}/${key}`, + 400, + reqId, + ); + } const multipart = await findMultipartUpload(uploadId); if (!multipart || multipart.s3Key !== key) { @@ -889,10 +905,7 @@ const handleUploadPart = async ( etag, }); - return new Response(null, { - status: 200, - headers: { etag: `"${etag}"`, 'x-amz-request-id': reqId }, - }); + return s3Response(null, 200, reqId, { etag: `"${etag}"` }); }; const handleCompleteMultipartUpload = async ( @@ -917,6 +930,33 @@ const handleCompleteMultipartUpload = async ( const parts = parseCompleteMultipartBody(body); const storedParts = await listMultipartParts(uploadId); + // Validate parts match stored parts in ascending order and correct ETags + const sortedParts = [...parts].sort((a, b) => a.partNumber - b.partNumber); + for (let i = 0; i < sortedParts.length; i++) { + if (sortedParts[i].partNumber !== i + 1) { + return s3ErrorResponse( + 'InvalidPartOrder', + 'The list of parts was not in ascending order. Parts must be ordered by part number.', + `/${bucket}/${key}`, + 400, + reqId, + ); + } + } + const storedMap = new Map(storedParts.map((p) => [p.partNumber, p])); + for (const part of parts) { + const stored = storedMap.get(part.partNumber); + if (!stored || stored.etag !== part.etag) { + return s3ErrorResponse( + 'InvalidPart', + 'One or more specified parts could not be found. The part might not have been uploaded, or the specified ETag might not match.', + `/${bucket}/${key}`, + 400, + reqId, + ); + } + } + if (parts.length !== storedParts.length) { return s3ErrorResponse( 'InvalidPart', @@ -958,10 +998,47 @@ const handleCompleteMultipartUpload = async ( const combinedEtag = storedParts.map((p) => p.etag).join('-'); const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location); - return new Response(xml, { - status: 200, - headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId }, - }); + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); +}; + +const handleListMultipartUploads = async ( + bucket: string, + searchParams: URLSearchParams, + reqId: string, +): Promise => { + const bucketRecord = await findBucketByName(bucket); + if (!bucketRecord) + return s3ErrorResponse( + 'NoSuchBucket', + 'The specified bucket does not exist.', + `/${bucket}`, + 404, + reqId, + ); + + const maxUploads = Math.min(parseInt(searchParams.get('max-uploads') || '1000', 10), 1000); + const keyMarker = searchParams.get('key-marker') || null; + const { uploads, isTruncated, nextKeyMarker } = await listMultipartUploadsByBucket( + bucketRecord.id, + maxUploads, + keyMarker, + ); + + const xml = listMultipartUploadsXml( + bucket, + uploads.map((u) => ({ + key: u.s3Key, + uploadId: u.uploadId, + initiatedAt: u.initiatedAt, + initiatedBy: u.initiatedBy, + })), + maxUploads, + isTruncated, + nextKeyMarker, + reqId, + ); + + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; const handleAbortMultipartUpload = async ( @@ -983,7 +1060,7 @@ const handleAbortMultipartUpload = async ( } await abortMultipartUpload(uploadId); - return new Response(null, { status: 204, headers: { 'x-amz-request-id': reqId } }); + return s3Response(null, 204, reqId); }; const handleListParts = async ( @@ -1022,8 +1099,5 @@ const handleListParts = async ( reqId, ); - return new Response(xml, { - status: 200, - headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId }, - }); + return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); }; diff --git a/src/utils/s3/auth.ts b/src/utils/s3/auth.ts index 830c32c..935ead6 100644 --- a/src/utils/s3/auth.ts +++ b/src/utils/s3/auth.ts @@ -174,6 +174,9 @@ export const verifySignature = async ( const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams); const contentSha256 = headers['x-amz-content-sha256'] || null; + if (contentSha256?.startsWith('STREAMING-')) { + return { isValid: false, credential: null, errorCode: 'NotImplemented' }; + } const hashedPayload = await getHashedPayload(body, contentSha256); const canonicalRequest = buildCanonicalRequest( diff --git a/src/utils/s3/headers.ts b/src/utils/s3/headers.ts new file mode 100644 index 0000000..809527d --- /dev/null +++ b/src/utils/s3/headers.ts @@ -0,0 +1,44 @@ +export const S3_CORS_HEADERS: Record = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS', + 'access-control-allow-headers': [ + 'Authorization', + 'Content-Type', + 'Content-MD5', + 'Range', + 'If-Match', + 'If-None-Match', + 'If-Modified-Since', + 'If-Unmodified-Since', + 'X-Amz-*', + 'x-amz-*', + ].join(', '), + 'access-control-expose-headers': [ + 'Accept-Ranges', + 'Content-Length', + 'Content-Range', + 'Content-Type', + 'ETag', + 'Last-Modified', + 'x-amz-id-2', + 'x-amz-request-id', + ].join(', '), + 'access-control-max-age': '86400', +}; + +export const s3Headers = ( + requestId: string, + extraHeaders: Record = {}, +): Record => ({ + ...S3_CORS_HEADERS, + ...(requestId ? { 'x-amz-request-id': requestId, 'x-amz-id-2': requestId } : {}), + ...extraHeaders, +}); + +export const applyS3Headers = (headers: Headers, requestId: string): Headers => { + const result = new Headers(headers); + for (const [key, value] of Object.entries(s3Headers(requestId))) { + result.set(key, value); + } + return result; +}; diff --git a/src/utils/s3/object-stream.ts b/src/utils/s3/object-stream.ts index 52af667..2b4fcf8 100644 --- a/src/utils/s3/object-stream.ts +++ b/src/utils/s3/object-stream.ts @@ -1,3 +1,4 @@ +import { applyS3Headers } from './headers'; import { contentRange, type RangeParseResult } from './range'; export interface ObjectPartSource { @@ -97,7 +98,7 @@ export const createGetObjectResponse = async (input: ObjectResponseInput): Promi const end = input.range.type === 'valid' ? input.range.end : input.totalSize - 1; const plannedParts = planParts(input.parts, start, end); const contentLength = end >= start ? end - start + 1 : 0; - const headers = baseHeaders(input, contentLength); + const headers = applyS3Headers(baseHeaders(input, contentLength), input.reqId); if (input.range.type === 'valid') { headers.set('content-range', contentRange(start, end, input.totalSize)); diff --git a/src/utils/s3/virtual-host.ts b/src/utils/s3/virtual-host.ts new file mode 100644 index 0000000..026281a --- /dev/null +++ b/src/utils/s3/virtual-host.ts @@ -0,0 +1,23 @@ +const stripPort = (host: string): string => { + if (host.startsWith('[')) return host; + return host.split(':')[0].toLowerCase().replace(/\.$/, ''); +}; + +const isValidBucketLabel = (bucket: string): boolean => + /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) && + !bucket.includes('..') && + !bucket.includes('.-') && + !bucket.includes('-.'); + +export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => { + const normalizedHost = stripPort(host); + for (const domain of domains) { + const normalizedDomain = stripPort(domain); + if (!normalizedDomain || normalizedHost === normalizedDomain) continue; + if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue; + + const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1)); + return isValidBucketLabel(bucket) ? bucket : null; + } + return null; +}; diff --git a/src/utils/s3/xml.ts b/src/utils/s3/xml.ts index d22df96..656ef38 100644 --- a/src/utils/s3/xml.ts +++ b/src/utils/s3/xml.ts @@ -1,3 +1,5 @@ +import { s3Headers } from './headers'; + const escapeXml = (str: string): string => str .replace(/&/g, '&') @@ -8,6 +10,9 @@ const escapeXml = (str: string): string => const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z'); +const encodeKey = (value: string, encodingType: string | null = null): string => + encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value); + // ─────── Bucket operations ─────── export const listBucketsXml = ( @@ -40,18 +45,20 @@ export const listBucketResultXml = ( delimiter: string | null, nextMarker: string | null, _requestId: string, + encodingType: string | null = null, ): string => ` ${escapeXml(bucketName)} - ${escapeXml(prefix)} - ${escapeXml(marker || '')} + ${encodeKey(prefix, encodingType)} + ${encodeKey(marker || '', encodingType)} ${maxKeys} - ${escapeXml(delimiter || '')} + ${encodeKey(delimiter || '', encodingType)} + ${encodingType ? `${escapeXml(encodingType)}` : ''} ${isTruncated} ${objects .map( (o) => ` - ${escapeXml(o.key)} + ${encodeKey(o.key, encodingType)} ${isoDate(o.lastModified)} "${o.etag}" ${o.sizeBytes} @@ -62,11 +69,11 @@ export const listBucketResultXml = ( ${prefixes .map( (p) => ` - ${escapeXml(p)} + ${encodeKey(p, encodingType)} `, ) .join('')} - ${nextMarker ? `${escapeXml(nextMarker)}` : ''} + ${nextMarker ? `${encodeKey(nextMarker, encodingType)}` : ''} `; export const listBucketV2ResultXml = ( @@ -81,19 +88,21 @@ export const listBucketV2ResultXml = ( nextContinuationToken: string | null, keyCount: number, _requestId: string, + encodingType: string | null = null, ): string => ` ${escapeXml(bucketName)} - ${escapeXml(prefix)} + ${encodeKey(prefix, encodingType)} ${maxKeys} ${keyCount} - ${delimiter ? `${escapeXml(delimiter)}` : ''} - ${continuationToken ? `${escapeXml(continuationToken)}` : ''} + ${delimiter ? `${encodeKey(delimiter, encodingType)}` : ''} + ${encodingType ? `${escapeXml(encodingType)}` : ''} + ${continuationToken ? `${encodeKey(continuationToken, encodingType)}` : ''} ${isTruncated} ${objects .map( (o) => ` - ${escapeXml(o.key)} + ${encodeKey(o.key, encodingType)} ${isoDate(o.lastModified)} "${o.etag}" ${o.sizeBytes} @@ -104,11 +113,11 @@ export const listBucketV2ResultXml = ( ${prefixes .map( (p) => ` - ${escapeXml(p)} + ${encodeKey(p, encodingType)} `, ) .join('')} - ${nextContinuationToken ? `${escapeXml(nextContinuationToken)}` : ''} + ${nextContinuationToken ? `${encodeKey(nextContinuationToken, encodingType)}` : ''} `; // ─────── Multipart ─────── @@ -151,6 +160,35 @@ export const listPartsXml = ( .join('')} `; +export const listMultipartUploadsXml = ( + bucketName: string, + uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[], + maxUploads: number, + isTruncated: boolean, + nextKeyMarker: string | null, + _requestId: string, +): string => ` + + ${escapeXml(bucketName)} + + + ${nextKeyMarker ? `${escapeXml(nextKeyMarker)}` : ''} + ${maxUploads} + ${isTruncated} + ${uploads + .map( + (u) => ` + ${escapeXml(u.key)} + ${u.uploadId} + ${escapeXml(u.initiatedBy || 's3')}${escapeXml(u.initiatedBy || 's3')} + ${escapeXml(u.initiatedBy || 's3')}${escapeXml(u.initiatedBy || 's3')} + STANDARD + ${isoDate(u.initiatedAt)} + `, + ) + .join('')} +`; + export const completeMultipartUploadXml = ( bucketName: string, key: string, @@ -213,6 +251,7 @@ export const s3ErrorXml = ( ${escapeXml(message)} ${escapeXml(resource)} ${requestId} + ${requestId} `; export const s3ErrorResponse = ( @@ -225,11 +264,10 @@ export const s3ErrorResponse = ( ): Response => new Response(s3ErrorXml(code, message, resource, requestId), { status, - headers: { + headers: s3Headers(requestId, { 'content-type': 'application/xml', - ...(requestId ? { 'x-amz-request-id': requestId } : {}), ...extraHeaders, - }, + }), }); // ─────── DeleteObjects XML parser ───────