feat: S3 client compatibility — virtual-hosted style, CORS, presigned multi-method, ListMultipartUploads, edge case fixes

- Virtual-hosted bucket detection from Host header (extractS3BucketFromHost)
- S3 CORS headers + OPTIONS preflight + x-amz-id-2/HostId everywhere
- Presigned GET/PUT/HEAD/DELETE via centralized auth (no GET-only restriction)
- ListMultipartUploads with DB/xml helpers
- UploadPart partNumber range validation (1-10000)
- CompleteMultipartUpload ETag matching and ascending order validation
- DeleteObjects quiet mode support
- CopyObject URL-decode and conditional if-match/if-none-match
- encoding-type=url support in ListObjects V1/V2 XML
- Safe range-based prefix matching (replaces SQL LIKE)
- STREAMING-AWS4-HMAC-SHA256-PAYLOAD → 501 NotImplemented
- Traefik wildcard HostRegex for virtual-hosted style
- S3_VHOST_DOMAINS config env var
This commit is contained in:
asepharyana
2026-07-07 05:39:23 +07:00
parent 6aee3723fb
commit 8e6ccf2110
12 changed files with 417 additions and 179 deletions
+1
View File
@@ -14,3 +14,4 @@ RATE_LIMIT_MAX_REQUESTS=30
# S3_ACCESS_KEY=teleuploader-admin
# S3_SECRET_KEY=your-secret-key-here
# S3_DEFAULT_REGION=us-east-1
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
+1 -1
View File
@@ -48,7 +48,7 @@ services:
- app-shared-net
labels:
- "traefik.enable=true"
- "traefik.http.routers.teleuploader.rule=Host(`upload.asepharyana.my.id`) || Host(`upload.asepharyana.web.id`)"
- "traefik.http.routers.teleuploader.rule=Host(`upload.asepharyana.my.id`) || Host(`upload.asepharyana.web.id`) || HostRegexp(`{subhost:[a-z0-9][a-z0-9.-]+}.upload.asepharyana.my.id`) || HostRegexp(`{subhost:[a-z0-9][a-z0-9.-]+}.upload.asepharyana.web.id`)"
- "traefik.http.routers.teleuploader.entrypoints=websecure"
- "traefik.http.routers.teleuploader.tls=true"
- "traefik.http.routers.teleuploader.tls.certresolver=cloudflare"
+3 -1
View File
@@ -62,7 +62,9 @@ export const listObjectsByPrefix = async (
maxKeys: number,
startAfter: string | null,
): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => {
let query = sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${prefix}%`}`;
let query = prefix
? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key >= ${prefix} AND s3_key < ${`${prefix}￿`}`
: sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`;
if (startAfter) {
query = sql`${query} AND s3_key > ${startAfter}`;
+37
View File
@@ -90,3 +90,40 @@ export const listMultipartParts = async (uploadId: string): Promise<MultipartPar
createdAt: new Date(r.created_at as string),
}));
};
const mapRowToMultipartUpload = (r: Record<string, unknown>): MultipartUpload => ({
uploadId: r.upload_id as string,
bucketId: r.bucket_id as string,
s3Key: r.s3_key as string,
initiatedAt: new Date(r.initiated_at as string),
status: r.status as string,
initiatedBy: (r.initiated_by as string | null) || '',
});
export const listMultipartUploadsByBucket = async (
bucketId: string,
maxUploads: number,
keyMarker: string | null,
): Promise<{ uploads: MultipartUpload[]; isTruncated: boolean; nextKeyMarker: string | null }> => {
const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000);
const result = (await db.execute(
keyMarker
? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
FROM multipart_uploads
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker}
ORDER BY s3_key, initiated_at
LIMIT ${limit + 1}`
: sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
FROM multipart_uploads
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress'
ORDER BY s3_key, initiated_at
LIMIT ${limit + 1}`,
)) as unknown as Record<string, unknown>[];
const uploads = result.slice(0, limit).map(mapRowToMultipartUpload);
return {
uploads,
isTruncated: result.length > limit,
nextKeyMarker: result.length > limit ? uploads.at(-1)?.s3Key || null : null,
};
};
+12
View File
@@ -20,6 +20,7 @@ interface AppConfig {
s3SecretKey: string;
s3DefaultRegion: string;
proxyS3Get: boolean;
s3VhostDomains: string[];
}
const requiredEnv = {
@@ -50,6 +51,14 @@ const parseTokens = (value: string | undefined): string[] =>
.map((t) => t.trim())
.filter((t) => t !== '');
const parseDomains = (value: string | undefined): string[] =>
parseTokens(value).map((domain) =>
domain
.replace(/^https?:\/\//, '')
.split('/')[0]
.toLowerCase(),
);
const maskSecret = (value: string): string => {
if (!value) return '';
if (value.length <= 10) return '***';
@@ -80,6 +89,9 @@ export const config: AppConfig = {
s3SecretKey: process.env.S3_SECRET_KEY || '',
s3DefaultRegion: process.env.S3_DEFAULT_REGION || 'us-east-1',
proxyS3Get: process.env.PROXY_S3_GET !== 'false',
s3VhostDomains: parseDomains(
process.env.S3_VHOST_DOMAINS || 'upload.asepharyana.my.id,upload.asepharyana.web.id',
),
};
logger.info('Environment variables loaded', {
+36 -33
View File
@@ -13,6 +13,7 @@ import logger from './utils/logger';
import { metricsCollector } from './utils/metrics';
import { cleanupRateLimitCache, withRateLimit } from './utils/rateLimit';
import { isS3Request } from './utils/s3/auth';
import { extractS3BucketFromHost } from './utils/s3/virtual-host';
// ─── Auto-run migration at startup ──────────────────────────────────────────
try {
@@ -30,6 +31,29 @@ try {
logger.warn('Auto-migration warning (non-fatal)');
}
const getS3RouteBucket = (req: Request): string | null => {
const host = req.headers.get('host') || '';
return extractS3BucketFromHost(host, config.s3VhostDomains);
};
const shouldHandleS3 = (req: Request, headers: Record<string, string>): boolean => {
const url = new URL(req.url);
return Boolean(
getS3RouteBucket(req) || isS3Request(headers) || url.searchParams.has('X-Amz-Signature'),
);
};
const handleMaybeS3Root = (req: Request): Response | Promise<Response> => {
if (req.method === 'OPTIONS') {
return handleS3Request(req, getS3RouteBucket(req));
}
const headers = Object.fromEntries(req.headers);
if (shouldHandleS3(req, headers)) {
return handleS3Request(req, getS3RouteBucket(req));
}
return new Response('Not Allowed', { status: 405 });
};
const server = serve({
port: config.port,
routes: {
@@ -54,40 +78,16 @@ const server = serve({
'/': {
GET: (req: Request) => {
const headers = Object.fromEntries(req.headers);
const url = new URL(req.url);
if (isS3Request(headers) || url.searchParams.has('X-Amz-Signature')) {
return handleS3Request(req);
if (shouldHandleS3(req, headers)) {
return handleS3Request(req, getS3RouteBucket(req));
}
return handleHome();
},
PUT: (req: Request) => {
const headers = Object.fromEntries(req.headers);
if (isS3Request(headers)) {
return handleS3Request(req);
}
return new Response('Not Allowed', { status: 405 });
},
HEAD: (req: Request) => {
const headers = Object.fromEntries(req.headers);
if (isS3Request(headers)) {
return handleS3Request(req);
}
return new Response('Not Allowed', { status: 405 });
},
DELETE: (req: Request) => {
const headers = Object.fromEntries(req.headers);
if (isS3Request(headers)) {
return handleS3Request(req);
}
return new Response('Not Allowed', { status: 405 });
},
POST: (req: Request) => {
const headers = Object.fromEntries(req.headers);
if (isS3Request(headers)) {
return handleS3Request(req);
}
return new Response('Not Allowed', { status: 405 });
},
PUT: handleMaybeS3Root,
HEAD: handleMaybeS3Root,
DELETE: handleMaybeS3Root,
POST: handleMaybeS3Root,
OPTIONS: handleMaybeS3Root,
},
'/api/v1/*': {
GET: handleWebApiV1,
@@ -97,9 +97,12 @@ const server = serve({
},
},
fetch: async (req: Request) => {
if (req.method === 'OPTIONS') {
return handleS3Request(req, getS3RouteBucket(req));
}
const headers = Object.fromEntries(req.headers);
if (isS3Request(headers) || new URL(req.url).searchParams.has('X-Amz-Signature')) {
return handleS3Request(req);
if (shouldHandleS3(req, headers)) {
return handleS3Request(req, getS3RouteBucket(req));
}
return new Response('Not Found', { status: 404 });
},
+201 -127
View File
@@ -14,12 +14,14 @@ import {
findMultipartUpload,
insertMultipartPart,
listMultipartParts,
listMultipartUploadsByBucket,
} from '../db/multipart';
import type { File } from '../db/schema';
import { config } from '../env';
import { cleanupTempFile, computeHash, ensureExtension, getErrorMessage } from '../utils/file';
import logger from '../utils/logger';
import { verifyPresignedUrl, verifySignature } from '../utils/s3/auth';
import { S3_CORS_HEADERS, s3Headers } from '../utils/s3/headers';
import { createGetObjectResponse, type ObjectPartSource } from '../utils/s3/object-stream';
import { parseRangeHeader, unsatisfiedContentRange } from '../utils/s3/range';
import {
@@ -30,6 +32,7 @@ import {
listBucketResultXml,
listBucketsXml,
listBucketV2ResultXml,
listMultipartUploadsXml,
listPartsXml,
parseCompleteMultipartBody,
parseDeleteObjectsBody,
@@ -40,6 +43,16 @@ import { forwardToStorage, getFileInfo } from '../utils/telegram';
const REGION = config.s3DefaultRegion || 'us-east-1';
const REQUEST_ID = () => nanoid(16);
const s3Response = (
body: string | null,
status: number,
reqId: string,
extraHeaders: Record<string, string> = {},
): Response => new Response(body, { status, headers: s3Headers(reqId, extraHeaders) });
const s3OptionsResponse = (): Response =>
new Response(null, { status: 204, headers: S3_CORS_HEADERS });
const parseS3Path = (pathname: string): { bucket: string | null; key: string | null } => {
const parts = pathname.split('/').filter(Boolean);
if (parts.length === 0) return { bucket: null, key: null };
@@ -57,45 +70,62 @@ const headersToRecord = (req: Request): Record<string, string> => {
// ─────── Main Dispatcher ───────
export const handleS3Request = async (req: Request): Promise<Response> => {
export const handleS3Request = async (
req: Request,
virtualHostBucket: string | null = null,
): Promise<Response> => {
const method = req.method;
const url = new URL(req.url);
const pathname = url.pathname;
const { bucket, key } = parseS3Path(pathname);
const { bucket, key } = virtualHostBucket
? {
bucket: virtualHostBucket,
key: pathname === '/' ? null : decodeURIComponent(pathname.slice(1)),
}
: parseS3Path(pathname);
const headers = headersToRecord(req);
const searchParams = url.searchParams;
const reqId = REQUEST_ID();
// Presigned URLs: only supported for GET (verified in handleGetObject)
if (searchParams.has('X-Amz-Signature')) {
if (method !== 'GET') {
return s3ErrorResponse(
'AccessDenied',
'Presigned URLs only supported for GET',
pathname,
403,
reqId,
if (method === 'OPTIONS') {
return s3OptionsResponse();
}
const isPresigned = searchParams.has('X-Amz-Signature');
const authResult = isPresigned
? await verifyPresignedUrl({
url: req.url,
method,
headers,
s3AccessKey: config.s3AccessKey,
s3SecretKey: config.s3SecretKey,
region: REGION,
})
: await verifySignature(
method,
req.url,
headers,
null,
config.s3AccessKey,
config.s3SecretKey,
REGION,
);
}
} else {
const authResult = await verifySignature(
method,
req.url,
headers,
null,
config.s3AccessKey,
config.s3SecretKey,
REGION,
if (!authResult.isValid) {
const status = authResult.errorCode === 'NotImplemented' ? 501 : 403;
const message =
authResult.errorCode === 'NotImplemented'
? 'aws-chunked streaming payloads are not supported.'
: isPresigned
? 'Presigned URL verification failed'
: 'Authentication required';
return s3ErrorResponse(
authResult.errorCode || 'AccessDenied',
message,
pathname,
status,
reqId,
);
if (!authResult.isValid) {
return s3ErrorResponse(
authResult.errorCode || 'AccessDenied',
'Authentication required',
pathname,
403,
reqId,
);
}
}
try {
@@ -116,6 +146,9 @@ export const handleS3Request = async (req: Request): Promise<Response> => {
// Bucket-level operations
if (!key) {
if (method === 'GET') {
if (searchParams.has('uploads')) {
return handleListMultipartUploads(bucket, searchParams, reqId);
}
const listType = searchParams.get('list-type');
if (listType === '2') {
return handleListObjectsV2(bucket, searchParams, reqId);
@@ -131,7 +164,7 @@ export const handleS3Request = async (req: Request): Promise<Response> => {
return handleDeleteObjects(bucket, body, reqId);
}
if (searchParams.has('tagging')) {
return new Response(null, { status: 204 });
return s3Response(null, 204, reqId);
}
}
return s3ErrorResponse(
@@ -162,8 +195,7 @@ export const handleS3Request = async (req: Request): Promise<Response> => {
}
// Standard object operations
if (method === 'GET')
return handleGetObject(bucket, key, searchParams, headers, req.url, reqId);
if (method === 'GET') return handleGetObject(bucket, key, searchParams, headers, reqId);
if (method === 'HEAD') return handleHeadObject(bucket, key, reqId);
if (method === 'PUT') return handlePutObject(bucket, key, searchParams, headers, req, reqId);
if (method === 'DELETE') return handleDeleteObject(bucket, key, reqId);
@@ -192,10 +224,7 @@ export const handleS3Request = async (req: Request): Promise<Response> => {
const handleListBuckets = async (reqId: string): Promise<Response> => {
const buckets = await listBuckets();
const xml = listBucketsXml(buckets, reqId);
return new Response(xml, {
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Response> => {
@@ -219,7 +248,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
);
}
await createBucket(bucketName);
return new Response(null, { status: 200, headers: { 'x-amz-request-id': reqId } });
return s3Response(null, 200, reqId);
};
const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Response> => {
@@ -233,7 +262,7 @@ const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Resp
reqId,
);
}
return new Response(null, { status: 200, headers: { 'x-amz-request-id': reqId } });
return s3Response(null, 200, reqId);
};
const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Response> => {
@@ -258,7 +287,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
);
}
await deleteBucket(bucketName);
return new Response(null, { status: 204, headers: { 'x-amz-request-id': reqId } });
return s3Response(null, 204, reqId);
};
// ─────── Object Operations ───────
@@ -266,31 +295,10 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
const handleGetObject = async (
bucket: string,
key: string,
searchParams: URLSearchParams,
_searchParams: URLSearchParams,
headers: Record<string, string>,
requestUrl: string,
reqId: string,
): Promise<Response> => {
if (searchParams.has('X-Amz-Signature')) {
const presignedResult = await verifyPresignedUrl({
url: requestUrl,
method: 'GET',
headers,
s3AccessKey: config.s3AccessKey,
s3SecretKey: config.s3SecretKey,
region: REGION,
});
if (!presignedResult.isValid) {
return s3ErrorResponse(
presignedResult.errorCode || 'AccessDenied',
'Presigned URL verification failed',
`/${bucket}/${key}`,
403,
reqId,
);
}
}
const bucketRecord = await findBucketByName(bucket);
if (!bucketRecord)
return s3ErrorResponse(
@@ -335,10 +343,7 @@ const handleGetObject = async (
if (!config.proxyS3Get) {
// Legacy 302 redirect path (when proxy is disabled)
return new Response(null, {
status: 302,
headers: { Location: redirectUrl, 'x-amz-request-id': reqId },
});
return s3Response(null, 302, reqId, { location: redirectUrl });
}
const part: ObjectPartSource = {
@@ -420,10 +425,7 @@ const handleGetMultipartObject = async (
}
if (!config.proxyS3Get) {
return new Response(null, {
status: 302,
headers: { Location: sources[0].telegramUrl, 'x-amz-request-id': reqId },
});
return s3Response(null, 302, reqId, { location: sources[0].telegramUrl });
}
try {
@@ -472,16 +474,14 @@ const handleHeadObject = async (bucket: string, key: string, reqId: string): Pro
reqId,
);
return new Response(null, {
status: 200,
headers: {
'content-type': file.mimeType,
'content-length': String(file.sizeBytes),
etag: `"${file.fileHash || nanoid(16)}"`,
'last-modified':
file.createdAt instanceof Date ? file.createdAt.toUTCString() : new Date().toUTCString(),
'x-amz-request-id': reqId,
},
return s3Response(null, 200, reqId, {
'content-type': file.mimeType,
'content-length': String(file.sizeBytes),
etag: `"${file.fileHash || nanoid(16)}"`,
'last-modified':
file.createdAt instanceof Date ? file.createdAt.toUTCString() : new Date().toUTCString(),
'accept-ranges': 'bytes',
'cache-control': 'public, max-age=31536000',
});
};
@@ -504,12 +504,12 @@ const handlePutObject = async (
);
if (searchParams.has('tagging')) {
return new Response(null, { status: 204 });
return s3Response(null, 204, reqId);
}
const copySource = headers['x-amz-copy-source'];
if (copySource) {
return handleCopyObject(bucket, key, copySource, bucketRecord.id, reqId);
return handleCopyObject(bucket, key, copySource, headers, bucketRecord.id, reqId);
}
// Read raw body — S3 clients send raw binary, not multipart/form-data
@@ -520,10 +520,7 @@ const handlePutObject = async (
const existing = await findFileByBucketAndKey(bucketRecord.id, key);
if (existing) {
return new Response(null, {
status: 200,
headers: { etag: `"${hash}"`, 'x-amz-request-id': reqId },
});
return s3Response(null, 200, reqId, { etag: `"${hash}"` });
}
return await storeFileToTelegram(fileBuffer, hash, key, bucketRecord, contentType, reqId);
@@ -579,19 +576,18 @@ const storeFileToTelegram = async (
await cleanupTempFile(tempPath);
return new Response(null, {
status: 200,
headers: { etag: `"${hash}"`, 'x-amz-request-id': reqId },
});
return s3Response(null, 200, reqId, { etag: `"${hash}"` });
};
const handleCopyObject = async (
_destBucket: string,
destKey: string,
copySource: string,
rawCopySource: string,
headers: Record<string, string>,
destBucketId: string,
reqId: string,
): Promise<Response> => {
const copySource = decodeURIComponent(rawCopySource);
const sourcePath = copySource.startsWith('/') ? copySource.slice(1) : copySource;
const parts = sourcePath.split('/');
const sourceBucket = parts[0];
@@ -617,6 +613,28 @@ const handleCopyObject = async (
reqId,
);
// Conditional copy: if-match / if-none-match checks
const ifMatch = headers['x-amz-copy-source-if-match'];
const ifNoneMatch = headers['x-amz-copy-source-if-none-match'];
if (ifMatch && sourceFile.fileHash && ifMatch !== `"${sourceFile.fileHash}"`) {
return s3ErrorResponse(
'PreconditionFailed',
'The preconditions you specified did not hold.',
copySource,
412,
reqId,
);
}
if (ifNoneMatch && sourceFile.fileHash && ifNoneMatch === `"${sourceFile.fileHash}"`) {
return s3ErrorResponse(
'PreconditionFailed',
'The preconditions you specified did not hold.',
copySource,
412,
reqId,
);
}
const publicId = nanoid();
const { db, files: fileSchema } = await import('../db/index');
@@ -641,10 +659,7 @@ const handleCopyObject = async (
});
const xml = copyObjectResultXml(sourceFile.fileHash || nanoid(16), new Date());
return new Response(xml, {
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
const handleDeleteObject = async (
@@ -663,7 +678,7 @@ const handleDeleteObject = async (
);
await softDeleteFile(bucketRecord.id, key);
return new Response(null, { status: 204, headers: { 'x-amz-request-id': reqId } });
return s3Response(null, 204, reqId);
};
const handleDeleteObjects = async (
@@ -681,17 +696,14 @@ const handleDeleteObjects = async (
reqId,
);
const { keys } = parseDeleteObjectsBody(body);
const { keys, quiet } = parseDeleteObjectsBody(body);
const deletedKeys: string[] = [];
for (const key of keys) {
const ok = await softDeleteFile(bucketRecord.id, key);
if (ok) deletedKeys.push(key);
}
const xml = deleteResultXml(deletedKeys, []);
return new Response(xml, {
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
const xml = quiet ? deleteResultXml([], []) : deleteResultXml(deletedKeys, []);
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
// ─────── Object Listing ───────
@@ -715,6 +727,7 @@ const handleListObjectsV1 = async (
const delimiter = searchParams.get('delimiter') || null;
const maxKeys = Math.min(parseInt(searchParams.get('max-keys') || '1000', 10), 1000);
const marker = searchParams.get('marker') || null;
const encodingType = searchParams.get('encoding-type') || null;
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix(
bucketRecord.id,
@@ -747,12 +760,10 @@ const handleListObjectsV1 = async (
delimiter,
nextMarker,
reqId,
encodingType,
);
return new Response(xml, {
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
const handleListObjectsV2 = async (
@@ -775,6 +786,7 @@ const handleListObjectsV2 = async (
const maxKeys = Math.min(parseInt(searchParams.get('max-keys') || '1000', 10), 1000);
const continuationToken = searchParams.get('continuation-token') || null;
const startAfter = searchParams.get('start-after') || null;
const encodingType = searchParams.get('encoding-type') || null;
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix(
bucketRecord.id,
@@ -808,12 +820,10 @@ const handleListObjectsV2 = async (
nextContinuationToken,
displayObjects.length,
reqId,
encodingType,
);
return new Response(xml, {
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
// ─────── Multipart Upload ───────
@@ -837,10 +847,7 @@ const handleCreateMultipartUpload = async (
const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3');
const xml = initiateMultipartUploadXml(bucket, key, uploadId);
return new Response(xml, {
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
const handleUploadPart = async (
@@ -852,6 +859,15 @@ const handleUploadPart = async (
): Promise<Response> => {
const uploadId = searchParams.get('uploadId')!;
const partNumber = parseInt(searchParams.get('partNumber')!, 10);
if (partNumber < 1 || partNumber > 10000) {
return s3ErrorResponse(
'InvalidArgument',
'Part number must be an integer between 1 and 10000',
`/${bucket}/${key}`,
400,
reqId,
);
}
const multipart = await findMultipartUpload(uploadId);
if (!multipart || multipart.s3Key !== key) {
@@ -889,10 +905,7 @@ const handleUploadPart = async (
etag,
});
return new Response(null, {
status: 200,
headers: { etag: `"${etag}"`, 'x-amz-request-id': reqId },
});
return s3Response(null, 200, reqId, { etag: `"${etag}"` });
};
const handleCompleteMultipartUpload = async (
@@ -917,6 +930,33 @@ const handleCompleteMultipartUpload = async (
const parts = parseCompleteMultipartBody(body);
const storedParts = await listMultipartParts(uploadId);
// Validate parts match stored parts in ascending order and correct ETags
const sortedParts = [...parts].sort((a, b) => a.partNumber - b.partNumber);
for (let i = 0; i < sortedParts.length; i++) {
if (sortedParts[i].partNumber !== i + 1) {
return s3ErrorResponse(
'InvalidPartOrder',
'The list of parts was not in ascending order. Parts must be ordered by part number.',
`/${bucket}/${key}`,
400,
reqId,
);
}
}
const storedMap = new Map(storedParts.map((p) => [p.partNumber, p]));
for (const part of parts) {
const stored = storedMap.get(part.partNumber);
if (!stored || stored.etag !== part.etag) {
return s3ErrorResponse(
'InvalidPart',
'One or more specified parts could not be found. The part might not have been uploaded, or the specified ETag might not match.',
`/${bucket}/${key}`,
400,
reqId,
);
}
}
if (parts.length !== storedParts.length) {
return s3ErrorResponse(
'InvalidPart',
@@ -958,10 +998,47 @@ const handleCompleteMultipartUpload = async (
const combinedEtag = storedParts.map((p) => p.etag).join('-');
const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location);
return new Response(xml, {
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
const handleListMultipartUploads = async (
bucket: string,
searchParams: URLSearchParams,
reqId: string,
): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket);
if (!bucketRecord)
return s3ErrorResponse(
'NoSuchBucket',
'The specified bucket does not exist.',
`/${bucket}`,
404,
reqId,
);
const maxUploads = Math.min(parseInt(searchParams.get('max-uploads') || '1000', 10), 1000);
const keyMarker = searchParams.get('key-marker') || null;
const { uploads, isTruncated, nextKeyMarker } = await listMultipartUploadsByBucket(
bucketRecord.id,
maxUploads,
keyMarker,
);
const xml = listMultipartUploadsXml(
bucket,
uploads.map((u) => ({
key: u.s3Key,
uploadId: u.uploadId,
initiatedAt: u.initiatedAt,
initiatedBy: u.initiatedBy,
})),
maxUploads,
isTruncated,
nextKeyMarker,
reqId,
);
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
const handleAbortMultipartUpload = async (
@@ -983,7 +1060,7 @@ const handleAbortMultipartUpload = async (
}
await abortMultipartUpload(uploadId);
return new Response(null, { status: 204, headers: { 'x-amz-request-id': reqId } });
return s3Response(null, 204, reqId);
};
const handleListParts = async (
@@ -1022,8 +1099,5 @@ const handleListParts = async (
reqId,
);
return new Response(xml, {
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
};
+3
View File
@@ -174,6 +174,9 @@ export const verifySignature = async (
const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams);
const contentSha256 = headers['x-amz-content-sha256'] || null;
if (contentSha256?.startsWith('STREAMING-')) {
return { isValid: false, credential: null, errorCode: 'NotImplemented' };
}
const hashedPayload = await getHashedPayload(body, contentSha256);
const canonicalRequest = buildCanonicalRequest(
+44
View File
@@ -0,0 +1,44 @@
export const S3_CORS_HEADERS: Record<string, string> = {
'access-control-allow-origin': '*',
'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS',
'access-control-allow-headers': [
'Authorization',
'Content-Type',
'Content-MD5',
'Range',
'If-Match',
'If-None-Match',
'If-Modified-Since',
'If-Unmodified-Since',
'X-Amz-*',
'x-amz-*',
].join(', '),
'access-control-expose-headers': [
'Accept-Ranges',
'Content-Length',
'Content-Range',
'Content-Type',
'ETag',
'Last-Modified',
'x-amz-id-2',
'x-amz-request-id',
].join(', '),
'access-control-max-age': '86400',
};
export const s3Headers = (
requestId: string,
extraHeaders: Record<string, string> = {},
): Record<string, string> => ({
...S3_CORS_HEADERS,
...(requestId ? { 'x-amz-request-id': requestId, 'x-amz-id-2': requestId } : {}),
...extraHeaders,
});
export const applyS3Headers = (headers: Headers, requestId: string): Headers => {
const result = new Headers(headers);
for (const [key, value] of Object.entries(s3Headers(requestId))) {
result.set(key, value);
}
return result;
};
+2 -1
View File
@@ -1,3 +1,4 @@
import { applyS3Headers } from './headers';
import { contentRange, type RangeParseResult } from './range';
export interface ObjectPartSource {
@@ -97,7 +98,7 @@ export const createGetObjectResponse = async (input: ObjectResponseInput): Promi
const end = input.range.type === 'valid' ? input.range.end : input.totalSize - 1;
const plannedParts = planParts(input.parts, start, end);
const contentLength = end >= start ? end - start + 1 : 0;
const headers = baseHeaders(input, contentLength);
const headers = applyS3Headers(baseHeaders(input, contentLength), input.reqId);
if (input.range.type === 'valid') {
headers.set('content-range', contentRange(start, end, input.totalSize));
+23
View File
@@ -0,0 +1,23 @@
const stripPort = (host: string): string => {
if (host.startsWith('[')) return host;
return host.split(':')[0].toLowerCase().replace(/\.$/, '');
};
const isValidBucketLabel = (bucket: string): boolean =>
/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) &&
!bucket.includes('..') &&
!bucket.includes('.-') &&
!bucket.includes('-.');
export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => {
const normalizedHost = stripPort(host);
for (const domain of domains) {
const normalizedDomain = stripPort(domain);
if (!normalizedDomain || normalizedHost === normalizedDomain) continue;
if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue;
const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1));
return isValidBucketLabel(bucket) ? bucket : null;
}
return null;
};
+53 -15
View File
@@ -1,3 +1,5 @@
import { s3Headers } from './headers';
const escapeXml = (str: string): string =>
str
.replace(/&/g, '&amp;')
@@ -8,6 +10,9 @@ const escapeXml = (str: string): string =>
const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z');
const encodeKey = (value: string, encodingType: string | null = null): string =>
encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value);
// ─────── Bucket operations ───────
export const listBucketsXml = (
@@ -40,18 +45,20 @@ export const listBucketResultXml = (
delimiter: string | null,
nextMarker: string | null,
_requestId: string,
encodingType: string | null = null,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>${escapeXml(bucketName)}</Name>
<Prefix>${escapeXml(prefix)}</Prefix>
<Marker>${escapeXml(marker || '')}</Marker>
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
<Marker>${encodeKey(marker || '', encodingType)}</Marker>
<MaxKeys>${maxKeys}</MaxKeys>
<Delimiter>${escapeXml(delimiter || '')}</Delimiter>
<Delimiter>${encodeKey(delimiter || '', encodingType)}</Delimiter>
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
<IsTruncated>${isTruncated}</IsTruncated>
${objects
.map(
(o) => `<Contents>
<Key>${escapeXml(o.key)}</Key>
<Key>${encodeKey(o.key, encodingType)}</Key>
<LastModified>${isoDate(o.lastModified)}</LastModified>
<ETag>"${o.etag}"</ETag>
<Size>${o.sizeBytes}</Size>
@@ -62,11 +69,11 @@ export const listBucketResultXml = (
${prefixes
.map(
(p) => `<CommonPrefixes>
<Prefix>${escapeXml(p)}</Prefix>
<Prefix>${encodeKey(p, encodingType)}</Prefix>
</CommonPrefixes>`,
)
.join('')}
${nextMarker ? `<NextMarker>${escapeXml(nextMarker)}</NextMarker>` : ''}
${nextMarker ? `<NextMarker>${encodeKey(nextMarker, encodingType)}</NextMarker>` : ''}
</ListBucketResult>`;
export const listBucketV2ResultXml = (
@@ -81,19 +88,21 @@ export const listBucketV2ResultXml = (
nextContinuationToken: string | null,
keyCount: number,
_requestId: string,
encodingType: string | null = null,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>${escapeXml(bucketName)}</Name>
<Prefix>${escapeXml(prefix)}</Prefix>
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
<MaxKeys>${maxKeys}</MaxKeys>
<KeyCount>${keyCount}</KeyCount>
${delimiter ? `<Delimiter>${escapeXml(delimiter)}</Delimiter>` : ''}
${continuationToken ? `<ContinuationToken>${escapeXml(continuationToken)}</ContinuationToken>` : ''}
${delimiter ? `<Delimiter>${encodeKey(delimiter, encodingType)}</Delimiter>` : ''}
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
${continuationToken ? `<ContinuationToken>${encodeKey(continuationToken, encodingType)}</ContinuationToken>` : ''}
<IsTruncated>${isTruncated}</IsTruncated>
${objects
.map(
(o) => `<Contents>
<Key>${escapeXml(o.key)}</Key>
<Key>${encodeKey(o.key, encodingType)}</Key>
<LastModified>${isoDate(o.lastModified)}</LastModified>
<ETag>"${o.etag}"</ETag>
<Size>${o.sizeBytes}</Size>
@@ -104,11 +113,11 @@ export const listBucketV2ResultXml = (
${prefixes
.map(
(p) => `<CommonPrefixes>
<Prefix>${escapeXml(p)}</Prefix>
<Prefix>${encodeKey(p, encodingType)}</Prefix>
</CommonPrefixes>`,
)
.join('')}
${nextContinuationToken ? `<NextContinuationToken>${escapeXml(nextContinuationToken)}</NextContinuationToken>` : ''}
${nextContinuationToken ? `<NextContinuationToken>${encodeKey(nextContinuationToken, encodingType)}</NextContinuationToken>` : ''}
</ListBucketResultV2>`;
// ─────── Multipart ───────
@@ -151,6 +160,35 @@ export const listPartsXml = (
.join('')}
</ListPartsResult>`;
export const listMultipartUploadsXml = (
bucketName: string,
uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[],
maxUploads: number,
isTruncated: boolean,
nextKeyMarker: string | null,
_requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>${escapeXml(bucketName)}</Bucket>
<KeyMarker></KeyMarker>
<UploadIdMarker></UploadIdMarker>
${nextKeyMarker ? `<NextKeyMarker>${escapeXml(nextKeyMarker)}</NextKeyMarker>` : ''}
<MaxUploads>${maxUploads}</MaxUploads>
<IsTruncated>${isTruncated}</IsTruncated>
${uploads
.map(
(u) => `<Upload>
<Key>${escapeXml(u.key)}</Key>
<UploadId>${u.uploadId}</UploadId>
<Initiator><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Initiator>
<Owner><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Owner>
<StorageClass>STANDARD</StorageClass>
<Initiated>${isoDate(u.initiatedAt)}</Initiated>
</Upload>`,
)
.join('')}
</ListMultipartUploadsResult>`;
export const completeMultipartUploadXml = (
bucketName: string,
key: string,
@@ -213,6 +251,7 @@ export const s3ErrorXml = (
<Message>${escapeXml(message)}</Message>
<Resource>${escapeXml(resource)}</Resource>
<RequestId>${requestId}</RequestId>
<HostId>${requestId}</HostId>
</Error>`;
export const s3ErrorResponse = (
@@ -225,11 +264,10 @@ export const s3ErrorResponse = (
): Response =>
new Response(s3ErrorXml(code, message, resource, requestId), {
status,
headers: {
headers: s3Headers(requestId, {
'content-type': 'application/xml',
...(requestId ? { 'x-amz-request-id': requestId } : {}),
...extraHeaders,
},
}),
});
// ─────── DeleteObjects XML parser ───────