feat: S3 client compatibility — virtual-hosted style, CORS, presigned multi-method, ListMultipartUploads, edge case fixes

- Virtual-hosted bucket detection from Host header (extractS3BucketFromHost)
- S3 CORS headers + OPTIONS preflight + x-amz-id-2/HostId everywhere
- Presigned GET/PUT/HEAD/DELETE via centralized auth (no GET-only restriction)
- ListMultipartUploads with DB/xml helpers
- UploadPart partNumber range validation (1-10000)
- CompleteMultipartUpload ETag matching and ascending order validation
- DeleteObjects quiet mode support
- CopyObject URL-decode and conditional if-match/if-none-match
- encoding-type=url support in ListObjects V1/V2 XML
- Safe range-based prefix matching (replaces SQL LIKE)
- STREAMING-AWS4-HMAC-SHA256-PAYLOAD → 501 NotImplemented
- Traefik wildcard HostRegex for virtual-hosted style
- S3_VHOST_DOMAINS config env var
This commit is contained in:
asepharyana
2026-07-07 05:39:23 +07:00
parent 6aee3723fb
commit 8e6ccf2110
12 changed files with 417 additions and 179 deletions
+2 -1
View File
@@ -13,4 +13,5 @@ RATE_LIMIT_MAX_REQUESTS=30
# S3-compatible API credentials # S3-compatible API credentials
# S3_ACCESS_KEY=teleuploader-admin # S3_ACCESS_KEY=teleuploader-admin
# S3_SECRET_KEY=your-secret-key-here # S3_SECRET_KEY=your-secret-key-here
# S3_DEFAULT_REGION=us-east-1 # S3_DEFAULT_REGION=us-east-1
# S3_VHOST_DOMAINS=upload.asepharyana.my.id,upload.asepharyana.web.id
+1 -1
View File
@@ -48,7 +48,7 @@ services:
- app-shared-net - app-shared-net
labels: labels:
- "traefik.enable=true" - "traefik.enable=true"
- "traefik.http.routers.teleuploader.rule=Host(`upload.asepharyana.my.id`) || Host(`upload.asepharyana.web.id`)" - "traefik.http.routers.teleuploader.rule=Host(`upload.asepharyana.my.id`) || Host(`upload.asepharyana.web.id`) || HostRegexp(`{subhost:[a-z0-9][a-z0-9.-]+}.upload.asepharyana.my.id`) || HostRegexp(`{subhost:[a-z0-9][a-z0-9.-]+}.upload.asepharyana.web.id`)"
- "traefik.http.routers.teleuploader.entrypoints=websecure" - "traefik.http.routers.teleuploader.entrypoints=websecure"
- "traefik.http.routers.teleuploader.tls=true" - "traefik.http.routers.teleuploader.tls=true"
- "traefik.http.routers.teleuploader.tls.certresolver=cloudflare" - "traefik.http.routers.teleuploader.tls.certresolver=cloudflare"
+3 -1
View File
@@ -62,7 +62,9 @@ export const listObjectsByPrefix = async (
maxKeys: number, maxKeys: number,
startAfter: string | null, startAfter: string | null,
): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => { ): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => {
let query = sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${prefix}%`}`; let query = prefix
? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key >= ${prefix} AND s3_key < ${`${prefix}￿`}`
: sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`;
if (startAfter) { if (startAfter) {
query = sql`${query} AND s3_key > ${startAfter}`; query = sql`${query} AND s3_key > ${startAfter}`;
+37
View File
@@ -90,3 +90,40 @@ export const listMultipartParts = async (uploadId: string): Promise<MultipartPar
createdAt: new Date(r.created_at as string), createdAt: new Date(r.created_at as string),
})); }));
}; };
const mapRowToMultipartUpload = (r: Record<string, unknown>): MultipartUpload => ({
uploadId: r.upload_id as string,
bucketId: r.bucket_id as string,
s3Key: r.s3_key as string,
initiatedAt: new Date(r.initiated_at as string),
status: r.status as string,
initiatedBy: (r.initiated_by as string | null) || '',
});
export const listMultipartUploadsByBucket = async (
bucketId: string,
maxUploads: number,
keyMarker: string | null,
): Promise<{ uploads: MultipartUpload[]; isTruncated: boolean; nextKeyMarker: string | null }> => {
const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000);
const result = (await db.execute(
keyMarker
? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
FROM multipart_uploads
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker}
ORDER BY s3_key, initiated_at
LIMIT ${limit + 1}`
: sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
FROM multipart_uploads
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress'
ORDER BY s3_key, initiated_at
LIMIT ${limit + 1}`,
)) as unknown as Record<string, unknown>[];
const uploads = result.slice(0, limit).map(mapRowToMultipartUpload);
return {
uploads,
isTruncated: result.length > limit,
nextKeyMarker: result.length > limit ? uploads.at(-1)?.s3Key || null : null,
};
};
+12
View File
@@ -20,6 +20,7 @@ interface AppConfig {
s3SecretKey: string; s3SecretKey: string;
s3DefaultRegion: string; s3DefaultRegion: string;
proxyS3Get: boolean; proxyS3Get: boolean;
s3VhostDomains: string[];
} }
const requiredEnv = { const requiredEnv = {
@@ -50,6 +51,14 @@ const parseTokens = (value: string | undefined): string[] =>
.map((t) => t.trim()) .map((t) => t.trim())
.filter((t) => t !== ''); .filter((t) => t !== '');
const parseDomains = (value: string | undefined): string[] =>
parseTokens(value).map((domain) =>
domain
.replace(/^https?:\/\//, '')
.split('/')[0]
.toLowerCase(),
);
const maskSecret = (value: string): string => { const maskSecret = (value: string): string => {
if (!value) return ''; if (!value) return '';
if (value.length <= 10) return '***'; if (value.length <= 10) return '***';
@@ -80,6 +89,9 @@ export const config: AppConfig = {
s3SecretKey: process.env.S3_SECRET_KEY || '', s3SecretKey: process.env.S3_SECRET_KEY || '',
s3DefaultRegion: process.env.S3_DEFAULT_REGION || 'us-east-1', s3DefaultRegion: process.env.S3_DEFAULT_REGION || 'us-east-1',
proxyS3Get: process.env.PROXY_S3_GET !== 'false', proxyS3Get: process.env.PROXY_S3_GET !== 'false',
s3VhostDomains: parseDomains(
process.env.S3_VHOST_DOMAINS || 'upload.asepharyana.my.id,upload.asepharyana.web.id',
),
}; };
logger.info('Environment variables loaded', { logger.info('Environment variables loaded', {
+36 -33
View File
@@ -13,6 +13,7 @@ import logger from './utils/logger';
import { metricsCollector } from './utils/metrics'; import { metricsCollector } from './utils/metrics';
import { cleanupRateLimitCache, withRateLimit } from './utils/rateLimit'; import { cleanupRateLimitCache, withRateLimit } from './utils/rateLimit';
import { isS3Request } from './utils/s3/auth'; import { isS3Request } from './utils/s3/auth';
import { extractS3BucketFromHost } from './utils/s3/virtual-host';
// ─── Auto-run migration at startup ────────────────────────────────────────── // ─── Auto-run migration at startup ──────────────────────────────────────────
try { try {
@@ -30,6 +31,29 @@ try {
logger.warn('Auto-migration warning (non-fatal)'); logger.warn('Auto-migration warning (non-fatal)');
} }
const getS3RouteBucket = (req: Request): string | null => {
const host = req.headers.get('host') || '';
return extractS3BucketFromHost(host, config.s3VhostDomains);
};
const shouldHandleS3 = (req: Request, headers: Record<string, string>): boolean => {
const url = new URL(req.url);
return Boolean(
getS3RouteBucket(req) || isS3Request(headers) || url.searchParams.has('X-Amz-Signature'),
);
};
const handleMaybeS3Root = (req: Request): Response | Promise<Response> => {
if (req.method === 'OPTIONS') {
return handleS3Request(req, getS3RouteBucket(req));
}
const headers = Object.fromEntries(req.headers);
if (shouldHandleS3(req, headers)) {
return handleS3Request(req, getS3RouteBucket(req));
}
return new Response('Not Allowed', { status: 405 });
};
const server = serve({ const server = serve({
port: config.port, port: config.port,
routes: { routes: {
@@ -54,40 +78,16 @@ const server = serve({
'/': { '/': {
GET: (req: Request) => { GET: (req: Request) => {
const headers = Object.fromEntries(req.headers); const headers = Object.fromEntries(req.headers);
const url = new URL(req.url); if (shouldHandleS3(req, headers)) {
if (isS3Request(headers) || url.searchParams.has('X-Amz-Signature')) { return handleS3Request(req, getS3RouteBucket(req));
return handleS3Request(req);
} }
return handleHome(); return handleHome();
}, },
PUT: (req: Request) => { PUT: handleMaybeS3Root,
const headers = Object.fromEntries(req.headers); HEAD: handleMaybeS3Root,
if (isS3Request(headers)) { DELETE: handleMaybeS3Root,
return handleS3Request(req); POST: handleMaybeS3Root,
} OPTIONS: handleMaybeS3Root,
return new Response('Not Allowed', { status: 405 });
},
HEAD: (req: Request) => {
const headers = Object.fromEntries(req.headers);
if (isS3Request(headers)) {
return handleS3Request(req);
}
return new Response('Not Allowed', { status: 405 });
},
DELETE: (req: Request) => {
const headers = Object.fromEntries(req.headers);
if (isS3Request(headers)) {
return handleS3Request(req);
}
return new Response('Not Allowed', { status: 405 });
},
POST: (req: Request) => {
const headers = Object.fromEntries(req.headers);
if (isS3Request(headers)) {
return handleS3Request(req);
}
return new Response('Not Allowed', { status: 405 });
},
}, },
'/api/v1/*': { '/api/v1/*': {
GET: handleWebApiV1, GET: handleWebApiV1,
@@ -97,9 +97,12 @@ const server = serve({
}, },
}, },
fetch: async (req: Request) => { fetch: async (req: Request) => {
if (req.method === 'OPTIONS') {
return handleS3Request(req, getS3RouteBucket(req));
}
const headers = Object.fromEntries(req.headers); const headers = Object.fromEntries(req.headers);
if (isS3Request(headers) || new URL(req.url).searchParams.has('X-Amz-Signature')) { if (shouldHandleS3(req, headers)) {
return handleS3Request(req); return handleS3Request(req, getS3RouteBucket(req));
} }
return new Response('Not Found', { status: 404 }); return new Response('Not Found', { status: 404 });
}, },
+201 -127
View File
@@ -14,12 +14,14 @@ import {
findMultipartUpload, findMultipartUpload,
insertMultipartPart, insertMultipartPart,
listMultipartParts, listMultipartParts,
listMultipartUploadsByBucket,
} from '../db/multipart'; } from '../db/multipart';
import type { File } from '../db/schema'; import type { File } from '../db/schema';
import { config } from '../env'; import { config } from '../env';
import { cleanupTempFile, computeHash, ensureExtension, getErrorMessage } from '../utils/file'; import { cleanupTempFile, computeHash, ensureExtension, getErrorMessage } from '../utils/file';
import logger from '../utils/logger'; import logger from '../utils/logger';
import { verifyPresignedUrl, verifySignature } from '../utils/s3/auth'; import { verifyPresignedUrl, verifySignature } from '../utils/s3/auth';
import { S3_CORS_HEADERS, s3Headers } from '../utils/s3/headers';
import { createGetObjectResponse, type ObjectPartSource } from '../utils/s3/object-stream'; import { createGetObjectResponse, type ObjectPartSource } from '../utils/s3/object-stream';
import { parseRangeHeader, unsatisfiedContentRange } from '../utils/s3/range'; import { parseRangeHeader, unsatisfiedContentRange } from '../utils/s3/range';
import { import {
@@ -30,6 +32,7 @@ import {
listBucketResultXml, listBucketResultXml,
listBucketsXml, listBucketsXml,
listBucketV2ResultXml, listBucketV2ResultXml,
listMultipartUploadsXml,
listPartsXml, listPartsXml,
parseCompleteMultipartBody, parseCompleteMultipartBody,
parseDeleteObjectsBody, parseDeleteObjectsBody,
@@ -40,6 +43,16 @@ import { forwardToStorage, getFileInfo } from '../utils/telegram';
const REGION = config.s3DefaultRegion || 'us-east-1'; const REGION = config.s3DefaultRegion || 'us-east-1';
const REQUEST_ID = () => nanoid(16); const REQUEST_ID = () => nanoid(16);
const s3Response = (
body: string | null,
status: number,
reqId: string,
extraHeaders: Record<string, string> = {},
): Response => new Response(body, { status, headers: s3Headers(reqId, extraHeaders) });
const s3OptionsResponse = (): Response =>
new Response(null, { status: 204, headers: S3_CORS_HEADERS });
const parseS3Path = (pathname: string): { bucket: string | null; key: string | null } => { const parseS3Path = (pathname: string): { bucket: string | null; key: string | null } => {
const parts = pathname.split('/').filter(Boolean); const parts = pathname.split('/').filter(Boolean);
if (parts.length === 0) return { bucket: null, key: null }; if (parts.length === 0) return { bucket: null, key: null };
@@ -57,45 +70,62 @@ const headersToRecord = (req: Request): Record<string, string> => {
// ─────── Main Dispatcher ─────── // ─────── Main Dispatcher ───────
export const handleS3Request = async (req: Request): Promise<Response> => { export const handleS3Request = async (
req: Request,
virtualHostBucket: string | null = null,
): Promise<Response> => {
const method = req.method; const method = req.method;
const url = new URL(req.url); const url = new URL(req.url);
const pathname = url.pathname; const pathname = url.pathname;
const { bucket, key } = parseS3Path(pathname); const { bucket, key } = virtualHostBucket
? {
bucket: virtualHostBucket,
key: pathname === '/' ? null : decodeURIComponent(pathname.slice(1)),
}
: parseS3Path(pathname);
const headers = headersToRecord(req); const headers = headersToRecord(req);
const searchParams = url.searchParams; const searchParams = url.searchParams;
const reqId = REQUEST_ID(); const reqId = REQUEST_ID();
// Presigned URLs: only supported for GET (verified in handleGetObject) if (method === 'OPTIONS') {
if (searchParams.has('X-Amz-Signature')) { return s3OptionsResponse();
if (method !== 'GET') { }
return s3ErrorResponse(
'AccessDenied', const isPresigned = searchParams.has('X-Amz-Signature');
'Presigned URLs only supported for GET', const authResult = isPresigned
pathname, ? await verifyPresignedUrl({
403, url: req.url,
reqId, method,
headers,
s3AccessKey: config.s3AccessKey,
s3SecretKey: config.s3SecretKey,
region: REGION,
})
: await verifySignature(
method,
req.url,
headers,
null,
config.s3AccessKey,
config.s3SecretKey,
REGION,
); );
}
} else { if (!authResult.isValid) {
const authResult = await verifySignature( const status = authResult.errorCode === 'NotImplemented' ? 501 : 403;
method, const message =
req.url, authResult.errorCode === 'NotImplemented'
headers, ? 'aws-chunked streaming payloads are not supported.'
null, : isPresigned
config.s3AccessKey, ? 'Presigned URL verification failed'
config.s3SecretKey, : 'Authentication required';
REGION, return s3ErrorResponse(
authResult.errorCode || 'AccessDenied',
message,
pathname,
status,
reqId,
); );
if (!authResult.isValid) {
return s3ErrorResponse(
authResult.errorCode || 'AccessDenied',
'Authentication required',
pathname,
403,
reqId,
);
}
} }
try { try {
@@ -116,6 +146,9 @@ export const handleS3Request = async (req: Request): Promise<Response> => {
// Bucket-level operations // Bucket-level operations
if (!key) { if (!key) {
if (method === 'GET') { if (method === 'GET') {
if (searchParams.has('uploads')) {
return handleListMultipartUploads(bucket, searchParams, reqId);
}
const listType = searchParams.get('list-type'); const listType = searchParams.get('list-type');
if (listType === '2') { if (listType === '2') {
return handleListObjectsV2(bucket, searchParams, reqId); return handleListObjectsV2(bucket, searchParams, reqId);
@@ -131,7 +164,7 @@ export const handleS3Request = async (req: Request): Promise<Response> => {
return handleDeleteObjects(bucket, body, reqId); return handleDeleteObjects(bucket, body, reqId);
} }
if (searchParams.has('tagging')) { if (searchParams.has('tagging')) {
return new Response(null, { status: 204 }); return s3Response(null, 204, reqId);
} }
} }
return s3ErrorResponse( return s3ErrorResponse(
@@ -162,8 +195,7 @@ export const handleS3Request = async (req: Request): Promise<Response> => {
} }
// Standard object operations // Standard object operations
if (method === 'GET') if (method === 'GET') return handleGetObject(bucket, key, searchParams, headers, reqId);
return handleGetObject(bucket, key, searchParams, headers, req.url, reqId);
if (method === 'HEAD') return handleHeadObject(bucket, key, reqId); if (method === 'HEAD') return handleHeadObject(bucket, key, reqId);
if (method === 'PUT') return handlePutObject(bucket, key, searchParams, headers, req, reqId); if (method === 'PUT') return handlePutObject(bucket, key, searchParams, headers, req, reqId);
if (method === 'DELETE') return handleDeleteObject(bucket, key, reqId); if (method === 'DELETE') return handleDeleteObject(bucket, key, reqId);
@@ -192,10 +224,7 @@ export const handleS3Request = async (req: Request): Promise<Response> => {
const handleListBuckets = async (reqId: string): Promise<Response> => { const handleListBuckets = async (reqId: string): Promise<Response> => {
const buckets = await listBuckets(); const buckets = await listBuckets();
const xml = listBucketsXml(buckets, reqId); const xml = listBucketsXml(buckets, reqId);
return new Response(xml, { return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
}; };
const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Response> => { const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Response> => {
@@ -219,7 +248,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
); );
} }
await createBucket(bucketName); await createBucket(bucketName);
return new Response(null, { status: 200, headers: { 'x-amz-request-id': reqId } }); return s3Response(null, 200, reqId);
}; };
const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Response> => { const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Response> => {
@@ -233,7 +262,7 @@ const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Resp
reqId, reqId,
); );
} }
return new Response(null, { status: 200, headers: { 'x-amz-request-id': reqId } }); return s3Response(null, 200, reqId);
}; };
const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Response> => { const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Response> => {
@@ -258,7 +287,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
); );
} }
await deleteBucket(bucketName); await deleteBucket(bucketName);
return new Response(null, { status: 204, headers: { 'x-amz-request-id': reqId } }); return s3Response(null, 204, reqId);
}; };
// ─────── Object Operations ─────── // ─────── Object Operations ───────
@@ -266,31 +295,10 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
const handleGetObject = async ( const handleGetObject = async (
bucket: string, bucket: string,
key: string, key: string,
searchParams: URLSearchParams, _searchParams: URLSearchParams,
headers: Record<string, string>, headers: Record<string, string>,
requestUrl: string,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
if (searchParams.has('X-Amz-Signature')) {
const presignedResult = await verifyPresignedUrl({
url: requestUrl,
method: 'GET',
headers,
s3AccessKey: config.s3AccessKey,
s3SecretKey: config.s3SecretKey,
region: REGION,
});
if (!presignedResult.isValid) {
return s3ErrorResponse(
presignedResult.errorCode || 'AccessDenied',
'Presigned URL verification failed',
`/${bucket}/${key}`,
403,
reqId,
);
}
}
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await findBucketByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
@@ -335,10 +343,7 @@ const handleGetObject = async (
if (!config.proxyS3Get) { if (!config.proxyS3Get) {
// Legacy 302 redirect path (when proxy is disabled) // Legacy 302 redirect path (when proxy is disabled)
return new Response(null, { return s3Response(null, 302, reqId, { location: redirectUrl });
status: 302,
headers: { Location: redirectUrl, 'x-amz-request-id': reqId },
});
} }
const part: ObjectPartSource = { const part: ObjectPartSource = {
@@ -420,10 +425,7 @@ const handleGetMultipartObject = async (
} }
if (!config.proxyS3Get) { if (!config.proxyS3Get) {
return new Response(null, { return s3Response(null, 302, reqId, { location: sources[0].telegramUrl });
status: 302,
headers: { Location: sources[0].telegramUrl, 'x-amz-request-id': reqId },
});
} }
try { try {
@@ -472,16 +474,14 @@ const handleHeadObject = async (bucket: string, key: string, reqId: string): Pro
reqId, reqId,
); );
return new Response(null, { return s3Response(null, 200, reqId, {
status: 200, 'content-type': file.mimeType,
headers: { 'content-length': String(file.sizeBytes),
'content-type': file.mimeType, etag: `"${file.fileHash || nanoid(16)}"`,
'content-length': String(file.sizeBytes), 'last-modified':
etag: `"${file.fileHash || nanoid(16)}"`, file.createdAt instanceof Date ? file.createdAt.toUTCString() : new Date().toUTCString(),
'last-modified': 'accept-ranges': 'bytes',
file.createdAt instanceof Date ? file.createdAt.toUTCString() : new Date().toUTCString(), 'cache-control': 'public, max-age=31536000',
'x-amz-request-id': reqId,
},
}); });
}; };
@@ -504,12 +504,12 @@ const handlePutObject = async (
); );
if (searchParams.has('tagging')) { if (searchParams.has('tagging')) {
return new Response(null, { status: 204 }); return s3Response(null, 204, reqId);
} }
const copySource = headers['x-amz-copy-source']; const copySource = headers['x-amz-copy-source'];
if (copySource) { if (copySource) {
return handleCopyObject(bucket, key, copySource, bucketRecord.id, reqId); return handleCopyObject(bucket, key, copySource, headers, bucketRecord.id, reqId);
} }
// Read raw body — S3 clients send raw binary, not multipart/form-data // Read raw body — S3 clients send raw binary, not multipart/form-data
@@ -520,10 +520,7 @@ const handlePutObject = async (
const existing = await findFileByBucketAndKey(bucketRecord.id, key); const existing = await findFileByBucketAndKey(bucketRecord.id, key);
if (existing) { if (existing) {
return new Response(null, { return s3Response(null, 200, reqId, { etag: `"${hash}"` });
status: 200,
headers: { etag: `"${hash}"`, 'x-amz-request-id': reqId },
});
} }
return await storeFileToTelegram(fileBuffer, hash, key, bucketRecord, contentType, reqId); return await storeFileToTelegram(fileBuffer, hash, key, bucketRecord, contentType, reqId);
@@ -579,19 +576,18 @@ const storeFileToTelegram = async (
await cleanupTempFile(tempPath); await cleanupTempFile(tempPath);
return new Response(null, { return s3Response(null, 200, reqId, { etag: `"${hash}"` });
status: 200,
headers: { etag: `"${hash}"`, 'x-amz-request-id': reqId },
});
}; };
const handleCopyObject = async ( const handleCopyObject = async (
_destBucket: string, _destBucket: string,
destKey: string, destKey: string,
copySource: string, rawCopySource: string,
headers: Record<string, string>,
destBucketId: string, destBucketId: string,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const copySource = decodeURIComponent(rawCopySource);
const sourcePath = copySource.startsWith('/') ? copySource.slice(1) : copySource; const sourcePath = copySource.startsWith('/') ? copySource.slice(1) : copySource;
const parts = sourcePath.split('/'); const parts = sourcePath.split('/');
const sourceBucket = parts[0]; const sourceBucket = parts[0];
@@ -617,6 +613,28 @@ const handleCopyObject = async (
reqId, reqId,
); );
// Conditional copy: if-match / if-none-match checks
const ifMatch = headers['x-amz-copy-source-if-match'];
const ifNoneMatch = headers['x-amz-copy-source-if-none-match'];
if (ifMatch && sourceFile.fileHash && ifMatch !== `"${sourceFile.fileHash}"`) {
return s3ErrorResponse(
'PreconditionFailed',
'The preconditions you specified did not hold.',
copySource,
412,
reqId,
);
}
if (ifNoneMatch && sourceFile.fileHash && ifNoneMatch === `"${sourceFile.fileHash}"`) {
return s3ErrorResponse(
'PreconditionFailed',
'The preconditions you specified did not hold.',
copySource,
412,
reqId,
);
}
const publicId = nanoid(); const publicId = nanoid();
const { db, files: fileSchema } = await import('../db/index'); const { db, files: fileSchema } = await import('../db/index');
@@ -641,10 +659,7 @@ const handleCopyObject = async (
}); });
const xml = copyObjectResultXml(sourceFile.fileHash || nanoid(16), new Date()); const xml = copyObjectResultXml(sourceFile.fileHash || nanoid(16), new Date());
return new Response(xml, { return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
}; };
const handleDeleteObject = async ( const handleDeleteObject = async (
@@ -663,7 +678,7 @@ const handleDeleteObject = async (
); );
await softDeleteFile(bucketRecord.id, key); await softDeleteFile(bucketRecord.id, key);
return new Response(null, { status: 204, headers: { 'x-amz-request-id': reqId } }); return s3Response(null, 204, reqId);
}; };
const handleDeleteObjects = async ( const handleDeleteObjects = async (
@@ -681,17 +696,14 @@ const handleDeleteObjects = async (
reqId, reqId,
); );
const { keys } = parseDeleteObjectsBody(body); const { keys, quiet } = parseDeleteObjectsBody(body);
const deletedKeys: string[] = []; const deletedKeys: string[] = [];
for (const key of keys) { for (const key of keys) {
const ok = await softDeleteFile(bucketRecord.id, key); const ok = await softDeleteFile(bucketRecord.id, key);
if (ok) deletedKeys.push(key); if (ok) deletedKeys.push(key);
} }
const xml = deleteResultXml(deletedKeys, []); const xml = quiet ? deleteResultXml([], []) : deleteResultXml(deletedKeys, []);
return new Response(xml, { return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
}; };
// ─────── Object Listing ─────── // ─────── Object Listing ───────
@@ -715,6 +727,7 @@ const handleListObjectsV1 = async (
const delimiter = searchParams.get('delimiter') || null; const delimiter = searchParams.get('delimiter') || null;
const maxKeys = Math.min(parseInt(searchParams.get('max-keys') || '1000', 10), 1000); const maxKeys = Math.min(parseInt(searchParams.get('max-keys') || '1000', 10), 1000);
const marker = searchParams.get('marker') || null; const marker = searchParams.get('marker') || null;
const encodingType = searchParams.get('encoding-type') || null;
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix( const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix(
bucketRecord.id, bucketRecord.id,
@@ -747,12 +760,10 @@ const handleListObjectsV1 = async (
delimiter, delimiter,
nextMarker, nextMarker,
reqId, reqId,
encodingType,
); );
return new Response(xml, { return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
}; };
const handleListObjectsV2 = async ( const handleListObjectsV2 = async (
@@ -775,6 +786,7 @@ const handleListObjectsV2 = async (
const maxKeys = Math.min(parseInt(searchParams.get('max-keys') || '1000', 10), 1000); const maxKeys = Math.min(parseInt(searchParams.get('max-keys') || '1000', 10), 1000);
const continuationToken = searchParams.get('continuation-token') || null; const continuationToken = searchParams.get('continuation-token') || null;
const startAfter = searchParams.get('start-after') || null; const startAfter = searchParams.get('start-after') || null;
const encodingType = searchParams.get('encoding-type') || null;
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix( const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix(
bucketRecord.id, bucketRecord.id,
@@ -808,12 +820,10 @@ const handleListObjectsV2 = async (
nextContinuationToken, nextContinuationToken,
displayObjects.length, displayObjects.length,
reqId, reqId,
encodingType,
); );
return new Response(xml, { return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
}; };
// ─────── Multipart Upload ─────── // ─────── Multipart Upload ───────
@@ -837,10 +847,7 @@ const handleCreateMultipartUpload = async (
const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3'); const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3');
const xml = initiateMultipartUploadXml(bucket, key, uploadId); const xml = initiateMultipartUploadXml(bucket, key, uploadId);
return new Response(xml, { return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
}; };
const handleUploadPart = async ( const handleUploadPart = async (
@@ -852,6 +859,15 @@ const handleUploadPart = async (
): Promise<Response> => { ): Promise<Response> => {
const uploadId = searchParams.get('uploadId')!; const uploadId = searchParams.get('uploadId')!;
const partNumber = parseInt(searchParams.get('partNumber')!, 10); const partNumber = parseInt(searchParams.get('partNumber')!, 10);
if (partNumber < 1 || partNumber > 10000) {
return s3ErrorResponse(
'InvalidArgument',
'Part number must be an integer between 1 and 10000',
`/${bucket}/${key}`,
400,
reqId,
);
}
const multipart = await findMultipartUpload(uploadId); const multipart = await findMultipartUpload(uploadId);
if (!multipart || multipart.s3Key !== key) { if (!multipart || multipart.s3Key !== key) {
@@ -889,10 +905,7 @@ const handleUploadPart = async (
etag, etag,
}); });
return new Response(null, { return s3Response(null, 200, reqId, { etag: `"${etag}"` });
status: 200,
headers: { etag: `"${etag}"`, 'x-amz-request-id': reqId },
});
}; };
const handleCompleteMultipartUpload = async ( const handleCompleteMultipartUpload = async (
@@ -917,6 +930,33 @@ const handleCompleteMultipartUpload = async (
const parts = parseCompleteMultipartBody(body); const parts = parseCompleteMultipartBody(body);
const storedParts = await listMultipartParts(uploadId); const storedParts = await listMultipartParts(uploadId);
// Validate parts match stored parts in ascending order and correct ETags
const sortedParts = [...parts].sort((a, b) => a.partNumber - b.partNumber);
for (let i = 0; i < sortedParts.length; i++) {
if (sortedParts[i].partNumber !== i + 1) {
return s3ErrorResponse(
'InvalidPartOrder',
'The list of parts was not in ascending order. Parts must be ordered by part number.',
`/${bucket}/${key}`,
400,
reqId,
);
}
}
const storedMap = new Map(storedParts.map((p) => [p.partNumber, p]));
for (const part of parts) {
const stored = storedMap.get(part.partNumber);
if (!stored || stored.etag !== part.etag) {
return s3ErrorResponse(
'InvalidPart',
'One or more specified parts could not be found. The part might not have been uploaded, or the specified ETag might not match.',
`/${bucket}/${key}`,
400,
reqId,
);
}
}
if (parts.length !== storedParts.length) { if (parts.length !== storedParts.length) {
return s3ErrorResponse( return s3ErrorResponse(
'InvalidPart', 'InvalidPart',
@@ -958,10 +998,47 @@ const handleCompleteMultipartUpload = async (
const combinedEtag = storedParts.map((p) => p.etag).join('-'); const combinedEtag = storedParts.map((p) => p.etag).join('-');
const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location); const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location);
return new Response(xml, { return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
status: 200, };
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
}); const handleListMultipartUploads = async (
bucket: string,
searchParams: URLSearchParams,
reqId: string,
): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket);
if (!bucketRecord)
return s3ErrorResponse(
'NoSuchBucket',
'The specified bucket does not exist.',
`/${bucket}`,
404,
reqId,
);
const maxUploads = Math.min(parseInt(searchParams.get('max-uploads') || '1000', 10), 1000);
const keyMarker = searchParams.get('key-marker') || null;
const { uploads, isTruncated, nextKeyMarker } = await listMultipartUploadsByBucket(
bucketRecord.id,
maxUploads,
keyMarker,
);
const xml = listMultipartUploadsXml(
bucket,
uploads.map((u) => ({
key: u.s3Key,
uploadId: u.uploadId,
initiatedAt: u.initiatedAt,
initiatedBy: u.initiatedBy,
})),
maxUploads,
isTruncated,
nextKeyMarker,
reqId,
);
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
}; };
const handleAbortMultipartUpload = async ( const handleAbortMultipartUpload = async (
@@ -983,7 +1060,7 @@ const handleAbortMultipartUpload = async (
} }
await abortMultipartUpload(uploadId); await abortMultipartUpload(uploadId);
return new Response(null, { status: 204, headers: { 'x-amz-request-id': reqId } }); return s3Response(null, 204, reqId);
}; };
const handleListParts = async ( const handleListParts = async (
@@ -1022,8 +1099,5 @@ const handleListParts = async (
reqId, reqId,
); );
return new Response(xml, { return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
status: 200,
headers: { 'content-type': 'application/xml', 'x-amz-request-id': reqId },
});
}; };
+3
View File
@@ -174,6 +174,9 @@ export const verifySignature = async (
const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams); const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams);
const contentSha256 = headers['x-amz-content-sha256'] || null; const contentSha256 = headers['x-amz-content-sha256'] || null;
if (contentSha256?.startsWith('STREAMING-')) {
return { isValid: false, credential: null, errorCode: 'NotImplemented' };
}
const hashedPayload = await getHashedPayload(body, contentSha256); const hashedPayload = await getHashedPayload(body, contentSha256);
const canonicalRequest = buildCanonicalRequest( const canonicalRequest = buildCanonicalRequest(
+44
View File
@@ -0,0 +1,44 @@
export const S3_CORS_HEADERS: Record<string, string> = {
'access-control-allow-origin': '*',
'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS',
'access-control-allow-headers': [
'Authorization',
'Content-Type',
'Content-MD5',
'Range',
'If-Match',
'If-None-Match',
'If-Modified-Since',
'If-Unmodified-Since',
'X-Amz-*',
'x-amz-*',
].join(', '),
'access-control-expose-headers': [
'Accept-Ranges',
'Content-Length',
'Content-Range',
'Content-Type',
'ETag',
'Last-Modified',
'x-amz-id-2',
'x-amz-request-id',
].join(', '),
'access-control-max-age': '86400',
};
export const s3Headers = (
requestId: string,
extraHeaders: Record<string, string> = {},
): Record<string, string> => ({
...S3_CORS_HEADERS,
...(requestId ? { 'x-amz-request-id': requestId, 'x-amz-id-2': requestId } : {}),
...extraHeaders,
});
export const applyS3Headers = (headers: Headers, requestId: string): Headers => {
const result = new Headers(headers);
for (const [key, value] of Object.entries(s3Headers(requestId))) {
result.set(key, value);
}
return result;
};
+2 -1
View File
@@ -1,3 +1,4 @@
import { applyS3Headers } from './headers';
import { contentRange, type RangeParseResult } from './range'; import { contentRange, type RangeParseResult } from './range';
export interface ObjectPartSource { export interface ObjectPartSource {
@@ -97,7 +98,7 @@ export const createGetObjectResponse = async (input: ObjectResponseInput): Promi
const end = input.range.type === 'valid' ? input.range.end : input.totalSize - 1; const end = input.range.type === 'valid' ? input.range.end : input.totalSize - 1;
const plannedParts = planParts(input.parts, start, end); const plannedParts = planParts(input.parts, start, end);
const contentLength = end >= start ? end - start + 1 : 0; const contentLength = end >= start ? end - start + 1 : 0;
const headers = baseHeaders(input, contentLength); const headers = applyS3Headers(baseHeaders(input, contentLength), input.reqId);
if (input.range.type === 'valid') { if (input.range.type === 'valid') {
headers.set('content-range', contentRange(start, end, input.totalSize)); headers.set('content-range', contentRange(start, end, input.totalSize));
+23
View File
@@ -0,0 +1,23 @@
const stripPort = (host: string): string => {
if (host.startsWith('[')) return host;
return host.split(':')[0].toLowerCase().replace(/\.$/, '');
};
const isValidBucketLabel = (bucket: string): boolean =>
/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) &&
!bucket.includes('..') &&
!bucket.includes('.-') &&
!bucket.includes('-.');
export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => {
const normalizedHost = stripPort(host);
for (const domain of domains) {
const normalizedDomain = stripPort(domain);
if (!normalizedDomain || normalizedHost === normalizedDomain) continue;
if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue;
const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1));
return isValidBucketLabel(bucket) ? bucket : null;
}
return null;
};
+53 -15
View File
@@ -1,3 +1,5 @@
import { s3Headers } from './headers';
const escapeXml = (str: string): string => const escapeXml = (str: string): string =>
str str
.replace(/&/g, '&amp;') .replace(/&/g, '&amp;')
@@ -8,6 +10,9 @@ const escapeXml = (str: string): string =>
const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z'); const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z');
const encodeKey = (value: string, encodingType: string | null = null): string =>
encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value);
// ─────── Bucket operations ─────── // ─────── Bucket operations ───────
export const listBucketsXml = ( export const listBucketsXml = (
@@ -40,18 +45,20 @@ export const listBucketResultXml = (
delimiter: string | null, delimiter: string | null,
nextMarker: string | null, nextMarker: string | null,
_requestId: string, _requestId: string,
encodingType: string | null = null,
): string => `<?xml version="1.0" encoding="UTF-8"?> ): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>${escapeXml(bucketName)}</Name> <Name>${escapeXml(bucketName)}</Name>
<Prefix>${escapeXml(prefix)}</Prefix> <Prefix>${encodeKey(prefix, encodingType)}</Prefix>
<Marker>${escapeXml(marker || '')}</Marker> <Marker>${encodeKey(marker || '', encodingType)}</Marker>
<MaxKeys>${maxKeys}</MaxKeys> <MaxKeys>${maxKeys}</MaxKeys>
<Delimiter>${escapeXml(delimiter || '')}</Delimiter> <Delimiter>${encodeKey(delimiter || '', encodingType)}</Delimiter>
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
<IsTruncated>${isTruncated}</IsTruncated> <IsTruncated>${isTruncated}</IsTruncated>
${objects ${objects
.map( .map(
(o) => `<Contents> (o) => `<Contents>
<Key>${escapeXml(o.key)}</Key> <Key>${encodeKey(o.key, encodingType)}</Key>
<LastModified>${isoDate(o.lastModified)}</LastModified> <LastModified>${isoDate(o.lastModified)}</LastModified>
<ETag>"${o.etag}"</ETag> <ETag>"${o.etag}"</ETag>
<Size>${o.sizeBytes}</Size> <Size>${o.sizeBytes}</Size>
@@ -62,11 +69,11 @@ export const listBucketResultXml = (
${prefixes ${prefixes
.map( .map(
(p) => `<CommonPrefixes> (p) => `<CommonPrefixes>
<Prefix>${escapeXml(p)}</Prefix> <Prefix>${encodeKey(p, encodingType)}</Prefix>
</CommonPrefixes>`, </CommonPrefixes>`,
) )
.join('')} .join('')}
${nextMarker ? `<NextMarker>${escapeXml(nextMarker)}</NextMarker>` : ''} ${nextMarker ? `<NextMarker>${encodeKey(nextMarker, encodingType)}</NextMarker>` : ''}
</ListBucketResult>`; </ListBucketResult>`;
export const listBucketV2ResultXml = ( export const listBucketV2ResultXml = (
@@ -81,19 +88,21 @@ export const listBucketV2ResultXml = (
nextContinuationToken: string | null, nextContinuationToken: string | null,
keyCount: number, keyCount: number,
_requestId: string, _requestId: string,
encodingType: string | null = null,
): string => `<?xml version="1.0" encoding="UTF-8"?> ): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>${escapeXml(bucketName)}</Name> <Name>${escapeXml(bucketName)}</Name>
<Prefix>${escapeXml(prefix)}</Prefix> <Prefix>${encodeKey(prefix, encodingType)}</Prefix>
<MaxKeys>${maxKeys}</MaxKeys> <MaxKeys>${maxKeys}</MaxKeys>
<KeyCount>${keyCount}</KeyCount> <KeyCount>${keyCount}</KeyCount>
${delimiter ? `<Delimiter>${escapeXml(delimiter)}</Delimiter>` : ''} ${delimiter ? `<Delimiter>${encodeKey(delimiter, encodingType)}</Delimiter>` : ''}
${continuationToken ? `<ContinuationToken>${escapeXml(continuationToken)}</ContinuationToken>` : ''} ${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
${continuationToken ? `<ContinuationToken>${encodeKey(continuationToken, encodingType)}</ContinuationToken>` : ''}
<IsTruncated>${isTruncated}</IsTruncated> <IsTruncated>${isTruncated}</IsTruncated>
${objects ${objects
.map( .map(
(o) => `<Contents> (o) => `<Contents>
<Key>${escapeXml(o.key)}</Key> <Key>${encodeKey(o.key, encodingType)}</Key>
<LastModified>${isoDate(o.lastModified)}</LastModified> <LastModified>${isoDate(o.lastModified)}</LastModified>
<ETag>"${o.etag}"</ETag> <ETag>"${o.etag}"</ETag>
<Size>${o.sizeBytes}</Size> <Size>${o.sizeBytes}</Size>
@@ -104,11 +113,11 @@ export const listBucketV2ResultXml = (
${prefixes ${prefixes
.map( .map(
(p) => `<CommonPrefixes> (p) => `<CommonPrefixes>
<Prefix>${escapeXml(p)}</Prefix> <Prefix>${encodeKey(p, encodingType)}</Prefix>
</CommonPrefixes>`, </CommonPrefixes>`,
) )
.join('')} .join('')}
${nextContinuationToken ? `<NextContinuationToken>${escapeXml(nextContinuationToken)}</NextContinuationToken>` : ''} ${nextContinuationToken ? `<NextContinuationToken>${encodeKey(nextContinuationToken, encodingType)}</NextContinuationToken>` : ''}
</ListBucketResultV2>`; </ListBucketResultV2>`;
// ─────── Multipart ─────── // ─────── Multipart ───────
@@ -151,6 +160,35 @@ export const listPartsXml = (
.join('')} .join('')}
</ListPartsResult>`; </ListPartsResult>`;
export const listMultipartUploadsXml = (
bucketName: string,
uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[],
maxUploads: number,
isTruncated: boolean,
nextKeyMarker: string | null,
_requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>${escapeXml(bucketName)}</Bucket>
<KeyMarker></KeyMarker>
<UploadIdMarker></UploadIdMarker>
${nextKeyMarker ? `<NextKeyMarker>${escapeXml(nextKeyMarker)}</NextKeyMarker>` : ''}
<MaxUploads>${maxUploads}</MaxUploads>
<IsTruncated>${isTruncated}</IsTruncated>
${uploads
.map(
(u) => `<Upload>
<Key>${escapeXml(u.key)}</Key>
<UploadId>${u.uploadId}</UploadId>
<Initiator><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Initiator>
<Owner><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Owner>
<StorageClass>STANDARD</StorageClass>
<Initiated>${isoDate(u.initiatedAt)}</Initiated>
</Upload>`,
)
.join('')}
</ListMultipartUploadsResult>`;
export const completeMultipartUploadXml = ( export const completeMultipartUploadXml = (
bucketName: string, bucketName: string,
key: string, key: string,
@@ -213,6 +251,7 @@ export const s3ErrorXml = (
<Message>${escapeXml(message)}</Message> <Message>${escapeXml(message)}</Message>
<Resource>${escapeXml(resource)}</Resource> <Resource>${escapeXml(resource)}</Resource>
<RequestId>${requestId}</RequestId> <RequestId>${requestId}</RequestId>
<HostId>${requestId}</HostId>
</Error>`; </Error>`;
export const s3ErrorResponse = ( export const s3ErrorResponse = (
@@ -225,11 +264,10 @@ export const s3ErrorResponse = (
): Response => ): Response =>
new Response(s3ErrorXml(code, message, resource, requestId), { new Response(s3ErrorXml(code, message, resource, requestId), {
status, status,
headers: { headers: s3Headers(requestId, {
'content-type': 'application/xml', 'content-type': 'application/xml',
...(requestId ? { 'x-amz-request-id': requestId } : {}),
...extraHeaders, ...extraHeaders,
}, }),
}); });
// ─────── DeleteObjects XML parser ─────── // ─────── DeleteObjects XML parser ───────