refactor: full DDD + Clean Architecture refactor
Deploy FileDrop / deploy (push) Successful in 48s
Deploy FileDrop / deploy (push) Successful in 48s
- Hapus src/utils/ (17 files) + src/db/ (8 files) dead code - Absorb 7 re-export stubs → real impl di lokasi DDD - Buat src/infrastructure/di.ts (DI container) - Rewrite 5 controllers pakai repository/DI - Fix shared/utils imports, env.ts, routes, index.ts - Update package.json build path migrate - Lint clean, build clean
This commit is contained in:
+1
-1
@@ -5,7 +5,7 @@
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "bun --hot src/index.ts",
|
||||
"build": "bun build src/index.ts --target=bun --outfile=dist/index.js && bun build src/db/migrate.ts --target=bun --outfile=dist/migrate.js",
|
||||
"build": "bun build src/index.ts --target=bun --outfile=dist/index.js && bun build src/infrastructure/persistence/drizzle/migrate.ts --target=bun --outfile=dist/migrate.js",
|
||||
"start": "NODE_ENV=production bun dist/index.js",
|
||||
"db:migrate": "bun dist/migrate.js",
|
||||
"test": "bun test --preload ./test/helpers/setup-env.ts test/rateLimit.test.ts && bun test --preload ./test/helpers/setup-env.ts test/file.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegram.test.ts && bun test --preload ./test/helpers/setup-env.ts test/upload.test.ts && bun test --preload ./test/helpers/setup-env.ts test/files.test.ts && bun test --preload ./test/helpers/setup-env.ts test/health.test.ts && bun test --preload ./test/helpers/setup-env.ts test/db.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bot.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bootstrap.test.ts && bun test --preload ./test/helpers/setup-env.ts test/swagger.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth-routes.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-operations.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-bucket-config.test.ts && bun test --preload ./test/helpers/setup-env.ts test/web-api.test.ts && bun test --preload ./test/helpers/setup-env.ts test/env.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegramQueue.test.ts",
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { db } from './index';
|
||||
|
||||
export interface Bucket {
|
||||
id: string;
|
||||
name: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
type QueryRow = Record<string, unknown>;
|
||||
type QueryResult = QueryRow[];
|
||||
|
||||
export const createBucket = async (name: string): Promise<Bucket> => {
|
||||
const result = (await db.execute(
|
||||
sql`INSERT INTO buckets (name) VALUES (${name}) RETURNING id, name, created_at, updated_at`,
|
||||
)) as unknown as QueryResult;
|
||||
const row = result[0]!;
|
||||
return {
|
||||
id: row.id as string,
|
||||
name: row.name as string,
|
||||
createdAt: new Date(row.created_at as string),
|
||||
updatedAt: new Date(row.updated_at as string),
|
||||
};
|
||||
};
|
||||
|
||||
export const findBucketByName = async (name: string): Promise<Bucket | null> => {
|
||||
const result = (await db.execute(
|
||||
sql`SELECT id, name, created_at, updated_at FROM buckets WHERE name = ${name}`,
|
||||
)) as unknown as QueryResult;
|
||||
if (result.length === 0) return null;
|
||||
const row = result[0]!;
|
||||
return {
|
||||
id: row.id as string,
|
||||
name: row.name as string,
|
||||
createdAt: new Date(row.created_at as string),
|
||||
updatedAt: new Date(row.updated_at as string),
|
||||
};
|
||||
};
|
||||
|
||||
export const listBuckets = async (): Promise<Bucket[]> => {
|
||||
const result = (await db.execute(
|
||||
sql`SELECT id, name, created_at, updated_at FROM buckets ORDER BY name`,
|
||||
)) as unknown as QueryResult;
|
||||
return result.map((row) => ({
|
||||
id: row.id as string,
|
||||
name: row.name as string,
|
||||
createdAt: new Date(row.created_at as string),
|
||||
updatedAt: new Date(row.updated_at as string),
|
||||
}));
|
||||
};
|
||||
|
||||
export const deleteBucket = async (name: string): Promise<boolean> => {
|
||||
// Cascade-delete rows that hold FK references to the bucket
|
||||
await db
|
||||
.execute(
|
||||
sql`DELETE FROM multipart_parts WHERE upload_id IN (SELECT upload_id FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name}))`,
|
||||
)
|
||||
.catch(() => {});
|
||||
await db
|
||||
.execute(
|
||||
sql`DELETE FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
|
||||
)
|
||||
.catch(() => {});
|
||||
await db
|
||||
.execute(
|
||||
sql`DELETE FROM files WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
|
||||
)
|
||||
.catch(() => {});
|
||||
const result = (await db.execute(
|
||||
sql`DELETE FROM buckets WHERE name = ${name}`,
|
||||
)) as unknown as QueryResult;
|
||||
return result.length > 0;
|
||||
};
|
||||
|
||||
export const bucketExists = async (name: string): Promise<boolean> => {
|
||||
const result = (await db.execute(
|
||||
sql`SELECT 1 FROM buckets WHERE name = ${name}`,
|
||||
)) as unknown as QueryResult;
|
||||
return result.length > 0;
|
||||
};
|
||||
@@ -1,97 +0,0 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { db } from './index';
|
||||
|
||||
export type CompressionAlgorithm = 'gzip' | null;
|
||||
|
||||
export interface FilePart {
|
||||
id: number;
|
||||
fileId: string;
|
||||
partNumber: number;
|
||||
telegramFileId: string;
|
||||
telegramFileUniqueId: string;
|
||||
storageChatId: number;
|
||||
storageMessageId: number;
|
||||
sizeBytes: number;
|
||||
storedSizeBytes: number;
|
||||
compressionAlgorithm: CompressionAlgorithm;
|
||||
etag: string;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export type NewFilePartInput = Omit<FilePart, 'id' | 'createdAt'>;
|
||||
|
||||
const toNumber = (value: unknown): number => Number(value ?? 0);
|
||||
|
||||
const mapRowToFilePart = (row: Record<string, unknown>): FilePart => ({
|
||||
id: toNumber(row.id),
|
||||
fileId: row.file_id as string,
|
||||
partNumber: toNumber(row.part_number),
|
||||
telegramFileId: row.telegram_file_id as string,
|
||||
telegramFileUniqueId: row.telegram_file_unique_id as string,
|
||||
storageChatId: toNumber(row.storage_chat_id),
|
||||
storageMessageId: toNumber(row.storage_message_id),
|
||||
sizeBytes: toNumber(row.size_bytes),
|
||||
storedSizeBytes: toNumber(row.stored_size_bytes),
|
||||
compressionAlgorithm: (row.compression_algorithm as CompressionAlgorithm) || null,
|
||||
etag: row.etag as string,
|
||||
createdAt: new Date(row.created_at as string),
|
||||
});
|
||||
|
||||
export const insertFileParts = async (parts: NewFilePartInput[]): Promise<void> => {
|
||||
for (const part of parts) {
|
||||
await db.execute(
|
||||
sql`INSERT INTO file_parts (
|
||||
file_id,
|
||||
part_number,
|
||||
telegram_file_id,
|
||||
telegram_file_unique_id,
|
||||
storage_chat_id,
|
||||
storage_message_id,
|
||||
size_bytes,
|
||||
stored_size_bytes,
|
||||
compression_algorithm,
|
||||
etag
|
||||
) VALUES (
|
||||
${part.fileId}::uuid,
|
||||
${part.partNumber},
|
||||
${part.telegramFileId},
|
||||
${part.telegramFileUniqueId},
|
||||
${part.storageChatId},
|
||||
${part.storageMessageId},
|
||||
${part.sizeBytes},
|
||||
${part.storedSizeBytes},
|
||||
${part.compressionAlgorithm},
|
||||
${part.etag}
|
||||
)`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const listFileParts = async (fileId: string): Promise<FilePart[]> => {
|
||||
const result = (await db.execute(
|
||||
sql`SELECT id,
|
||||
file_id,
|
||||
part_number,
|
||||
telegram_file_id,
|
||||
telegram_file_unique_id,
|
||||
storage_chat_id,
|
||||
storage_message_id,
|
||||
size_bytes,
|
||||
stored_size_bytes,
|
||||
compression_algorithm,
|
||||
etag,
|
||||
created_at
|
||||
FROM file_parts
|
||||
WHERE file_id = ${fileId}::uuid
|
||||
ORDER BY part_number`,
|
||||
)) as unknown as Record<string, unknown>[];
|
||||
|
||||
return result.map(mapRowToFilePart);
|
||||
};
|
||||
|
||||
export const countFileParts = async (fileId: string): Promise<number> => {
|
||||
const result = (await db.execute(
|
||||
sql`SELECT COUNT(*) AS count FROM file_parts WHERE file_id = ${fileId}::uuid`,
|
||||
)) as unknown as Record<string, unknown>[];
|
||||
return toNumber(result[0]?.count);
|
||||
};
|
||||
@@ -1,143 +0,0 @@
|
||||
import { and, eq, sql } from 'drizzle-orm';
|
||||
import { db, files as fileSchema } from './index';
|
||||
import type { File } from './schema';
|
||||
|
||||
export interface S3FileRecord extends File {
|
||||
bucketId: string;
|
||||
s3Key: string;
|
||||
}
|
||||
|
||||
export const findFileByBucketAndKey = async (
|
||||
bucketId: string,
|
||||
s3Key: string,
|
||||
): Promise<File | null> => {
|
||||
const result = await db
|
||||
.select()
|
||||
.from(fileSchema)
|
||||
.where(
|
||||
and(
|
||||
eq(fileSchema.bucketId, bucketId),
|
||||
eq(fileSchema.s3Key, s3Key),
|
||||
eq(fileSchema.isDeleted, false),
|
||||
),
|
||||
)
|
||||
.limit(1);
|
||||
return result[0] || null;
|
||||
};
|
||||
|
||||
const mapDbRowToS3Record = (row: Record<string, unknown>): S3FileRecord => {
|
||||
return {
|
||||
id: row.id as string,
|
||||
publicId: row.public_id as string,
|
||||
telegramFileId: row.telegram_file_id as string,
|
||||
telegramFileUniqueId: row.telegram_file_unique_id as string,
|
||||
storageChatId: toNumber(row.storage_chat_id),
|
||||
storageMessageId: toNumber(row.storage_message_id),
|
||||
fileName: row.file_name as string,
|
||||
mimeType: row.mime_type as string,
|
||||
sizeBytes: toNumber(row.size_bytes),
|
||||
fileType: row.file_type as string,
|
||||
uploaderId: toNumber(row.uploader_id),
|
||||
fileHash: row.file_hash as string | null,
|
||||
archiveTelegramFileId: row.archive_telegram_file_id as string | null,
|
||||
archiveStorageMessageId:
|
||||
row.archive_storage_message_id === null ? null : toNumber(row.archive_storage_message_id),
|
||||
archiveFileName: row.archive_file_name as string | null,
|
||||
archiveEntryName: row.archive_entry_name as string | null,
|
||||
archiveMimeType: row.archive_mime_type as string | null,
|
||||
archiveSizeBytes: row.archive_size_bytes === null ? null : toNumber(row.archive_size_bytes),
|
||||
bucketId: row.bucket_id as string,
|
||||
s3Key: row.s3_key as string,
|
||||
storageBackend: (row.storage_backend as string) || 'telegram',
|
||||
isDeleted: row.is_deleted as boolean,
|
||||
multipartUploadId: row.multipart_upload_id as string | null,
|
||||
partCount:
|
||||
row.part_count === null || row.part_count === undefined ? null : toNumber(row.part_count),
|
||||
createdAt: new Date(row.created_at as string),
|
||||
updatedAt: new Date(row.updated_at as string),
|
||||
};
|
||||
};
|
||||
|
||||
const escapeLike = (s: string): string => s.replace(/[%_\\]/g, '\\$&');
|
||||
|
||||
const toNumber = (value: unknown): number => Number(value ?? 0);
|
||||
|
||||
export const listObjectsByPrefix = async (
|
||||
bucketId: string,
|
||||
prefix: string,
|
||||
delimiter: string | null,
|
||||
maxKeys: number,
|
||||
startAfter: string | null,
|
||||
): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => {
|
||||
let query = prefix
|
||||
? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${escapeLike(prefix)}%`}`
|
||||
: sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`;
|
||||
|
||||
if (startAfter) {
|
||||
query = sql`${query} AND s3_key > ${startAfter}`;
|
||||
}
|
||||
|
||||
query = sql`${query} ORDER BY s3_key LIMIT ${maxKeys + 1}`;
|
||||
|
||||
const rawResult = (await db.execute(query)) as unknown as Record<string, unknown>[];
|
||||
|
||||
if (delimiter === '/') {
|
||||
const prefixSet = new Set<string>();
|
||||
const objects: S3FileRecord[] = [];
|
||||
|
||||
for (const row of rawResult) {
|
||||
const s3Key = row.s3_key as string;
|
||||
const relativeKey = s3Key.substring(prefix.length);
|
||||
const slashIndex = relativeKey.indexOf('/');
|
||||
if (slashIndex >= 0) {
|
||||
const folderPrefix = prefix + relativeKey.substring(0, slashIndex + 1);
|
||||
if (folderPrefix !== prefix) {
|
||||
prefixSet.add(folderPrefix);
|
||||
}
|
||||
} else {
|
||||
objects.push(mapDbRowToS3Record(row));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
objects: objects.slice(0, maxKeys),
|
||||
prefixes: Array.from(prefixSet).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
objects: rawResult.slice(0, maxKeys).map(mapDbRowToS3Record),
|
||||
prefixes: [],
|
||||
};
|
||||
};
|
||||
|
||||
export const softDeleteFile = async (bucketId: string, s3Key: string): Promise<boolean> => {
|
||||
const result = (await db.execute(
|
||||
sql`UPDATE files SET is_deleted = true WHERE bucket_id = ${bucketId}::uuid AND s3_key = ${s3Key} RETURNING id`,
|
||||
)) as unknown as Record<string, unknown>[];
|
||||
return result.length > 0;
|
||||
};
|
||||
|
||||
export const softDeleteFilesBatch = async (bucketId: string, keys: string[]): Promise<number> => {
|
||||
let deleted = 0;
|
||||
for (const key of keys) {
|
||||
const ok = await softDeleteFile(bucketId, key);
|
||||
if (ok) deleted++;
|
||||
}
|
||||
return deleted;
|
||||
};
|
||||
|
||||
export const countBucketObjects = async (bucketId: string): Promise<number> => {
|
||||
const result = (await db.execute(
|
||||
sql`SELECT count(*) as count FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`,
|
||||
)) as unknown as Record<string, unknown>[];
|
||||
return Number(result[0]?.count || 0);
|
||||
};
|
||||
|
||||
export const findOrphanFilesByBucket = async (bucketId: string): Promise<File[]> => {
|
||||
return await db
|
||||
.select()
|
||||
.from(fileSchema)
|
||||
.where(and(eq(fileSchema.bucketId, bucketId), eq(fileSchema.isDeleted, true)))
|
||||
.limit(100);
|
||||
};
|
||||
@@ -1,26 +0,0 @@
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { db, files as fileSchema } from './index';
|
||||
import type { File } from './schema';
|
||||
|
||||
export const findFileByHash = async (hash: string): Promise<File | null> => {
|
||||
const result = await db.select().from(fileSchema).where(eq(fileSchema.fileHash, hash)).limit(1);
|
||||
return result[0] || null;
|
||||
};
|
||||
|
||||
export const findFileByPublicId = async (publicId: string): Promise<File | null> => {
|
||||
const result = await db
|
||||
.select()
|
||||
.from(fileSchema)
|
||||
.where(eq(fileSchema.publicId, publicId))
|
||||
.limit(1);
|
||||
return result[0] || null;
|
||||
};
|
||||
|
||||
export const findFileByUniqueId = async (telegramFileUniqueId: string): Promise<File | null> => {
|
||||
const result = await db
|
||||
.select()
|
||||
.from(fileSchema)
|
||||
.where(eq(fileSchema.telegramFileUniqueId, telegramFileUniqueId))
|
||||
.limit(1);
|
||||
return result[0] || null;
|
||||
};
|
||||
@@ -1,13 +0,0 @@
|
||||
import { drizzle } from 'drizzle-orm/postgres-js';
|
||||
import postgres from 'postgres';
|
||||
import { fileParts, files } from './schema';
|
||||
|
||||
const client = postgres(process.env.DATABASE_URL!, {
|
||||
max: 10,
|
||||
idle_timeout: 20,
|
||||
connect_timeout: 10,
|
||||
});
|
||||
|
||||
export const db = drizzle(client, { schema: { fileParts, files } });
|
||||
export { fileParts, files };
|
||||
export default db;
|
||||
@@ -1,53 +0,0 @@
|
||||
import postgres from 'postgres';
|
||||
import { config } from '../env';
|
||||
import { getErrorMessage } from '../utils/file';
|
||||
import logger from '../utils/logger';
|
||||
|
||||
/**
|
||||
* Run raw SQL migration from schema.sql.
|
||||
* Safe to call multiple times — all statements use IF NOT EXISTS.
|
||||
*/
|
||||
export const runMigration = async (): Promise<void> => {
|
||||
// In compiled dist: import.meta.dir = .../dist/
|
||||
// In source via bun --hot: import.meta.dir = .../src/db/
|
||||
const dir = import.meta.dir || '';
|
||||
const candidates = [
|
||||
`${dir}/../../schema.sql`, // from dist/
|
||||
`${dir}/../schema.sql`, // from src/ (bun --hot src/index.ts)
|
||||
`${dir}/../schema.sql`, // from src/db/ (bun --hot src/db/migrate.ts)
|
||||
`${dir}/schema.sql`, // from src/ (bun run db:migrate)
|
||||
];
|
||||
|
||||
let schemaSql: string | null = null;
|
||||
for (const p of candidates) {
|
||||
const file = Bun.file(p);
|
||||
const exists = await file.exists();
|
||||
if (exists) {
|
||||
schemaSql = await file.text();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!schemaSql) {
|
||||
logger.error(`Migration failed: schema.sql not found (tried ${candidates.join(', ')})`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const sql = postgres(config.databaseUrl, { max: 1 });
|
||||
|
||||
try {
|
||||
await sql.unsafe(schemaSql);
|
||||
logger.info('Database migration completed');
|
||||
} catch (error: unknown) {
|
||||
logger.error('Database migration failed', { error: getErrorMessage(error) });
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await sql.end();
|
||||
}
|
||||
};
|
||||
|
||||
// When run directly: `bun src/db/migrate.ts` or `bun dist/migrate.js`
|
||||
if (import.meta.path === Bun.main) {
|
||||
await runMigration();
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { db } from './index';
|
||||
|
||||
export interface MultipartUpload {
|
||||
uploadId: string;
|
||||
bucketId: string;
|
||||
s3Key: string;
|
||||
initiatedAt: Date;
|
||||
status: string;
|
||||
initiatedBy: string;
|
||||
contentType: string | null;
|
||||
}
|
||||
|
||||
export interface MultipartPart {
|
||||
id: number;
|
||||
uploadId: string;
|
||||
partNumber: number;
|
||||
telegramFileId: string;
|
||||
telegramFileUniqueId: string;
|
||||
storageMessageId: number;
|
||||
sizeBytes: number;
|
||||
etag: string;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export const createMultipartUpload = async (
|
||||
bucketId: string,
|
||||
s3Key: string,
|
||||
initiatedBy: string,
|
||||
contentType?: string | null,
|
||||
): Promise<string> => {
|
||||
const uploadId = nanoid(32);
|
||||
await db.execute(
|
||||
contentType
|
||||
? sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by, content_type) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy}, ${contentType})`
|
||||
: sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy})`,
|
||||
);
|
||||
return uploadId;
|
||||
};
|
||||
|
||||
export const findMultipartUpload = async (uploadId: string): Promise<MultipartUpload | null> => {
|
||||
const result = (await db.execute(
|
||||
sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, content_type FROM multipart_uploads WHERE upload_id = ${uploadId} AND status = 'in_progress'`,
|
||||
)) as unknown as Record<string, unknown>[];
|
||||
if (result.length === 0) return null;
|
||||
const r = result[0]!;
|
||||
return {
|
||||
uploadId: r.upload_id as string,
|
||||
bucketId: r.bucket_id as string,
|
||||
s3Key: r.s3_key as string,
|
||||
initiatedAt: new Date(r.initiated_at as string),
|
||||
status: r.status as string,
|
||||
initiatedBy: '',
|
||||
contentType: (r.content_type as string | null) || null,
|
||||
};
|
||||
};
|
||||
|
||||
export const completeMultipartUpload = async (uploadId: string): Promise<void> => {
|
||||
await db.execute(
|
||||
sql`UPDATE multipart_uploads SET status = 'completed' WHERE upload_id = ${uploadId}`,
|
||||
);
|
||||
};
|
||||
|
||||
export const abortMultipartUpload = async (uploadId: string): Promise<void> => {
|
||||
// H7: FK cascade only fires on DELETE, not UPDATE. Delete parts explicitly
|
||||
// before updating the upload status.
|
||||
await db.execute(sql`DELETE FROM multipart_parts WHERE upload_id = ${uploadId}`);
|
||||
await db.execute(
|
||||
sql`UPDATE multipart_uploads SET status = 'aborted' WHERE upload_id = ${uploadId}`,
|
||||
);
|
||||
};
|
||||
|
||||
export const insertMultipartPart = async (
|
||||
part: Omit<MultipartPart, 'id' | 'createdAt'>,
|
||||
): Promise<void> => {
|
||||
await db.execute(
|
||||
sql`INSERT INTO multipart_parts (upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag)
|
||||
VALUES (${part.uploadId}, ${part.partNumber}, ${part.telegramFileId}, ${part.telegramFileUniqueId}, ${part.storageMessageId}, ${part.sizeBytes}, ${part.etag})`,
|
||||
);
|
||||
};
|
||||
|
||||
export const listMultipartParts = async (uploadId: string): Promise<MultipartPart[]> => {
|
||||
const result = (await db.execute(
|
||||
sql`SELECT id, upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag, created_at
|
||||
FROM multipart_parts WHERE upload_id = ${uploadId} ORDER BY part_number`,
|
||||
)) as unknown as Record<string, unknown>[];
|
||||
return result.map((r) => ({
|
||||
id: r.id as number,
|
||||
uploadId: r.upload_id as string,
|
||||
partNumber: r.part_number as number,
|
||||
telegramFileId: r.telegram_file_id as string,
|
||||
telegramFileUniqueId: r.telegram_file_unique_id as string,
|
||||
storageMessageId: r.storage_message_id as number,
|
||||
sizeBytes: Number(r.size_bytes),
|
||||
etag: r.etag as string,
|
||||
createdAt: new Date(r.created_at as string),
|
||||
}));
|
||||
};
|
||||
|
||||
const mapRowToMultipartUpload = (r: Record<string, unknown>): MultipartUpload => ({
|
||||
uploadId: r.upload_id as string,
|
||||
bucketId: r.bucket_id as string,
|
||||
s3Key: r.s3_key as string,
|
||||
initiatedAt: new Date(r.initiated_at as string),
|
||||
status: r.status as string,
|
||||
initiatedBy: (r.initiated_by as string | null) || '',
|
||||
contentType: (r.content_type as string | null) || null,
|
||||
});
|
||||
|
||||
export const listMultipartUploadsByBucket = async (
|
||||
bucketId: string,
|
||||
maxUploads: number,
|
||||
keyMarker: string | null,
|
||||
): Promise<{ uploads: MultipartUpload[]; isTruncated: boolean; nextKeyMarker: string | null }> => {
|
||||
const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000);
|
||||
const result = (await db.execute(
|
||||
keyMarker
|
||||
? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
|
||||
FROM multipart_uploads
|
||||
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker}
|
||||
ORDER BY s3_key, initiated_at
|
||||
LIMIT ${limit + 1}`
|
||||
: sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
|
||||
FROM multipart_uploads
|
||||
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress'
|
||||
ORDER BY s3_key, initiated_at
|
||||
LIMIT ${limit + 1}`,
|
||||
)) as unknown as Record<string, unknown>[];
|
||||
|
||||
const uploads = result.slice(0, limit).map(mapRowToMultipartUpload);
|
||||
return {
|
||||
uploads,
|
||||
isTruncated: result.length > limit,
|
||||
nextKeyMarker: result.length > limit ? uploads.at(-1)?.s3Key || null : null,
|
||||
};
|
||||
};
|
||||
@@ -1,71 +0,0 @@
|
||||
import { eq, type InferInsertModel, type InferSelectModel } from 'drizzle-orm';
|
||||
import {
|
||||
bigint,
|
||||
boolean,
|
||||
integer,
|
||||
pgTable,
|
||||
serial,
|
||||
text,
|
||||
timestamp,
|
||||
uniqueIndex,
|
||||
uuid,
|
||||
} from 'drizzle-orm/pg-core';
|
||||
|
||||
export const files = pgTable(
|
||||
'files',
|
||||
{
|
||||
id: uuid('id').primaryKey().defaultRandom(),
|
||||
publicId: text('public_id').unique().notNull(),
|
||||
telegramFileId: text('telegram_file_id').notNull(),
|
||||
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
|
||||
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
|
||||
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
|
||||
fileName: text('file_name').notNull(),
|
||||
mimeType: text('mime_type').notNull(),
|
||||
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
|
||||
fileType: text('file_type').notNull(),
|
||||
uploaderId: bigint('uploader_id', { mode: 'number' }).notNull(),
|
||||
fileHash: text('file_hash'),
|
||||
archiveTelegramFileId: text('archive_telegram_file_id'),
|
||||
archiveStorageMessageId: bigint('archive_storage_message_id', { mode: 'number' }),
|
||||
archiveFileName: text('archive_file_name'),
|
||||
archiveEntryName: text('archive_entry_name'),
|
||||
archiveMimeType: text('archive_mime_type'),
|
||||
archiveSizeBytes: bigint('archive_size_bytes', { mode: 'number' }),
|
||||
bucketId: text('bucket_id'),
|
||||
s3Key: text('s3_key'),
|
||||
storageBackend: text('storage_backend').default('telegram'),
|
||||
isDeleted: boolean('is_deleted').default(false),
|
||||
multipartUploadId: text('multipart_upload_id'),
|
||||
partCount: integer('part_count'),
|
||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
||||
updatedAt: timestamp('updated_at').defaultNow().notNull(),
|
||||
},
|
||||
(table) => ({
|
||||
// H2: Prevent TOCTOU race on concurrent PUT — only one active (non-deleted)
|
||||
// object per (bucket_id, s3_key) pair.
|
||||
activeObjectIdx: uniqueIndex('active_object_idx')
|
||||
.on(table.bucketId, table.s3Key)
|
||||
.where(eq(table.isDeleted, false)),
|
||||
}),
|
||||
);
|
||||
|
||||
export const fileParts = pgTable('file_parts', {
|
||||
id: serial('id').primaryKey(),
|
||||
fileId: uuid('file_id').notNull(),
|
||||
partNumber: integer('part_number').notNull(),
|
||||
telegramFileId: text('telegram_file_id').notNull(),
|
||||
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
|
||||
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
|
||||
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
|
||||
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
|
||||
storedSizeBytes: bigint('stored_size_bytes', { mode: 'number' }).notNull(),
|
||||
compressionAlgorithm: text('compression_algorithm'),
|
||||
etag: text('etag').notNull(),
|
||||
createdAt: timestamp('created_at').defaultNow().notNull(),
|
||||
});
|
||||
|
||||
export type File = InferSelectModel<typeof files>;
|
||||
export type NewFile = InferInsertModel<typeof files>;
|
||||
export type FilePart = InferSelectModel<typeof fileParts>;
|
||||
export type NewFilePart = InferInsertModel<typeof fileParts>;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
import logger from './utils/logger';
|
||||
import logger from './shared/logger/index';
|
||||
|
||||
interface AppConfig {
|
||||
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import { metricsCollector } from './shared/metrics/index';
|
||||
|
||||
// ─── Auto-run migration at startup ──────────────────────────────────────────
|
||||
try {
|
||||
const { runMigration } = await import('./db/migrate');
|
||||
const { runMigration } = await import('./infrastructure/persistence/drizzle/migrate');
|
||||
await runMigration();
|
||||
} catch {
|
||||
logger.warn('Auto-migration skipped (non-fatal)');
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Dependency Injection container.
|
||||
*
|
||||
* Wires up singleton instances of all repositories and application services,
|
||||
* making them available to controllers and other adapters without requiring
|
||||
* a full DI framework.
|
||||
*
|
||||
* @module infrastructure/di
|
||||
*/
|
||||
|
||||
import type { IBucketRepository } from '../domain/ports/bucket-repository';
|
||||
import type { IFilePartRepository } from '../domain/ports/file-part-repository';
|
||||
import type { IFileRepository } from '../domain/ports/file-repository';
|
||||
import type { IMultipartRepository } from '../domain/ports/multipart-repository';
|
||||
import type { ITelegramService } from '../domain/ports/telegram-service';
|
||||
import { DrizzleBucketRepository } from './persistence/repositories/bucket-repository';
|
||||
import { DrizzleFilePartRepository } from './persistence/repositories/file-part-repository';
|
||||
import { DrizzleFileRepository } from './persistence/repositories/file-repository';
|
||||
import { DrizzleMultipartRepository } from './persistence/repositories/multipart-repository';
|
||||
import { botPool } from './telegram/bot-pool';
|
||||
import { ChunkedStorage } from './telegram/chunked-storage';
|
||||
import { UploadBatcher } from './telegram/upload-batcher';
|
||||
|
||||
// ─── Repository Singletons ──────────────────────────────────────────
|
||||
|
||||
/** Singleton IFileRepository instance backed by Drizzle ORM. */
|
||||
export const fileRepository: IFileRepository = new DrizzleFileRepository();
|
||||
|
||||
/** Singleton IBucketRepository instance backed by Drizzle ORM. */
|
||||
export const bucketRepository: IBucketRepository = new DrizzleBucketRepository();
|
||||
|
||||
/** Singleton IFilePartRepository instance backed by Drizzle ORM. */
|
||||
export const filePartRepository: IFilePartRepository = new DrizzleFilePartRepository();
|
||||
|
||||
/** Singleton IMultipartRepository instance backed by Drizzle ORM. */
|
||||
export const multipartRepository: IMultipartRepository = new DrizzleMultipartRepository();
|
||||
|
||||
/** Singleton ITelegramService instance backed by the bot pool. */
|
||||
export const telegramService: ITelegramService = botPool;
|
||||
|
||||
// ─── Service Singletons ─────────────────────────────────────────────
|
||||
|
||||
/** Singleton ChunkedStorage for large file chunked uploads. */
|
||||
export const chunkedStorage = new ChunkedStorage(
|
||||
fileRepository,
|
||||
filePartRepository,
|
||||
telegramService,
|
||||
);
|
||||
|
||||
/** Singleton UploadBatcher for batched small-file uploads. */
|
||||
export const uploadBatcher = new UploadBatcher(fileRepository, telegramService);
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
createSessionCookie,
|
||||
getAuthSession,
|
||||
isAuthEnabled,
|
||||
} from '../../../utils/auth';
|
||||
} from '../middleware/auth';
|
||||
|
||||
/**
|
||||
* Helper that builds a JSON Response with optional extra headers.
|
||||
|
||||
@@ -2,11 +2,11 @@ import { createReadStream } from 'node:fs';
|
||||
import { nanoid } from 'nanoid';
|
||||
import type { TelegramFileInfo } from '../../../domain/ports/telegram-service';
|
||||
import { fileInfoCache } from '../../../infrastructure/cache/index';
|
||||
import { chunkedStorage } from '../../../infrastructure/di';
|
||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||
import logger from '../../../shared/logger/index';
|
||||
import { cleanupTempFile, formatCreatedAt, getErrorMessage } from '../../../shared/utils/file';
|
||||
import { createChunkedObjectResponse } from '../../../utils/chunked-storage';
|
||||
import { locateZipEntry } from '../../../utils/zip';
|
||||
import { locateZipEntry } from '../../../shared/utils/zip';
|
||||
|
||||
/**
|
||||
* Extended Request type that includes route parameter access.
|
||||
@@ -115,7 +115,7 @@ export const handleFileRedirect = async (req: RequestWithParams): Promise<Respon
|
||||
return fail(501, 'Archive entry extraction is not supported for chunked files');
|
||||
}
|
||||
const range = { type: 'none' as const };
|
||||
return createChunkedObjectResponse({ file, range, reqId: '' });
|
||||
return chunkedStorage.createChunkedObjectResponse({ file, range, reqId: '' });
|
||||
}
|
||||
|
||||
const archiveEntryName = file.archiveEntryName;
|
||||
|
||||
@@ -1,36 +1,22 @@
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { config } from '../../../config/index';
|
||||
import { createBucket, deleteBucket, findBucketByName, listBuckets } from '../../../db/buckets';
|
||||
import {
|
||||
countBucketObjects,
|
||||
findFileByBucketAndKey,
|
||||
listObjectsByPrefix,
|
||||
softDeleteFile,
|
||||
} from '../../../db/files-ext';
|
||||
import { db, files as fileSchema } from '../../../db/index';
|
||||
import {
|
||||
abortMultipartUpload,
|
||||
completeMultipartUpload,
|
||||
createMultipartUpload,
|
||||
findMultipartUpload,
|
||||
insertMultipartPart,
|
||||
listMultipartParts,
|
||||
listMultipartUploadsByBucket,
|
||||
} from '../../../db/multipart';
|
||||
import type { File } from '../../../db/schema';
|
||||
import type { File as FileEntity } from '../../../domain/entities/file';
|
||||
import type { ForwardResult } from '../../../domain/ports/telegram-service';
|
||||
import {
|
||||
bucketRepository,
|
||||
chunkedStorage,
|
||||
fileRepository,
|
||||
multipartRepository,
|
||||
} from '../../../infrastructure/di';
|
||||
import { db, files as fileSchema } from '../../../infrastructure/persistence/drizzle/index';
|
||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||
import logger from '../../../shared/logger/index';
|
||||
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
|
||||
import {
|
||||
createChunkedObjectResponse,
|
||||
storeFileInTelegramChunks,
|
||||
} from '../../../utils/chunked-storage';
|
||||
import { verifyBodyHash, verifyPresignedUrl, verifySignature } from '../../../utils/s3/auth';
|
||||
import { S3_CORS_HEADERS, s3Headers } from '../../../utils/s3/headers';
|
||||
import { createGetObjectResponse, type ObjectPartSource } from '../../../utils/s3/object-stream';
|
||||
import { parseRangeHeader, unsatisfiedContentRange } from '../../../utils/s3/range';
|
||||
import { verifyBodyHash, verifyPresignedUrl, verifySignature } from '../../s3/auth';
|
||||
import { S3_CORS_HEADERS, s3Headers } from '../../s3/headers';
|
||||
import { createGetObjectResponse, type ObjectPartSource } from '../../s3/object-stream';
|
||||
import { parseRangeHeader, unsatisfiedContentRange } from '../../s3/range';
|
||||
import {
|
||||
bucketVersioningConfigurationXml,
|
||||
completeMultipartUploadXml,
|
||||
@@ -45,7 +31,7 @@ import {
|
||||
parseCompleteMultipartBody,
|
||||
parseDeleteObjectsBody,
|
||||
s3ErrorResponse,
|
||||
} from '../../../utils/s3/xml';
|
||||
} from '../../s3/xml';
|
||||
|
||||
/**
|
||||
* The default S3 region returned when no region is explicitly configured.
|
||||
@@ -309,7 +295,7 @@ export const handleS3Request = async (
|
||||
* @returns An S3 XML response with the bucket list.
|
||||
*/
|
||||
const handleListBuckets = async (reqId: string): Promise<Response> => {
|
||||
const buckets = await listBuckets();
|
||||
const buckets = await bucketRepository.list();
|
||||
const xml = listBucketsXml(buckets, reqId);
|
||||
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
||||
};
|
||||
@@ -339,7 +325,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
|
||||
reqId,
|
||||
);
|
||||
}
|
||||
const existing = await findBucketByName(bucketName);
|
||||
const existing = await bucketRepository.findByName(bucketName);
|
||||
if (existing) {
|
||||
return s3ErrorResponse(
|
||||
'BucketAlreadyExists',
|
||||
@@ -349,7 +335,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
|
||||
reqId,
|
||||
);
|
||||
}
|
||||
await createBucket(bucketName);
|
||||
await bucketRepository.create(bucketName);
|
||||
return s3Response(null, 200, reqId);
|
||||
};
|
||||
|
||||
@@ -361,7 +347,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
|
||||
* @returns A 200 response when the bucket exists, or an S3 XML error.
|
||||
*/
|
||||
const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Response> => {
|
||||
const bucket = await findBucketByName(bucketName);
|
||||
const bucket = await bucketRepository.findByName(bucketName);
|
||||
if (!bucket) {
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -384,7 +370,7 @@ const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Resp
|
||||
* @returns A 204 response on success, or an S3 XML error.
|
||||
*/
|
||||
const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Response> => {
|
||||
const bucket = await findBucketByName(bucketName);
|
||||
const bucket = await bucketRepository.findByName(bucketName);
|
||||
if (!bucket) {
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -394,7 +380,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
|
||||
reqId,
|
||||
);
|
||||
}
|
||||
const objCount = await countBucketObjects(bucket.id);
|
||||
const objCount = await fileRepository.countByBucket(bucket.id);
|
||||
if (objCount > 0) {
|
||||
return s3ErrorResponse(
|
||||
'BucketNotEmpty',
|
||||
@@ -404,7 +390,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
|
||||
reqId,
|
||||
);
|
||||
}
|
||||
await deleteBucket(bucketName);
|
||||
await bucketRepository.delete(bucketName);
|
||||
return s3Response(null, 204, reqId);
|
||||
};
|
||||
|
||||
@@ -417,7 +403,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
|
||||
* @returns An S3 XML response with the versioning configuration.
|
||||
*/
|
||||
const handleGetBucketVersioning = async (bucketName: string, reqId: string): Promise<Response> => {
|
||||
const bucket = await findBucketByName(bucketName);
|
||||
const bucket = await bucketRepository.findByName(bucketName);
|
||||
if (!bucket) {
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -456,7 +442,7 @@ const handleGetObject = async (
|
||||
headers: Record<string, string>,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -466,7 +452,7 @@ const handleGetObject = async (
|
||||
reqId,
|
||||
);
|
||||
|
||||
const file = await findFileByBucketAndKey(bucketRecord.id, key);
|
||||
const file = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
|
||||
if (!file)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchKey',
|
||||
@@ -551,7 +537,7 @@ const handleGetObject = async (
|
||||
);
|
||||
}
|
||||
try {
|
||||
return await createChunkedObjectResponse({ file, range, reqId });
|
||||
return await chunkedStorage.createChunkedObjectResponse({ file, range, reqId });
|
||||
} catch (error) {
|
||||
logger.warn('Chunked object content fetch failed', { key, error: getErrorMessage(error) });
|
||||
return s3ErrorResponse(
|
||||
@@ -638,14 +624,14 @@ const handleGetObject = async (
|
||||
* @returns An S3 response streaming the assembled object content.
|
||||
*/
|
||||
const handleGetMultipartObject = async (
|
||||
file: File,
|
||||
file: FileEntity,
|
||||
bucket: string,
|
||||
key: string,
|
||||
headers: Record<string, string>,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const uploadId = file.multipartUploadId!;
|
||||
const parts = await listMultipartParts(uploadId);
|
||||
const parts = await multipartRepository.listParts(uploadId);
|
||||
|
||||
if (parts.length === 0) {
|
||||
return s3ErrorResponse(
|
||||
@@ -724,7 +710,7 @@ const handleHeadObject = async (
|
||||
headers: Record<string, string>,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -734,7 +720,7 @@ const handleHeadObject = async (
|
||||
reqId,
|
||||
);
|
||||
|
||||
const file = await findFileByBucketAndKey(bucketRecord.id, key);
|
||||
const file = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
|
||||
if (!file)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchKey',
|
||||
@@ -930,7 +916,7 @@ const handlePutObject = async (
|
||||
req: Request,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -1011,7 +997,7 @@ const handlePutObject = async (
|
||||
|
||||
// Idempotent PUT: if the object already exists, skip upload
|
||||
try {
|
||||
const existing = await findFileByBucketAndKey(bucketRecord.id, key);
|
||||
const existing = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
|
||||
if (existing) {
|
||||
await cleanupTempFile(streamed.tempPath);
|
||||
return s3Response(null, 200, reqId, { etag: `"${streamed.fileHash}"` });
|
||||
@@ -1057,7 +1043,7 @@ const storeFileFromTemp = async (
|
||||
const partFileNamePrefix = `s3-${bucketRecord.name}-${key.replace(/\//g, '_')}`;
|
||||
|
||||
if (streamed.sizeBytes > config.telegramChunkSizeBytes) {
|
||||
const file = await storeFileInTelegramChunks({
|
||||
const file = await chunkedStorage.storeFileInTelegramChunks({
|
||||
tempPath: streamed.tempPath,
|
||||
partFileNamePrefix,
|
||||
fileName: finalFileName,
|
||||
@@ -1138,7 +1124,7 @@ const handleCopyObject = async (
|
||||
const sourceBucket = parts[0];
|
||||
const sourceKey = parts.slice(1).join('/');
|
||||
|
||||
const sourceBucketRecord = await findBucketByName(sourceBucket);
|
||||
const sourceBucketRecord = await bucketRepository.findByName(sourceBucket);
|
||||
if (!sourceBucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -1148,7 +1134,7 @@ const handleCopyObject = async (
|
||||
reqId,
|
||||
);
|
||||
|
||||
const sourceFile = await findFileByBucketAndKey(sourceBucketRecord.id, sourceKey);
|
||||
const sourceFile = await fileRepository.findByBucketAndKey(sourceBucketRecord.id, sourceKey);
|
||||
if (!sourceFile)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchKey',
|
||||
@@ -1232,7 +1218,7 @@ const handleDeleteObject = async (
|
||||
key: string,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -1242,7 +1228,7 @@ const handleDeleteObject = async (
|
||||
reqId,
|
||||
);
|
||||
|
||||
await softDeleteFile(bucketRecord.id, key);
|
||||
await fileRepository.softDelete(bucketRecord.id, key);
|
||||
return s3Response(null, 204, reqId);
|
||||
};
|
||||
|
||||
@@ -1262,7 +1248,7 @@ const handleDeleteObjects = async (
|
||||
body: string,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -1288,7 +1274,7 @@ const handleDeleteObjects = async (
|
||||
const deletedKeys: string[] = [];
|
||||
const errors: Array<{ key: string; code: string; message: string }> = [];
|
||||
for (const key of keys) {
|
||||
const ok = await softDeleteFile(bucketRecord.id, key);
|
||||
const ok = await fileRepository.softDelete(bucketRecord.id, key);
|
||||
if (ok) {
|
||||
deletedKeys.push(key);
|
||||
} else {
|
||||
@@ -1316,7 +1302,7 @@ const handleListObjectsV1 = async (
|
||||
searchParams: URLSearchParams,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -1335,7 +1321,7 @@ const handleListObjectsV1 = async (
|
||||
const marker = searchParams.get('marker') || null;
|
||||
const encodingType = searchParams.get('encoding-type') || null;
|
||||
|
||||
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix(
|
||||
const { objects, prefixes: commonPrefixes } = await fileRepository.listByPrefix(
|
||||
bucketRecord.id,
|
||||
prefix,
|
||||
delimiter,
|
||||
@@ -1386,7 +1372,7 @@ const handleListObjectsV2 = async (
|
||||
searchParams: URLSearchParams,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -1403,7 +1389,7 @@ const handleListObjectsV2 = async (
|
||||
const startAfter = searchParams.get('start-after') || null;
|
||||
const encodingType = searchParams.get('encoding-type') || null;
|
||||
|
||||
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix(
|
||||
const { objects, prefixes: commonPrefixes } = await fileRepository.listByPrefix(
|
||||
bucketRecord.id,
|
||||
prefix,
|
||||
delimiter,
|
||||
@@ -1459,7 +1445,7 @@ const handleCreateMultipartUpload = async (
|
||||
headers: Record<string, string>,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -1470,7 +1456,7 @@ const handleCreateMultipartUpload = async (
|
||||
);
|
||||
|
||||
const contentType = headers['content-type'] || null;
|
||||
const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3', contentType);
|
||||
const uploadId = await multipartRepository.create(bucketRecord.id, key, 's3', contentType);
|
||||
|
||||
const xml = initiateMultipartUploadXml(bucket, key, uploadId);
|
||||
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
|
||||
@@ -1514,7 +1500,7 @@ const handleUploadPart = async (
|
||||
);
|
||||
}
|
||||
|
||||
const multipart = await findMultipartUpload(uploadId);
|
||||
const multipart = await multipartRepository.findById(uploadId);
|
||||
if (!multipart || multipart.s3Key !== key) {
|
||||
return s3ErrorResponse(
|
||||
'NoSuchUpload',
|
||||
@@ -1583,7 +1569,7 @@ const handleUploadPart = async (
|
||||
await cleanupTempFile(tempPath);
|
||||
|
||||
const etag = hasher.digest('hex');
|
||||
await insertMultipartPart({
|
||||
await multipartRepository.insertPart({
|
||||
uploadId,
|
||||
partNumber,
|
||||
telegramFileId: forwardResult.telegramFileId,
|
||||
@@ -1617,7 +1603,7 @@ const handleCompleteMultipartUpload = async (
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const uploadId = searchParams.get('uploadId')!;
|
||||
const multipart = await findMultipartUpload(uploadId);
|
||||
const multipart = await multipartRepository.findById(uploadId);
|
||||
// H5: Verify both upload exists AND key matches (consistent with handleUploadPart)
|
||||
if (!multipart || multipart.s3Key !== key) {
|
||||
return s3ErrorResponse(
|
||||
@@ -1630,7 +1616,7 @@ const handleCompleteMultipartUpload = async (
|
||||
}
|
||||
|
||||
const parts = parseCompleteMultipartBody(body);
|
||||
const storedParts = await listMultipartParts(uploadId);
|
||||
const storedParts = await multipartRepository.listParts(uploadId);
|
||||
|
||||
// Validate ascending part order
|
||||
const partNumbers = parts.map((p) => p.partNumber);
|
||||
@@ -1702,7 +1688,7 @@ const handleCompleteMultipartUpload = async (
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
await completeMultipartUpload(uploadId);
|
||||
await multipartRepository.complete(uploadId);
|
||||
|
||||
const location = `${config.baseUrl}/${bucket}/${key}`;
|
||||
const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location);
|
||||
@@ -1723,7 +1709,7 @@ const handleListMultipartUploads = async (
|
||||
searchParams: URLSearchParams,
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const bucketRecord = await findBucketByName(bucket);
|
||||
const bucketRecord = await bucketRepository.findByName(bucket);
|
||||
if (!bucketRecord)
|
||||
return s3ErrorResponse(
|
||||
'NoSuchBucket',
|
||||
@@ -1735,7 +1721,7 @@ const handleListMultipartUploads = async (
|
||||
|
||||
const maxUploads = Math.min(Number.parseInt(searchParams.get('max-uploads') || '1000', 10), 1000);
|
||||
const keyMarker = searchParams.get('key-marker') || null;
|
||||
const { uploads, isTruncated, nextKeyMarker } = await listMultipartUploadsByBucket(
|
||||
const { uploads, isTruncated, nextKeyMarker } = await multipartRepository.listByBucket(
|
||||
bucketRecord.id,
|
||||
maxUploads,
|
||||
keyMarker,
|
||||
@@ -1774,7 +1760,7 @@ const handleAbortMultipartUpload = async (
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const uploadId = searchParams.get('uploadId')!;
|
||||
const multipart = await findMultipartUpload(uploadId);
|
||||
const multipart = await multipartRepository.findById(uploadId);
|
||||
if (!multipart) {
|
||||
return s3ErrorResponse(
|
||||
'NoSuchUpload',
|
||||
@@ -1785,7 +1771,7 @@ const handleAbortMultipartUpload = async (
|
||||
);
|
||||
}
|
||||
|
||||
await abortMultipartUpload(uploadId);
|
||||
await multipartRepository.abort(uploadId);
|
||||
return s3Response(null, 204, reqId);
|
||||
};
|
||||
|
||||
@@ -1807,7 +1793,7 @@ const handleListParts = async (
|
||||
reqId: string,
|
||||
): Promise<Response> => {
|
||||
const uploadId = searchParams.get('uploadId')!;
|
||||
const multipart = await findMultipartUpload(uploadId);
|
||||
const multipart = await multipartRepository.findById(uploadId);
|
||||
if (!multipart) {
|
||||
return s3ErrorResponse(
|
||||
'NoSuchUpload',
|
||||
@@ -1818,7 +1804,7 @@ const handleListParts = async (
|
||||
);
|
||||
}
|
||||
|
||||
const parts = await listMultipartParts(uploadId);
|
||||
const parts = await multipartRepository.listParts(uploadId);
|
||||
const maxParts = Math.min(Number.parseInt(searchParams.get('max-parts') || '1000', 10), 1000);
|
||||
|
||||
const xml = listPartsXml(
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { createWriteStream } from 'node:fs';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { config } from '../../../config/index';
|
||||
import { findFileByHash } from '../../../db/files';
|
||||
import { chunkedStorage, fileRepository, uploadBatcher } from '../../../infrastructure/di';
|
||||
import type { PreparedUpload } from '../../../infrastructure/telegram/upload-batcher';
|
||||
import logger from '../../../shared/logger/index';
|
||||
import { metricsCollector } from '../../../shared/metrics/index';
|
||||
import {
|
||||
@@ -14,8 +15,6 @@ import {
|
||||
getErrorMessage,
|
||||
getFileType,
|
||||
} from '../../../shared/utils/file';
|
||||
import { storeFileInTelegramChunks } from '../../../utils/chunked-storage';
|
||||
import { enqueuePreparedUpload, type PreparedUpload } from '../../../utils/uploadBatcher';
|
||||
|
||||
/**
|
||||
* Maximum allowed size (in bytes) for a base64 JSON upload.
|
||||
@@ -223,7 +222,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
|
||||
|
||||
const prepared = await streamFileToTemp(file, config.maxRequestBodyBytes);
|
||||
|
||||
const existingFile = await findFileByHash(prepared.fileHash);
|
||||
const existingFile = await fileRepository.findByHash(prepared.fileHash);
|
||||
if (existingFile) {
|
||||
await cleanupTempFile(prepared.tempPath);
|
||||
return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 });
|
||||
@@ -243,7 +242,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
|
||||
}
|
||||
|
||||
if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
|
||||
const uploadedFile = await storeFileInTelegramChunks({
|
||||
const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
|
||||
tempPath: prepared.tempPath,
|
||||
partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'upload'}`,
|
||||
fileName: finalFileName,
|
||||
@@ -256,7 +255,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
|
||||
return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 });
|
||||
}
|
||||
|
||||
const uploaded = await enqueuePreparedUpload({
|
||||
const uploaded = await uploadBatcher.enqueuePreparedUpload({
|
||||
prepared,
|
||||
fileName: finalFileName,
|
||||
mimeType,
|
||||
@@ -317,7 +316,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
|
||||
const fileBytes = Buffer.from(base64Data, 'base64');
|
||||
const hash = computeHash(fileBytes);
|
||||
|
||||
const existingFile = await findFileByHash(hash);
|
||||
const existingFile = await fileRepository.findByHash(hash);
|
||||
if (existingFile) {
|
||||
return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 });
|
||||
}
|
||||
@@ -334,7 +333,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
|
||||
const prepared = await writeBufferToTemp(fileBytes, hash);
|
||||
|
||||
if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
|
||||
const uploadedFile = await storeFileInTelegramChunks({
|
||||
const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
|
||||
tempPath: prepared.tempPath,
|
||||
partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'json'}`,
|
||||
fileName: finalFileName,
|
||||
@@ -347,7 +346,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
|
||||
return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 });
|
||||
}
|
||||
|
||||
const uploaded = await enqueuePreparedUpload({
|
||||
const uploaded = await uploadBatcher.enqueuePreparedUpload({
|
||||
prepared,
|
||||
fileName: finalFileName,
|
||||
mimeType,
|
||||
|
||||
@@ -1,21 +1,11 @@
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { config } from '../../../config/index';
|
||||
import { createBucket, deleteBucket, findBucketByName, listBuckets } from '../../../db/buckets';
|
||||
import {
|
||||
countBucketObjects,
|
||||
findFileByBucketAndKey,
|
||||
listObjectsByPrefix,
|
||||
softDeleteFile,
|
||||
} from '../../../db/files-ext';
|
||||
import { db, files as fileSchema } from '../../../db/index';
|
||||
import { bucketRepository, chunkedStorage, fileRepository } from '../../../infrastructure/di';
|
||||
import { db, files as fileSchema } from '../../../infrastructure/persistence/drizzle/index';
|
||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||
import logger from '../../../shared/logger/index';
|
||||
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
|
||||
import {
|
||||
createChunkedObjectResponse,
|
||||
storeFileInTelegramChunks,
|
||||
} from '../../../utils/chunked-storage';
|
||||
|
||||
/**
|
||||
* Route parameters extracted from the URL path.
|
||||
@@ -48,13 +38,13 @@ const jsonError = (error: string, status: number): Response => Response.json({ e
|
||||
* @returns A JSON response with the bucket list.
|
||||
*/
|
||||
export const handleListBucketsV1 = async (): Promise<Response> => {
|
||||
const buckets = await listBuckets();
|
||||
const buckets = await bucketRepository.list();
|
||||
const result = await Promise.all(
|
||||
buckets.map(async (b) => ({
|
||||
id: b.id,
|
||||
name: b.name,
|
||||
createdAt: b.createdAt.toISOString(),
|
||||
objectCount: await countBucketObjects(b.id),
|
||||
objectCount: await fileRepository.countByBucket(b.id),
|
||||
})),
|
||||
);
|
||||
return json({ buckets: result });
|
||||
@@ -73,9 +63,9 @@ export const handleCreateBucketV1 = async (req: Request): Promise<Response> => {
|
||||
if (!body.name || !/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(body.name)) {
|
||||
return jsonError('Invalid bucket name. Use lowercase, 3-63 chars, no underscore', 400);
|
||||
}
|
||||
const existing = await findBucketByName(body.name);
|
||||
const existing = await bucketRepository.findByName(body.name);
|
||||
if (existing) return jsonError('Bucket already exists', 409);
|
||||
const bucket = await createBucket(body.name);
|
||||
const bucket = await bucketRepository.create(body.name);
|
||||
return json({ id: bucket.id, name: bucket.name }, 201);
|
||||
};
|
||||
|
||||
@@ -92,11 +82,11 @@ export const handleDeleteBucketV1 = async (
|
||||
_req: Request,
|
||||
params: RouteParams,
|
||||
): Promise<Response> => {
|
||||
const bucket = await findBucketByName(params.bucket!);
|
||||
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||
if (!bucket) return jsonError('Bucket not found', 404);
|
||||
const count = await countBucketObjects(bucket.id);
|
||||
const count = await fileRepository.countByBucket(bucket.id);
|
||||
if (count > 0) return jsonError('Bucket is not empty', 409);
|
||||
await deleteBucket(params.bucket!);
|
||||
await bucketRepository.delete(params.bucket!);
|
||||
return json({ success: true });
|
||||
};
|
||||
|
||||
@@ -110,7 +100,7 @@ export const handleDeleteBucketV1 = async (
|
||||
* @returns A JSON response with the object list.
|
||||
*/
|
||||
export const handleListObjectsV1 = async (req: Request, params: RouteParams): Promise<Response> => {
|
||||
const bucket = await findBucketByName(params.bucket!);
|
||||
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||
if (!bucket) return jsonError('Bucket not found', 404);
|
||||
|
||||
const url = new URL(req.url);
|
||||
@@ -119,7 +109,7 @@ export const handleListObjectsV1 = async (req: Request, params: RouteParams): Pr
|
||||
const maxKeys = Number.parseInt(url.searchParams.get('max-keys') || '1000', 10);
|
||||
const continuationToken = url.searchParams.get('continuation-token') || null;
|
||||
|
||||
const { objects, prefixes } = await listObjectsByPrefix(
|
||||
const { objects, prefixes } = await fileRepository.listByPrefix(
|
||||
bucket.id,
|
||||
prefix,
|
||||
delimiter,
|
||||
@@ -162,7 +152,7 @@ export const handleUploadObjectV1 = async (
|
||||
req: Request,
|
||||
params: RouteParams,
|
||||
): Promise<Response> => {
|
||||
const bucket = await findBucketByName(params.bucket!);
|
||||
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||
if (!bucket) return jsonError('Bucket not found', 404);
|
||||
|
||||
const formData = await req.formData();
|
||||
@@ -217,7 +207,7 @@ export const handleUploadObjectV1 = async (
|
||||
const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`;
|
||||
|
||||
if (sizeBytes > config.telegramChunkSizeBytes) {
|
||||
const uploadedFile = await storeFileInTelegramChunks({
|
||||
const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
|
||||
tempPath,
|
||||
partFileNamePrefix,
|
||||
fileName: finalFileName,
|
||||
@@ -287,9 +277,9 @@ export const handleDeleteObjectV1 = async (
|
||||
_req: Request,
|
||||
params: RouteParams,
|
||||
): Promise<Response> => {
|
||||
const bucket = await findBucketByName(params.bucket!);
|
||||
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||
if (!bucket) return jsonError('Bucket not found', 404);
|
||||
await softDeleteFile(bucket.id, params.key!);
|
||||
await fileRepository.softDelete(bucket.id, params.key!);
|
||||
return json({ success: true });
|
||||
};
|
||||
|
||||
@@ -307,15 +297,15 @@ export const handleDownloadObjectV1 = async (
|
||||
_req: Request,
|
||||
params: RouteParams,
|
||||
): Promise<Response> => {
|
||||
const bucket = await findBucketByName(params.bucket!);
|
||||
const bucket = await bucketRepository.findByName(params.bucket!);
|
||||
if (!bucket) return jsonError('Bucket not found', 404);
|
||||
|
||||
const file = await findFileByBucketAndKey(bucket.id, params.key!);
|
||||
const file = await fileRepository.findByBucketAndKey(bucket.id, params.key!);
|
||||
if (!file) return jsonError('Object not found', 404);
|
||||
|
||||
if (file.storageBackend === 'chunked') {
|
||||
const range = { type: 'none' as const };
|
||||
return createChunkedObjectResponse({ file, range, reqId: '' });
|
||||
return chunkedStorage.createChunkedObjectResponse({ file, range, reqId: '' });
|
||||
}
|
||||
|
||||
const fileInfo = await botPool.getFileInfo(file.telegramFileId);
|
||||
@@ -348,12 +338,12 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro
|
||||
}
|
||||
|
||||
const destBucketName = body.destBucket || params.bucket!;
|
||||
const sourceBucket = await findBucketByName(params.bucket!);
|
||||
const destBucket = await findBucketByName(destBucketName);
|
||||
const sourceBucket = await bucketRepository.findByName(params.bucket!);
|
||||
const destBucket = await bucketRepository.findByName(destBucketName);
|
||||
|
||||
if (!sourceBucket || !destBucket) return jsonError('Bucket not found', 404);
|
||||
|
||||
const sourceFile = await findFileByBucketAndKey(sourceBucket.id, body.sourceKey);
|
||||
const sourceFile = await fileRepository.findByBucketAndKey(sourceBucket.id, body.sourceKey);
|
||||
if (!sourceFile) return jsonError('Source object not found', 404);
|
||||
|
||||
if (sourceFile.storageBackend === 'chunked') {
|
||||
|
||||
@@ -1,7 +1,89 @@
|
||||
export {
|
||||
checkRateLimit,
|
||||
cleanupRateLimitCache,
|
||||
clearRateLimitCache,
|
||||
getRateLimitStats,
|
||||
withRateLimit,
|
||||
} from '../../../utils/rateLimit';
|
||||
import { config } from '../../../config/index';
|
||||
import logger from '../../../shared/logger/index';
|
||||
import { extractClientIp } from '../../../shared/utils/ip';
|
||||
|
||||
interface RateLimitEntry {
|
||||
count: number;
|
||||
resetTime: number;
|
||||
}
|
||||
|
||||
const rateLimitStore = new Map<string, RateLimitEntry>();
|
||||
const MAX_STORE_ENTRIES = 50000;
|
||||
|
||||
const evictExpiredEntries = (now = Date.now()): number => {
|
||||
let cleaned = 0;
|
||||
|
||||
for (const [key, entry] of rateLimitStore.entries()) {
|
||||
if (now > entry.resetTime) {
|
||||
rateLimitStore.delete(key);
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
};
|
||||
|
||||
const ensureStoreCapacity = (now: number): void => {
|
||||
if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
|
||||
|
||||
evictExpiredEntries(now);
|
||||
while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
|
||||
const oldestKey = rateLimitStore.keys().next().value;
|
||||
if (!oldestKey) break;
|
||||
rateLimitStore.delete(oldestKey);
|
||||
}
|
||||
};
|
||||
|
||||
export const checkRateLimit = (key: string): boolean => {
|
||||
const now = Date.now();
|
||||
const entry = rateLimitStore.get(key);
|
||||
|
||||
if (!entry || now > entry.resetTime) {
|
||||
ensureStoreCapacity(now);
|
||||
rateLimitStore.set(key, {
|
||||
count: 1,
|
||||
resetTime: now + config.rateLimitWindowMs,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
if (entry.count >= config.rateLimitMaxRequests) {
|
||||
logger.warn('Rate limit exceeded', { key, count: entry.count });
|
||||
return false;
|
||||
}
|
||||
|
||||
entry.count++;
|
||||
return true;
|
||||
};
|
||||
|
||||
export const withRateLimit = <T extends Request>(
|
||||
handler: (req: T) => Promise<Response>,
|
||||
): ((req: T) => Promise<Response>) => {
|
||||
return async (req: T): Promise<Response> => {
|
||||
const ip = extractClientIp(req);
|
||||
if (!checkRateLimit(ip)) {
|
||||
return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
|
||||
}
|
||||
|
||||
return handler(req);
|
||||
};
|
||||
};
|
||||
|
||||
export const cleanupRateLimitCache = (): void => {
|
||||
const cleaned = evictExpiredEntries();
|
||||
|
||||
if (cleaned > 0) {
|
||||
logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
|
||||
}
|
||||
};
|
||||
|
||||
export const getRateLimitStats = () => ({
|
||||
trackedIPs: rateLimitStore.size,
|
||||
windowSize: config.rateLimitWindowMs,
|
||||
maxRequests: config.rateLimitMaxRequests,
|
||||
maxTrackedIPs: MAX_STORE_ENTRIES,
|
||||
});
|
||||
|
||||
export const clearRateLimitCache = (): void => {
|
||||
rateLimitStore.clear();
|
||||
};
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { config } from '../../../config/index';
|
||||
import { handleSwaggerHtml, handleSwaggerJson } from '../../../routes/swagger';
|
||||
import { extractS3BucketFromHost } from '../../../utils/s3/virtual-host';
|
||||
import { isS3Request } from '../../s3/auth';
|
||||
import { extractS3BucketFromHost } from '../../s3/virtual-host';
|
||||
import { handleLogin, handleLogout, handleMe } from '../controllers/auth-controller';
|
||||
import { handleFileInfo, handleFileRedirect } from '../controllers/file-controller';
|
||||
import { handleHealth } from '../controllers/health-controller';
|
||||
|
||||
+463
-7
@@ -1,7 +1,463 @@
|
||||
export type { SigV4Result, VerifyPresignedUrlInput } from '../../utils/s3/auth';
|
||||
export {
|
||||
buildCanonicalQueryString,
|
||||
isS3Request,
|
||||
verifyPresignedUrl,
|
||||
verifySignature,
|
||||
} from '../../utils/s3/auth';
|
||||
import { timingSafeEqual } from 'node:crypto';
|
||||
|
||||
/**
|
||||
* Timing-safe string comparison that prevents timing attacks.
|
||||
*
|
||||
* Uses `crypto.timingSafeEqual` which runs in constant time regardless of
|
||||
* where the strings differ. Returns false for mismatched-length inputs
|
||||
* to avoid leaking length information via early return.
|
||||
*
|
||||
* @param left - The first string to compare.
|
||||
* @param right - The second string to compare.
|
||||
* @returns True if both strings are equal.
|
||||
*/
|
||||
const timingSafeCompare = (left: string, right: string): boolean => {
|
||||
const leftBuffer = Buffer.from(left);
|
||||
const rightBuffer = Buffer.from(right);
|
||||
|
||||
if (leftBuffer.length !== rightBuffer.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return timingSafeEqual(leftBuffer, rightBuffer);
|
||||
};
|
||||
|
||||
export interface SigV4Result {
|
||||
isValid: boolean;
|
||||
credential: {
|
||||
accessKey: string;
|
||||
date: string;
|
||||
region: string;
|
||||
service: string;
|
||||
} | null;
|
||||
errorCode?: string;
|
||||
}
|
||||
|
||||
export interface VerifyPresignedUrlInput {
|
||||
url: string;
|
||||
method: string;
|
||||
headers: Record<string, string>;
|
||||
s3AccessKey: string;
|
||||
s3SecretKey: string;
|
||||
region: string;
|
||||
now?: Date;
|
||||
}
|
||||
|
||||
const SERVICE = 's3';
|
||||
const TERMINATION = 'aws4_request';
|
||||
|
||||
/**
|
||||
* Maximum acceptable clock skew between client and server for header-based
|
||||
* SigV4 authentication. AWS allows 15 minutes.
|
||||
*/
|
||||
const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => {
|
||||
if (data instanceof Uint8Array) return data;
|
||||
if (data instanceof ArrayBuffer) return new Uint8Array(data);
|
||||
return new TextEncoder().encode(data);
|
||||
};
|
||||
|
||||
const sha256Hex = async (data: string | Uint8Array | ArrayBuffer): Promise<string> => {
|
||||
const hashBuffer = await crypto.subtle.digest('SHA-256', buf(data) as never);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
|
||||
};
|
||||
|
||||
const hmacSha256 = async (key: Uint8Array, message: string): Promise<Uint8Array> => {
|
||||
const cryptoKey = await crypto.subtle.importKey(
|
||||
'raw',
|
||||
key as never,
|
||||
{ name: 'HMAC', hash: 'SHA-256' },
|
||||
false,
|
||||
['sign'],
|
||||
);
|
||||
const result = await crypto.subtle.sign('HMAC', cryptoKey, buf(message) as never);
|
||||
return new Uint8Array(result);
|
||||
};
|
||||
|
||||
const getSigningKey = async (
|
||||
secretKey: string,
|
||||
dateStamp: string,
|
||||
region: string,
|
||||
): Promise<Uint8Array> => {
|
||||
let key = await hmacSha256(buf(`AWS4${secretKey}`), dateStamp);
|
||||
key = await hmacSha256(key, region);
|
||||
key = await hmacSha256(key, SERVICE);
|
||||
return await hmacSha256(key, TERMINATION);
|
||||
};
|
||||
|
||||
const hmacHex = async (key: Uint8Array, message: string): Promise<string> => {
|
||||
const result = await hmacSha256(key, message);
|
||||
return Array.from(result)
|
||||
.map((b) => b.toString(16).padStart(2, '0'))
|
||||
.join('');
|
||||
};
|
||||
|
||||
const parseAuthorizationHeader = (authHeader: string) => {
|
||||
const credentialMatch = authHeader.match(/Credential=([^,]+)/);
|
||||
const signedHeadersMatch = authHeader.match(/SignedHeaders=([^,]+)/);
|
||||
const signatureMatch = authHeader.match(/Signature=([^,]+)/);
|
||||
|
||||
if (!credentialMatch || !signedHeadersMatch || !signatureMatch) return null;
|
||||
|
||||
const credentialParts = credentialMatch[1].split('/');
|
||||
if (credentialParts.length !== 5) return null;
|
||||
|
||||
return {
|
||||
accessKey: credentialParts[0],
|
||||
date: credentialParts[1],
|
||||
region: credentialParts[2],
|
||||
service: credentialParts[3],
|
||||
termination: credentialParts[4],
|
||||
signedHeaders: signedHeadersMatch[1],
|
||||
signature: signatureMatch[1],
|
||||
};
|
||||
};
|
||||
|
||||
const buildCanonicalRequest = (
|
||||
method: string,
|
||||
canonicalUri: string,
|
||||
canonicalQueryString: string,
|
||||
signedHeaders: string,
|
||||
headers: Record<string, string>,
|
||||
hashedPayload: string,
|
||||
): string => {
|
||||
const canonicalHeaders = signedHeaders
|
||||
.split(';')
|
||||
.map((h) => {
|
||||
const value = headers[h.toLowerCase()] || '';
|
||||
return `${h.toLowerCase()}:${value.trim()}\n`;
|
||||
})
|
||||
.join('');
|
||||
|
||||
return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Normalizes a URI per AWS SigV4 requirements plus RFC 3986:
|
||||
*
|
||||
* 1. Decode percent-encoded characters
|
||||
* 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4
|
||||
*
|
||||
* @param uri - The raw URI path to normalize.
|
||||
* @returns The normalized URI path.
|
||||
*/
|
||||
const normalizeUri = (uri: string): string => {
|
||||
if (!uri || uri === '') return '/';
|
||||
|
||||
// AWS SigV4 requires URI-decoded paths in the canonical request
|
||||
// Only `.` and `..` segments are removed per RFC 3986 section 5.2.4
|
||||
// Empty segments (from `//` or trailing `/`) are preserved — they are
|
||||
// part of the URI and the SDK signs them.
|
||||
const decoded = decodeURIComponent(uri);
|
||||
const segments = decoded.split('/');
|
||||
const result: string[] = [];
|
||||
|
||||
for (const segment of segments) {
|
||||
if (segment === '.') continue;
|
||||
if (segment === '..') {
|
||||
result.pop();
|
||||
continue;
|
||||
}
|
||||
result.push(segment);
|
||||
}
|
||||
|
||||
// Join preserves empty first segment (from leading /) automatically
|
||||
return result.join('/') || '/';
|
||||
};
|
||||
|
||||
const awsEncode = (value: string): string =>
|
||||
encodeURIComponent(value).replace(
|
||||
/[!'()*]/g,
|
||||
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
|
||||
);
|
||||
|
||||
export const buildCanonicalQueryString = (
|
||||
searchParams: URLSearchParams,
|
||||
excludeKeys: Set<string> = new Set(),
|
||||
): string => {
|
||||
const pairs: Array<[string, string]> = [];
|
||||
for (const [key, value] of searchParams.entries()) {
|
||||
if (!excludeKeys.has(key)) pairs.push([key, value]);
|
||||
}
|
||||
// AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare
|
||||
pairs.sort(([ak, av], [bk, bv]) => {
|
||||
const a = `${awsEncode(ak)}=${awsEncode(av)}`;
|
||||
const b = `${awsEncode(bk)}=${awsEncode(bv)}`;
|
||||
if (a < b) return -1;
|
||||
if (a > b) return 1;
|
||||
return 0;
|
||||
});
|
||||
return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&');
|
||||
};
|
||||
|
||||
const getHashedPayload = async (body: string | null): Promise<string> => {
|
||||
if (!body || body.length === 0) return await sha256Hex('');
|
||||
return await sha256Hex(body);
|
||||
};
|
||||
|
||||
/**
|
||||
* Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date.
|
||||
*
|
||||
* @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format.
|
||||
* @returns The parsed Date, or null if the format is invalid.
|
||||
*/
|
||||
const parseAmzDateUtc = (amzDate: string): Date | null => {
|
||||
const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/);
|
||||
if (!match) return null;
|
||||
const [, year, month, day, hour, minute, second] = match;
|
||||
return new Date(
|
||||
Date.UTC(
|
||||
Number.parseInt(year, 10),
|
||||
Number.parseInt(month, 10) - 1,
|
||||
Number.parseInt(day, 10),
|
||||
Number.parseInt(hour, 10),
|
||||
Number.parseInt(minute, 10),
|
||||
Number.parseInt(second, 10),
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Validates that `host` is included in the signed headers list.
|
||||
*
|
||||
* AWS SigV4 mandates that `host` is always signed. Reject requests that
|
||||
* omit it to prevent header injection / replay variants.
|
||||
*
|
||||
* @param signedHeaders - The semicolon-separated signed headers string.
|
||||
* @returns True if `host` is present.
|
||||
*/
|
||||
const validateSignedHeaders = (signedHeaders: string): boolean => {
|
||||
return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host');
|
||||
};
|
||||
|
||||
export const verifySignature = async (
|
||||
method: string,
|
||||
url: string,
|
||||
headers: Record<string, string>,
|
||||
body: string | null,
|
||||
s3AccessKey: string,
|
||||
s3SecretKey: string,
|
||||
region: string,
|
||||
): Promise<SigV4Result> => {
|
||||
const authHeader = headers.authorization;
|
||||
if (!authHeader?.startsWith('AWS4-HMAC-SHA256')) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const parsed = parseAuthorizationHeader(authHeader);
|
||||
if (!parsed) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
if (!timingSafeCompare(parsed.accessKey, s3AccessKey)) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
if (!timingSafeCompare(parsed.region, region)) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
// Validate service and termination in credential scope (M2)
|
||||
if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
// Validate host is in signed headers (LOW/host)
|
||||
if (!validateSignedHeaders(parsed.signedHeaders)) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const parsedUrl = new URL(url, 'http://localhost');
|
||||
const canonicalUri = normalizeUri(parsedUrl.pathname);
|
||||
const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams);
|
||||
|
||||
const contentSha256 = headers['x-amz-content-sha256'] || null;
|
||||
if (contentSha256?.startsWith('STREAMING-')) {
|
||||
return { isValid: false, credential: null, errorCode: 'NotImplemented' };
|
||||
}
|
||||
|
||||
// CRITICAL: Use the x-amz-content-sha256 header value in the canonical
|
||||
// request because that's what the client signed. The actual body hash is
|
||||
// verified by verifyBodyHash() after streaming, ensuring integrity without
|
||||
// breaking SigV4.
|
||||
const hashedPayload = contentSha256 || (await getHashedPayload(body));
|
||||
|
||||
const canonicalRequest = buildCanonicalRequest(
|
||||
method,
|
||||
canonicalUri,
|
||||
canonicalQueryString,
|
||||
parsed.signedHeaders,
|
||||
headers,
|
||||
hashedPayload,
|
||||
);
|
||||
|
||||
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
|
||||
|
||||
// M1: Fall back to Date header if x-amz-date is missing
|
||||
const amzDate = headers['x-amz-date'] || headers.date || '';
|
||||
|
||||
// H5: Validate request freshness (clock skew / replay protection)
|
||||
if (amzDate) {
|
||||
const requestDate = parseAmzDateUtc(amzDate);
|
||||
if (requestDate) {
|
||||
const now = Date.now();
|
||||
const skew = Math.abs(now - requestDate.getTime());
|
||||
if (skew > MAX_CLOCK_SKEW_MS) {
|
||||
return { isValid: false, credential: null, errorCode: 'RequestExpired' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const dateStamp = parsed.date;
|
||||
|
||||
// M3: Ensure date in credential scope matches x-amz-date
|
||||
if (amzDate) {
|
||||
const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD"
|
||||
if (amzDateStamp !== dateStamp) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
}
|
||||
|
||||
const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`;
|
||||
|
||||
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
|
||||
|
||||
const signingKey = await getSigningKey(s3SecretKey, dateStamp, region);
|
||||
const expectedSignature = await hmacHex(signingKey, stringToSign);
|
||||
|
||||
if (!timingSafeCompare(expectedSignature, parsed.signature)) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
return {
|
||||
isValid: true,
|
||||
credential: {
|
||||
accessKey: parsed.accessKey,
|
||||
date: parsed.date,
|
||||
region: parsed.region,
|
||||
service: parsed.service,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const verifyPresignedUrl = async ({
|
||||
url,
|
||||
method,
|
||||
headers,
|
||||
s3AccessKey,
|
||||
s3SecretKey,
|
||||
region,
|
||||
now = new Date(),
|
||||
}: VerifyPresignedUrlInput): Promise<SigV4Result> => {
|
||||
const parsedUrl = new URL(url);
|
||||
const searchParams = parsedUrl.searchParams;
|
||||
|
||||
const algorithm = searchParams.get('X-Amz-Algorithm');
|
||||
const credential = searchParams.get('X-Amz-Credential');
|
||||
const signedHeaders = searchParams.get('X-Amz-SignedHeaders');
|
||||
const signature = searchParams.get('X-Amz-Signature');
|
||||
const expiresText = searchParams.get('X-Amz-Expires');
|
||||
const amzDate = searchParams.get('X-Amz-Date');
|
||||
|
||||
if (
|
||||
algorithm !== 'AWS4-HMAC-SHA256' ||
|
||||
!credential ||
|
||||
!signedHeaders ||
|
||||
!signature ||
|
||||
!expiresText ||
|
||||
!amzDate
|
||||
) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const expires = Number.parseInt(expiresText, 10);
|
||||
const signedAt = parseAmzDateUtc(amzDate);
|
||||
if (!Number.isFinite(expires) || expires <= 0 || !signedAt) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
// AWS S3 spec limits presigned URLs to 7 days (604800 seconds)
|
||||
const MAX_PRESIGNED_EXPIRY_SECONDS = 604800;
|
||||
if (
|
||||
now.getTime() > signedAt.getTime() + expires * 1000 ||
|
||||
expires > MAX_PRESIGNED_EXPIRY_SECONDS
|
||||
) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const credParts = credential.split('/');
|
||||
if (credParts.length !== 5) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
const [accessKey, dateStamp, credentialRegion, service, termination] = credParts;
|
||||
if (
|
||||
!timingSafeCompare(accessKey, s3AccessKey) ||
|
||||
!timingSafeCompare(credentialRegion, region) ||
|
||||
service !== SERVICE ||
|
||||
termination !== TERMINATION
|
||||
) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
// Validate host is in signed headers for presigned URLs too
|
||||
if (!validateSignedHeaders(signedHeaders)) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const signedHeaderList = signedHeaders.split(';').filter(Boolean);
|
||||
const canonicalHeaders = signedHeaderList
|
||||
.map((headerName) => {
|
||||
const lower = headerName.toLowerCase();
|
||||
const value = lower === 'host' ? headers.host || parsedUrl.host : headers[lower] || '';
|
||||
return `${lower}:${value.trim()}\n`;
|
||||
})
|
||||
.join('');
|
||||
|
||||
const canonicalRequest = `${method}\n${normalizeUri(parsedUrl.pathname)}\n${buildCanonicalQueryString(searchParams, new Set(['X-Amz-Signature']))}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`;
|
||||
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
|
||||
const credentialScope = `${dateStamp}/${region}/${SERVICE}/${TERMINATION}`;
|
||||
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
|
||||
const expectedSignature = await hmacHex(
|
||||
await getSigningKey(s3SecretKey, dateStamp, region),
|
||||
stringToSign,
|
||||
);
|
||||
|
||||
if (!timingSafeCompare(expectedSignature, signature)) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
return { isValid: true, credential: { accessKey, date: dateStamp, region, service } };
|
||||
};
|
||||
|
||||
export const isS3Request = (headers: Record<string, string>): boolean => {
|
||||
const auth = headers.authorization || '';
|
||||
return auth.startsWith('AWS4-HMAC-SHA256');
|
||||
};
|
||||
|
||||
/**
|
||||
* Verifies that the actual body SHA-256 matches the `x-amz-content-sha256`
|
||||
* header from the original request.
|
||||
*
|
||||
* This MUST be called AFTER the body has been fully streamed and hashed,
|
||||
* as a second pass after `verifySignature` (which cannot hash a streaming
|
||||
* body without consuming it).
|
||||
*
|
||||
* @param bodySha256 - The SHA-256 hex digest of the actual body content.
|
||||
* @param headers - The original request headers.
|
||||
* @returns An error result on mismatch, or null if the check passes.
|
||||
*/
|
||||
export const verifyBodyHash = (
|
||||
bodySha256: string,
|
||||
headers: Record<string, string>,
|
||||
): SigV4Result | null => {
|
||||
const claimedHash = headers['x-amz-content-sha256'];
|
||||
// If the client sent UNSIGNED-PAYLOAD, skip verification
|
||||
if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) {
|
||||
return null;
|
||||
}
|
||||
if (claimedHash !== bodySha256) {
|
||||
return { isValid: false, credential: null, errorCode: 'BadDigest' };
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
@@ -1,10 +1,52 @@
|
||||
/**
|
||||
* Re-export from the canonical headers implementation.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
export {
|
||||
applyS3Headers,
|
||||
S3_CORS_HEADERS,
|
||||
s3Headers,
|
||||
} from '../../utils/s3/headers';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
export const S3_CORS_HEADERS: Record<string, string> = {
|
||||
'access-control-allow-origin': '*',
|
||||
'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS',
|
||||
'access-control-allow-headers': [
|
||||
'Authorization',
|
||||
'Content-Type',
|
||||
'Content-MD5',
|
||||
'Range',
|
||||
'If-Match',
|
||||
'If-None-Match',
|
||||
'If-Modified-Since',
|
||||
'If-Unmodified-Since',
|
||||
'X-Amz-*',
|
||||
'x-amz-*',
|
||||
].join(', '),
|
||||
'access-control-expose-headers': [
|
||||
'Accept-Ranges',
|
||||
'Content-Length',
|
||||
'Content-Range',
|
||||
'Content-Type',
|
||||
'ETag',
|
||||
'Last-Modified',
|
||||
'x-amz-id-2',
|
||||
'x-amz-request-id',
|
||||
].join(', '),
|
||||
'access-control-max-age': '86400',
|
||||
};
|
||||
|
||||
export const s3Headers = (
|
||||
requestId: string,
|
||||
extraHeaders: Record<string, string> = {},
|
||||
): Record<string, string> => ({
|
||||
...S3_CORS_HEADERS,
|
||||
server: 'AmazonS3',
|
||||
...(requestId
|
||||
? {
|
||||
'x-amz-request-id': requestId,
|
||||
'x-amz-id-2': `${requestId}+${nanoid(16)}`,
|
||||
}
|
||||
: {}),
|
||||
...extraHeaders,
|
||||
});
|
||||
|
||||
export const applyS3Headers = (headers: Headers, requestId: string): Headers => {
|
||||
const result = new Headers(headers);
|
||||
for (const [key, value] of Object.entries(s3Headers(requestId))) {
|
||||
result.set(key, value);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
@@ -1 +1,27 @@
|
||||
export { extractS3BucketFromHost } from '../../utils/s3/virtual-host';
|
||||
const stripPort = (host: string): string => {
|
||||
// Handle IPv6: [::1]:8080 -> [::1]
|
||||
if (host.startsWith('[')) {
|
||||
const closeBracket = host.indexOf(']');
|
||||
return host.slice(0, closeBracket + 1).toLowerCase();
|
||||
}
|
||||
return host.split(':')[0].toLowerCase().replace(/\.$/, '');
|
||||
};
|
||||
|
||||
const isValidBucketLabel = (bucket: string): boolean =>
|
||||
/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) &&
|
||||
!bucket.includes('..') &&
|
||||
!bucket.includes('.-') &&
|
||||
!bucket.includes('-.');
|
||||
|
||||
export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => {
|
||||
const normalizedHost = stripPort(host);
|
||||
for (const domain of domains) {
|
||||
const normalizedDomain = stripPort(domain);
|
||||
if (!normalizedDomain || normalizedHost === normalizedDomain) continue;
|
||||
if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue;
|
||||
|
||||
const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1));
|
||||
return isValidBucketLabel(bucket) ? bucket : null;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
+311
-22
@@ -1,22 +1,311 @@
|
||||
/**
|
||||
* Re-export from the canonical XML implementation.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
export {
|
||||
bucketVersioningConfigurationXml,
|
||||
type CompletePart,
|
||||
completeMultipartUploadXml,
|
||||
copyObjectResultXml,
|
||||
deleteResultXml,
|
||||
initiateMultipartUploadXml,
|
||||
listBucketResultXml,
|
||||
listBucketsXml,
|
||||
listBucketV2ResultXml,
|
||||
listMultipartUploadsXml,
|
||||
listPartsXml,
|
||||
parseCompleteMultipartBody,
|
||||
parseDeleteObjectsBody,
|
||||
s3ErrorResponse,
|
||||
s3ErrorXml,
|
||||
} from '../../utils/s3/xml';
|
||||
import { s3Headers } from './headers';
|
||||
|
||||
const escapeXml = (str: string): string =>
|
||||
str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
|
||||
const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||
|
||||
const encodeKey = (value: string, encodingType: string | null = null): string =>
|
||||
encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value);
|
||||
|
||||
// ─────── Bucket operations ───────
|
||||
|
||||
export const listBucketsXml = (
|
||||
buckets: { name: string; createdAt: Date }[],
|
||||
_requestId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Buckets>
|
||||
${buckets
|
||||
.map(
|
||||
(b) => `<Bucket>
|
||||
<Name>${escapeXml(b.name)}</Name>
|
||||
<CreationDate>${isoDate(b.createdAt)}</CreationDate>
|
||||
</Bucket>`,
|
||||
)
|
||||
.join('')}
|
||||
</Buckets>
|
||||
</ListAllMyBucketsResult>`;
|
||||
|
||||
export const bucketVersioningConfigurationXml =
|
||||
(): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`;
|
||||
|
||||
// ─────── Object listing ───────
|
||||
|
||||
export const listBucketResultXml = (
|
||||
bucketName: string,
|
||||
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
|
||||
prefixes: string[],
|
||||
isTruncated: boolean,
|
||||
marker: string | null,
|
||||
maxKeys: number,
|
||||
prefix: string,
|
||||
delimiter: string | null,
|
||||
nextMarker: string | null,
|
||||
_requestId: string,
|
||||
encodingType: string | null = null,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>${escapeXml(bucketName)}</Name>
|
||||
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
|
||||
<Marker>${encodeKey(marker || '', encodingType)}</Marker>
|
||||
<MaxKeys>${maxKeys}</MaxKeys>
|
||||
<Delimiter>${encodeKey(delimiter || '', encodingType)}</Delimiter>
|
||||
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
|
||||
<IsTruncated>${isTruncated}</IsTruncated>
|
||||
${objects
|
||||
.map(
|
||||
(o) => `<Contents>
|
||||
<Key>${encodeKey(o.key, encodingType)}</Key>
|
||||
<LastModified>${isoDate(o.lastModified)}</LastModified>
|
||||
<ETag>"${o.etag}"</ETag>
|
||||
<Size>${o.sizeBytes}</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>`,
|
||||
)
|
||||
.join('')}
|
||||
${prefixes
|
||||
.map(
|
||||
(p) => `<CommonPrefixes>
|
||||
<Prefix>${encodeKey(p, encodingType)}</Prefix>
|
||||
</CommonPrefixes>`,
|
||||
)
|
||||
.join('')}
|
||||
${nextMarker ? `<NextMarker>${encodeKey(nextMarker, encodingType)}</NextMarker>` : ''}
|
||||
</ListBucketResult>`;
|
||||
|
||||
export const listBucketV2ResultXml = (
|
||||
bucketName: string,
|
||||
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
|
||||
prefixes: string[],
|
||||
isTruncated: boolean,
|
||||
maxKeys: number,
|
||||
prefix: string,
|
||||
delimiter: string | null,
|
||||
continuationToken: string | null,
|
||||
nextContinuationToken: string | null,
|
||||
keyCount: number,
|
||||
_requestId: string,
|
||||
encodingType: string | null = null,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>${escapeXml(bucketName)}</Name>
|
||||
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
|
||||
<MaxKeys>${maxKeys}</MaxKeys>
|
||||
<KeyCount>${keyCount}</KeyCount>
|
||||
${delimiter ? `<Delimiter>${encodeKey(delimiter, encodingType)}</Delimiter>` : ''}
|
||||
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
|
||||
${continuationToken ? `<ContinuationToken>${encodeKey(continuationToken, encodingType)}</ContinuationToken>` : ''}
|
||||
<IsTruncated>${isTruncated}</IsTruncated>
|
||||
${objects
|
||||
.map(
|
||||
(o) => `<Contents>
|
||||
<Key>${encodeKey(o.key, encodingType)}</Key>
|
||||
<LastModified>${isoDate(o.lastModified)}</LastModified>
|
||||
<ETag>"${o.etag}"</ETag>
|
||||
<Size>${o.sizeBytes}</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>`,
|
||||
)
|
||||
.join('')}
|
||||
${prefixes
|
||||
.map(
|
||||
(p) => `<CommonPrefixes>
|
||||
<Prefix>${encodeKey(p, encodingType)}</Prefix>
|
||||
</CommonPrefixes>`,
|
||||
)
|
||||
.join('')}
|
||||
${nextContinuationToken ? `<NextContinuationToken>${encodeKey(nextContinuationToken, encodingType)}</NextContinuationToken>` : ''}
|
||||
</ListBucketResultV2>`;
|
||||
|
||||
// ─────── Multipart ───────
|
||||
|
||||
export const initiateMultipartUploadXml = (
|
||||
bucketName: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||
<Key>${escapeXml(key)}</Key>
|
||||
<UploadId>${uploadId}</UploadId>
|
||||
</InitiateMultipartUploadResult>`;
|
||||
|
||||
export const listPartsXml = (
|
||||
bucketName: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[],
|
||||
maxParts: number,
|
||||
isTruncated: boolean,
|
||||
_requestId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||
<Key>${escapeXml(key)}</Key>
|
||||
<UploadId>${uploadId}</UploadId>
|
||||
<MaxParts>${maxParts}</MaxParts>
|
||||
<IsTruncated>${isTruncated}</IsTruncated>
|
||||
${parts
|
||||
.map(
|
||||
(p) => `<Part>
|
||||
<PartNumber>${p.partNumber}</PartNumber>
|
||||
<LastModified>${isoDate(p.createdAt)}</LastModified>
|
||||
<ETag>"${p.etag}"</ETag>
|
||||
<Size>${p.sizeBytes}</Size>
|
||||
</Part>`,
|
||||
)
|
||||
.join('')}
|
||||
</ListPartsResult>`;
|
||||
|
||||
export const listMultipartUploadsXml = (
|
||||
bucketName: string,
|
||||
uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[],
|
||||
maxUploads: number,
|
||||
isTruncated: boolean,
|
||||
nextKeyMarker: string | null,
|
||||
_requestId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||
<KeyMarker></KeyMarker>
|
||||
<UploadIdMarker></UploadIdMarker>
|
||||
${nextKeyMarker ? `<NextKeyMarker>${escapeXml(nextKeyMarker)}</NextKeyMarker>` : ''}
|
||||
<MaxUploads>${maxUploads}</MaxUploads>
|
||||
<IsTruncated>${isTruncated}</IsTruncated>
|
||||
${uploads
|
||||
.map(
|
||||
(u) => `<Upload>
|
||||
<Key>${escapeXml(u.key)}</Key>
|
||||
<UploadId>${u.uploadId}</UploadId>
|
||||
<Initiator><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Initiator>
|
||||
<Owner><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Owner>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
<Initiated>${isoDate(u.initiatedAt)}</Initiated>
|
||||
</Upload>`,
|
||||
)
|
||||
.join('')}
|
||||
</ListMultipartUploadsResult>`;
|
||||
|
||||
export const completeMultipartUploadXml = (
|
||||
bucketName: string,
|
||||
key: string,
|
||||
etag: string,
|
||||
location: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Location>${escapeXml(location)}</Location>
|
||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||
<Key>${escapeXml(key)}</Key>
|
||||
<ETag>"${etag}"</ETag>
|
||||
</CompleteMultipartUploadResult>`;
|
||||
|
||||
// ─────── Delete result ───────
|
||||
|
||||
export const deleteResultXml = (
|
||||
deleted: string[],
|
||||
errors: { key: string; code: string; message: string }[],
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
${deleted
|
||||
.map(
|
||||
(key) => `<Deleted>
|
||||
<Key>${escapeXml(key)}</Key>
|
||||
</Deleted>`,
|
||||
)
|
||||
.join('')}
|
||||
${errors
|
||||
.map(
|
||||
(e) => `<Error>
|
||||
<Key>${escapeXml(e.key)}</Key>
|
||||
<Code>${e.code}</Code>
|
||||
<Message>${escapeXml(e.message)}</Message>
|
||||
</Error>`,
|
||||
)
|
||||
.join('')}
|
||||
</DeleteResult>`;
|
||||
|
||||
// ─────── Copy ───────
|
||||
|
||||
export const copyObjectResultXml = (
|
||||
etag: string,
|
||||
lastModified: Date,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<ETag>"${etag}"</ETag>
|
||||
<LastModified>${isoDate(lastModified)}</LastModified>
|
||||
</CopyObjectResult>`;
|
||||
|
||||
// ─────── Error ───────
|
||||
|
||||
export const s3ErrorXml = (
|
||||
code: string,
|
||||
message: string,
|
||||
resource: string,
|
||||
requestId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Error>
|
||||
<Code>${code}</Code>
|
||||
<Message>${escapeXml(message)}</Message>
|
||||
<Resource>${escapeXml(resource)}</Resource>
|
||||
<RequestId>${requestId}</RequestId>
|
||||
<HostId>${requestId}</HostId>
|
||||
</Error>`;
|
||||
|
||||
export const s3ErrorResponse = (
|
||||
code: string,
|
||||
message: string,
|
||||
resource: string,
|
||||
status: number,
|
||||
requestId: string = '',
|
||||
extraHeaders: Record<string, string> = {},
|
||||
): Response =>
|
||||
new Response(s3ErrorXml(code, message, resource, requestId), {
|
||||
status,
|
||||
headers: s3Headers(requestId, {
|
||||
'content-type': 'application/xml',
|
||||
...extraHeaders,
|
||||
}),
|
||||
});
|
||||
|
||||
// ─────── DeleteObjects XML parser ───────
|
||||
|
||||
export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => {
|
||||
// H9: Use non-greedy match to handle keys containing < character
|
||||
const keys = Array.from(body.matchAll(/<Key>([\s\S]*?)<\/Key>/g), (match) => match[1]);
|
||||
// Handle whitespace inside <Quiet> element + namespace prefix support
|
||||
const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body);
|
||||
return { keys, quiet };
|
||||
};
|
||||
|
||||
// ─────── CompleteMultipartUpload XML parser ───────
|
||||
|
||||
export interface CompletePart {
|
||||
partNumber: number;
|
||||
etag: string;
|
||||
}
|
||||
|
||||
export const parseCompleteMultipartBody = (body: string): CompletePart[] => {
|
||||
const parts: CompletePart[] = [];
|
||||
const partRegex = /<Part>[\s\S]*?<\/Part>/g;
|
||||
const partMatch = body.match(partRegex) || [];
|
||||
|
||||
for (const partXml of partMatch) {
|
||||
const numMatch = partXml.match(/<PartNumber>(\d+)<\/PartNumber>/);
|
||||
const etagMatch = partXml.match(/<ETag>"?([^"<\s]+)"?<\/ETag>/);
|
||||
if (numMatch && etagMatch) {
|
||||
parts.push({
|
||||
partNumber: Number.parseInt(numMatch[1], 10),
|
||||
etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return parts;
|
||||
};
|
||||
|
||||
@@ -1,4 +1,26 @@
|
||||
import _logger from '../../utils/logger';
|
||||
export default _logger;
|
||||
export type { Logger } from 'winston';
|
||||
export { _logger as logger };
|
||||
import winston from 'winston';
|
||||
|
||||
const logger = winston.createLogger({
|
||||
level: process.env.LOG_LEVEL || 'info',
|
||||
format: winston.format.combine(
|
||||
winston.format.timestamp(),
|
||||
winston.format.errors({ stack: true }),
|
||||
winston.format.json(),
|
||||
),
|
||||
defaultMeta: { service: 'filedrop' },
|
||||
transports: [
|
||||
// Write all logs including error logs to file
|
||||
new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
|
||||
new winston.transports.File({ filename: 'logs/combined.log' }),
|
||||
// Console transport for docker logs / CLI visibility
|
||||
new winston.transports.Console({
|
||||
format:
|
||||
process.env.NODE_ENV !== 'production'
|
||||
? winston.format.combine(winston.format.colorize(), winston.format.simple())
|
||||
: winston.format.json(),
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
export default logger;
|
||||
export { logger };
|
||||
|
||||
+111
-1
@@ -1 +1,111 @@
|
||||
export { MetricsCollector, metricsCollector } from '../../utils/metrics';
|
||||
// No imports needed — logger used only by setInterval which moved to index.ts
|
||||
|
||||
interface Metric {
|
||||
name: string;
|
||||
value: number;
|
||||
timestamp: number;
|
||||
tags?: Record<string, string>;
|
||||
}
|
||||
|
||||
interface MetricsSnapshot {
|
||||
uploadLatency: { p50: number; p95: number; p99: number };
|
||||
uploadThroughput: number;
|
||||
queueSize: number;
|
||||
errorRate: number;
|
||||
cacheHitRate: number;
|
||||
botUtilization: number;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
class MetricsCollector {
|
||||
private metrics: Metric[] = [];
|
||||
private uploadTimes: number[] = [];
|
||||
private errorCount = 0;
|
||||
private totalRequests = 0;
|
||||
private cacheHits = 0;
|
||||
private cacheMisses = 0;
|
||||
private maxMetricsSize = 10000;
|
||||
|
||||
recordUploadTime(durationMs: number): void {
|
||||
this.uploadTimes.push(durationMs);
|
||||
this.totalRequests++;
|
||||
|
||||
// Keep only last 1000 measurements
|
||||
if (this.uploadTimes.length > 1000) {
|
||||
this.uploadTimes.shift();
|
||||
}
|
||||
}
|
||||
|
||||
recordError(): void {
|
||||
this.errorCount++;
|
||||
}
|
||||
|
||||
recordCacheHit(): void {
|
||||
this.cacheHits++;
|
||||
}
|
||||
|
||||
recordCacheMiss(): void {
|
||||
this.cacheMisses++;
|
||||
}
|
||||
|
||||
recordMetric(name: string, value: number, tags?: Record<string, string>): void {
|
||||
this.metrics.push({
|
||||
name,
|
||||
value,
|
||||
timestamp: Date.now(),
|
||||
tags,
|
||||
});
|
||||
|
||||
// Keep metrics bounded
|
||||
if (this.metrics.length > this.maxMetricsSize) {
|
||||
this.metrics = this.metrics.slice(-this.maxMetricsSize);
|
||||
}
|
||||
}
|
||||
|
||||
private calculatePercentile(arr: number[], percentile: number): number {
|
||||
if (arr.length === 0) return 0;
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const index = Math.ceil((percentile / 100) * sorted.length) - 1;
|
||||
return sorted[Math.max(0, index)];
|
||||
}
|
||||
|
||||
getSnapshot(): MetricsSnapshot {
|
||||
const errorRate = this.totalRequests > 0 ? (this.errorCount / this.totalRequests) * 100 : 0;
|
||||
const cacheHitRate =
|
||||
this.cacheHits + this.cacheMisses > 0
|
||||
? (this.cacheHits / (this.cacheHits + this.cacheMisses)) * 100
|
||||
: 0;
|
||||
|
||||
return {
|
||||
uploadLatency: {
|
||||
p50: this.calculatePercentile(this.uploadTimes, 50),
|
||||
p95: this.calculatePercentile(this.uploadTimes, 95),
|
||||
p99: this.calculatePercentile(this.uploadTimes, 99),
|
||||
},
|
||||
uploadThroughput: this.totalRequests > 0 ? this.totalRequests / 60 : 0,
|
||||
queueSize: 0, // Will be updated by queue
|
||||
errorRate,
|
||||
cacheHitRate,
|
||||
botUtilization: 0, // Will be updated by bot tracker
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.uploadTimes = [];
|
||||
this.errorCount = 0;
|
||||
this.totalRequests = 0;
|
||||
this.cacheHits = 0;
|
||||
this.cacheMisses = 0;
|
||||
this.metrics = [];
|
||||
}
|
||||
|
||||
getMetrics(name?: string): Metric[] {
|
||||
if (!name) return this.metrics;
|
||||
return this.metrics.filter((m) => m.name === name);
|
||||
}
|
||||
}
|
||||
|
||||
export const metricsCollector = new MetricsCollector();
|
||||
|
||||
export { MetricsCollector };
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { unlink } from 'node:fs/promises';
|
||||
import logger from '../../utils/logger';
|
||||
import logger from '../logger/index';
|
||||
|
||||
/**
|
||||
* Safely extracts an error message from an unknown value.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import logger from '../../utils/logger';
|
||||
import logger from '../logger/index';
|
||||
|
||||
/** Configuration options for retry behaviour. */
|
||||
interface RetryOptions {
|
||||
|
||||
@@ -1,205 +0,0 @@
|
||||
import { createHmac, timingSafeEqual } from 'node:crypto';
|
||||
import { config } from '../env';
|
||||
|
||||
const ADMIN_USERNAME = 'admin';
|
||||
const SIGNATURE_SEPARATOR = '.';
|
||||
|
||||
type Handler = (req: Request) => Response | Promise<Response>;
|
||||
|
||||
export interface AuthSession {
|
||||
username: string;
|
||||
expiresAt: Date | null;
|
||||
method: 'cookie' | 'bearer';
|
||||
}
|
||||
|
||||
interface CookieOptions {
|
||||
secret?: string;
|
||||
cookieName?: string;
|
||||
maxAgeMs?: number;
|
||||
}
|
||||
|
||||
interface SessionPayload {
|
||||
u: string;
|
||||
e: number;
|
||||
}
|
||||
|
||||
const getSecret = (secret?: string): string => secret ?? config.adminApiToken;
|
||||
const getCookieName = (cookieName?: string): string => cookieName ?? config.sessionCookieName;
|
||||
const getMaxAgeMs = (maxAgeMs?: number): number => maxAgeMs ?? config.sessionMaxAgeMs;
|
||||
|
||||
const encodePayload = (value: string): string => Buffer.from(value, 'utf8').toString('base64url');
|
||||
|
||||
const decodePayload = (value: string): string | null => {
|
||||
try {
|
||||
return Buffer.from(value, 'base64url').toString('utf8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const isAuthEnabled = (secret = config.adminApiToken): boolean => secret.length > 0;
|
||||
|
||||
export const timingSafeCompare = (left: string, right: string): boolean => {
|
||||
const leftBuffer = Buffer.from(left);
|
||||
const rightBuffer = Buffer.from(right);
|
||||
|
||||
if (leftBuffer.length !== rightBuffer.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return timingSafeEqual(leftBuffer, rightBuffer);
|
||||
};
|
||||
|
||||
export const signCookiePayload = (payload: string, secret: string): string =>
|
||||
createHmac('sha256', secret).update(payload).digest('base64url');
|
||||
|
||||
export const verifyCookieSignature = (cookieValue: string, secret: string): string | null => {
|
||||
const separatorIndex = cookieValue.lastIndexOf(SIGNATURE_SEPARATOR);
|
||||
if (separatorIndex <= 0 || separatorIndex === cookieValue.length - 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const payload = cookieValue.slice(0, separatorIndex);
|
||||
const signature = cookieValue.slice(separatorIndex + 1);
|
||||
const expectedSignature = signCookiePayload(payload, secret);
|
||||
|
||||
if (!timingSafeCompare(signature, expectedSignature)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return payload;
|
||||
};
|
||||
|
||||
const cookieAttributes = (maxAgeSeconds: number): string =>
|
||||
[`Max-Age=${maxAgeSeconds}`, 'Path=/', 'HttpOnly', 'SameSite=Lax', 'Secure'].join('; ');
|
||||
|
||||
export const createSessionCookie = (
|
||||
username = ADMIN_USERNAME,
|
||||
options: CookieOptions = {},
|
||||
): string => {
|
||||
const secret = getSecret(options.secret);
|
||||
const cookieName = getCookieName(options.cookieName);
|
||||
const maxAgeMs = getMaxAgeMs(options.maxAgeMs);
|
||||
const expiresAt = Date.now() + maxAgeMs;
|
||||
const payload = encodePayload(
|
||||
JSON.stringify({ u: username, e: expiresAt } satisfies SessionPayload),
|
||||
);
|
||||
const signature = signCookiePayload(payload, secret);
|
||||
const maxAgeSeconds = Math.max(1, Math.floor(maxAgeMs / 1000));
|
||||
|
||||
return `${cookieName}=${payload}${SIGNATURE_SEPARATOR}${signature}; ${cookieAttributes(maxAgeSeconds)}`;
|
||||
};
|
||||
|
||||
export const clearSessionCookie = (cookieName = config.sessionCookieName): string =>
|
||||
`${cookieName}=; ${cookieAttributes(0)}`;
|
||||
|
||||
const findCookieValue = (cookieHeader: string | null, cookieName: string): string | null => {
|
||||
if (!cookieHeader) return null;
|
||||
|
||||
for (const rawCookie of cookieHeader.split(';')) {
|
||||
const cookie = rawCookie.trim();
|
||||
const equalsIndex = cookie.indexOf('=');
|
||||
if (equalsIndex <= 0) continue;
|
||||
|
||||
const name = cookie.slice(0, equalsIndex);
|
||||
if (name === cookieName) {
|
||||
return cookie.slice(equalsIndex + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
export const parseSessionFromCookie = (
|
||||
cookieHeader: string | null,
|
||||
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
|
||||
): AuthSession | null => {
|
||||
const secret = getSecret(options.secret);
|
||||
const cookieName = getCookieName(options.cookieName);
|
||||
if (!isAuthEnabled(secret)) return null;
|
||||
|
||||
const cookieValue = findCookieValue(cookieHeader, cookieName);
|
||||
if (!cookieValue) return null;
|
||||
|
||||
const encodedPayload = verifyCookieSignature(cookieValue, secret);
|
||||
if (!encodedPayload) return null;
|
||||
|
||||
const rawPayload = decodePayload(encodedPayload);
|
||||
if (!rawPayload) return null;
|
||||
|
||||
try {
|
||||
const payload = JSON.parse(rawPayload) as Partial<SessionPayload>;
|
||||
if (payload.u !== ADMIN_USERNAME || typeof payload.e !== 'number') return null;
|
||||
if (!Number.isFinite(payload.e) || payload.e <= Date.now()) return null;
|
||||
|
||||
return {
|
||||
username: payload.u,
|
||||
expiresAt: new Date(payload.e),
|
||||
method: 'cookie',
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const checkBearerToken = (
|
||||
authorizationHeader: string | null,
|
||||
secret = config.adminApiToken,
|
||||
): boolean => {
|
||||
if (!isAuthEnabled(secret) || !authorizationHeader) return false;
|
||||
|
||||
const [scheme, ...rest] = authorizationHeader.split(' ');
|
||||
if (scheme !== 'Bearer' || rest.length === 0) return false;
|
||||
|
||||
const token = rest.join(' ').trim();
|
||||
return token.length > 0 && timingSafeCompare(token, secret);
|
||||
};
|
||||
|
||||
export const getAuthSession = (
|
||||
req: Request,
|
||||
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
|
||||
): AuthSession | null => {
|
||||
const secret = getSecret(options.secret);
|
||||
if (!isAuthEnabled(secret)) {
|
||||
return {
|
||||
username: ADMIN_USERNAME,
|
||||
expiresAt: null,
|
||||
method: 'bearer',
|
||||
};
|
||||
}
|
||||
|
||||
const cookieSession = parseSessionFromCookie(req.headers.get('cookie'), options);
|
||||
if (cookieSession) return cookieSession;
|
||||
|
||||
if (checkBearerToken(req.headers.get('authorization'), secret)) {
|
||||
return {
|
||||
username: ADMIN_USERNAME,
|
||||
expiresAt: null,
|
||||
method: 'bearer',
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
export const unauthorizedResponse = (): Response =>
|
||||
Response.json({ error: 'Unauthorized' }, { status: 401 });
|
||||
|
||||
export const requireAuth = (
|
||||
handler: Handler,
|
||||
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
|
||||
): ((req: Request) => Promise<Response>) => {
|
||||
return async (req: Request): Promise<Response> => {
|
||||
const secret = getSecret(options.secret);
|
||||
if (!isAuthEnabled(secret)) {
|
||||
return handler(req);
|
||||
}
|
||||
|
||||
const session = getAuthSession(req, options);
|
||||
if (!session) {
|
||||
return unauthorizedResponse();
|
||||
}
|
||||
|
||||
return handler(req);
|
||||
};
|
||||
};
|
||||
@@ -1,74 +0,0 @@
|
||||
// Simple in-memory cache with TTL support
|
||||
interface CacheEntry<T> {
|
||||
value: T;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
class Cache<T> {
|
||||
private store = new Map<string, CacheEntry<T>>();
|
||||
private ttlMs: number;
|
||||
|
||||
constructor(ttlSeconds: number = 3600) {
|
||||
this.ttlMs = ttlSeconds * 1000;
|
||||
}
|
||||
|
||||
set(key: string, value: T): void {
|
||||
this.store.set(key, {
|
||||
value,
|
||||
expiresAt: Date.now() + this.ttlMs,
|
||||
});
|
||||
}
|
||||
|
||||
get(key: string): T | null {
|
||||
const entry = this.store.get(key);
|
||||
if (!entry) return null;
|
||||
|
||||
if (Date.now() > entry.expiresAt) {
|
||||
this.store.delete(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.value;
|
||||
}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this.get(key) !== null;
|
||||
}
|
||||
|
||||
delete(key: string): void {
|
||||
this.store.delete(key);
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.store.clear();
|
||||
}
|
||||
|
||||
size(): number {
|
||||
return this.store.size;
|
||||
}
|
||||
|
||||
// Cleanup expired entries
|
||||
cleanup(): number {
|
||||
let removed = 0;
|
||||
const now = Date.now();
|
||||
|
||||
for (const [key, entry] of this.store.entries()) {
|
||||
if (now > entry.expiresAt) {
|
||||
this.store.delete(key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
|
||||
// File info cache (1 hour TTL)
|
||||
export const fileInfoCache = new Cache<{
|
||||
file_size: number;
|
||||
mime_type: string;
|
||||
file_path: string;
|
||||
bot_token: string;
|
||||
}>(3600);
|
||||
|
||||
export { Cache };
|
||||
@@ -1,249 +0,0 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { gzipSync } from 'node:zlib';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { db, files as fileSchema } from '../db';
|
||||
import { insertFileParts, listFileParts, type NewFilePartInput } from '../db/file-parts';
|
||||
import type { File } from '../db/schema';
|
||||
import { config } from '../env';
|
||||
import { botPool } from '../infrastructure/telegram/bot-pool';
|
||||
import { computeHash } from './file';
|
||||
import { createGetObjectResponse, type ObjectPartSource } from './s3/object-stream';
|
||||
import type { RangeParseResult } from './s3/range';
|
||||
|
||||
export type ChunkCompressionAlgorithm = 'gzip' | null;
|
||||
|
||||
export interface ChunkedUploadPart {
|
||||
partNumber: number;
|
||||
telegramFileId: string;
|
||||
telegramFileUniqueId: string;
|
||||
storageMessageId: number;
|
||||
sizeBytes: number;
|
||||
storedSizeBytes: number;
|
||||
compressionAlgorithm: ChunkCompressionAlgorithm;
|
||||
etag: string;
|
||||
}
|
||||
|
||||
export interface ChunkedUploadResult {
|
||||
parts: ChunkedUploadPart[];
|
||||
fileHash: string;
|
||||
totalSizeBytes: number;
|
||||
}
|
||||
|
||||
export interface ChunkedFileInput {
|
||||
tempPath: string;
|
||||
partFileNamePrefix: string;
|
||||
fileName: string;
|
||||
mimeType: string;
|
||||
sizeBytes: number;
|
||||
fileType: string;
|
||||
uploaderId: number;
|
||||
bucketId?: string | null;
|
||||
s3Key?: string | null;
|
||||
}
|
||||
|
||||
const asSafeChunkSize = (chunkSizeBytes: number): number => {
|
||||
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
|
||||
throw new Error('Invalid Telegram chunk size');
|
||||
}
|
||||
return chunkSizeBytes;
|
||||
};
|
||||
|
||||
const maybeCompressChunk = (
|
||||
chunk: Buffer,
|
||||
compress: boolean,
|
||||
compressionMinSizeBytes: number,
|
||||
): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => {
|
||||
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
|
||||
return { bytes: chunk, compressionAlgorithm: null };
|
||||
}
|
||||
|
||||
const gzipped = gzipSync(chunk);
|
||||
if (gzipped.byteLength >= chunk.byteLength) {
|
||||
return { bytes: chunk, compressionAlgorithm: null };
|
||||
}
|
||||
|
||||
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
|
||||
};
|
||||
|
||||
export const uploadFileInTelegramChunks = async (input: {
|
||||
tempPath: string;
|
||||
partFileNamePrefix: string;
|
||||
chunkSizeBytes: number;
|
||||
compress: boolean;
|
||||
compressionMinSizeBytes: number;
|
||||
}): Promise<ChunkedUploadResult> => {
|
||||
const chunkSizeBytes = asSafeChunkSize(input.chunkSizeBytes);
|
||||
const hasher = new Bun.CryptoHasher('sha256');
|
||||
const parts: ChunkedUploadPart[] = [];
|
||||
let totalSizeBytes = 0;
|
||||
let partNumber = 0;
|
||||
|
||||
const stream = createReadStream(input.tempPath, { highWaterMark: chunkSizeBytes });
|
||||
|
||||
// Concurrent upload set: tracks in-flight uploads and limits how many
|
||||
// chunks are being uploaded at once from this single file.
|
||||
const inFlight = new Set<Promise<void>>();
|
||||
|
||||
for await (const data of stream) {
|
||||
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data as Uint8Array);
|
||||
if (chunk.byteLength === 0) continue;
|
||||
|
||||
partNumber += 1;
|
||||
totalSizeBytes += chunk.byteLength;
|
||||
hasher.update(chunk);
|
||||
|
||||
const { bytes, compressionAlgorithm } = maybeCompressChunk(
|
||||
chunk,
|
||||
input.compress,
|
||||
input.compressionMinSizeBytes,
|
||||
);
|
||||
const currentPart = partNumber;
|
||||
|
||||
// Fire upload concurrently — don't await inside the read loop
|
||||
const uploadPromise = botPool
|
||||
.forwardToStorage(bytes, `${input.partFileNamePrefix}.part-${currentPart}`, 'document')
|
||||
.then((forwardResult) => {
|
||||
parts.push({
|
||||
partNumber: currentPart,
|
||||
telegramFileId: forwardResult.telegramFileId,
|
||||
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
|
||||
storageMessageId: forwardResult.storageMessageId,
|
||||
sizeBytes: chunk.byteLength,
|
||||
storedSizeBytes: bytes.byteLength,
|
||||
compressionAlgorithm,
|
||||
etag: computeHash(chunk),
|
||||
});
|
||||
});
|
||||
|
||||
// Clean up from in-flight set when done (regardless of success/failure)
|
||||
const trackPromise = uploadPromise.finally(() => {
|
||||
inFlight.delete(trackPromise);
|
||||
});
|
||||
|
||||
inFlight.add(trackPromise);
|
||||
|
||||
// Backpressure: if too many chunks are in-flight, wait for one to
|
||||
// finish before reading more — prevents unbounded memory growth.
|
||||
if (inFlight.size >= botPool.getEffectiveConcurrency() * 2) {
|
||||
await Promise.race(inFlight);
|
||||
// Yield microtask to let .finally() run and remove from inFlight
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for all remaining uploads to finish
|
||||
await Promise.all(inFlight);
|
||||
|
||||
return {
|
||||
parts,
|
||||
fileHash: hasher.digest('hex'),
|
||||
totalSizeBytes,
|
||||
};
|
||||
};
|
||||
|
||||
export const storeFileInTelegramChunks = async (input: ChunkedFileInput): Promise<File> => {
|
||||
const upload = await uploadFileInTelegramChunks({
|
||||
tempPath: input.tempPath,
|
||||
partFileNamePrefix: input.partFileNamePrefix,
|
||||
chunkSizeBytes: config.telegramChunkSizeBytes,
|
||||
compress: config.compressChunkedUploads,
|
||||
compressionMinSizeBytes: config.chunkCompressionMinSizeBytes,
|
||||
});
|
||||
|
||||
const firstPart = upload.parts[0];
|
||||
if (!firstPart) {
|
||||
throw new Error('Chunked upload produced no parts');
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const fileId = randomUUID();
|
||||
const publicId = nanoid();
|
||||
const file: File = {
|
||||
id: fileId,
|
||||
publicId,
|
||||
telegramFileId: firstPart.telegramFileId,
|
||||
telegramFileUniqueId: firstPart.telegramFileUniqueId,
|
||||
storageChatId: config.storageChatId,
|
||||
storageMessageId: firstPart.storageMessageId,
|
||||
fileName: input.fileName,
|
||||
mimeType: input.mimeType,
|
||||
sizeBytes: upload.totalSizeBytes,
|
||||
fileType: input.fileType,
|
||||
uploaderId: input.uploaderId,
|
||||
fileHash: upload.fileHash,
|
||||
archiveTelegramFileId: null,
|
||||
archiveStorageMessageId: null,
|
||||
archiveFileName: null,
|
||||
archiveEntryName: null,
|
||||
archiveMimeType: null,
|
||||
archiveSizeBytes: null,
|
||||
bucketId: input.bucketId ?? null,
|
||||
s3Key: input.s3Key ?? null,
|
||||
storageBackend: 'chunked',
|
||||
isDeleted: false,
|
||||
multipartUploadId: null,
|
||||
partCount: upload.parts.length,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
|
||||
await db.insert(fileSchema).values(file);
|
||||
|
||||
const fileParts: NewFilePartInput[] = upload.parts.map((part) => ({
|
||||
fileId,
|
||||
partNumber: part.partNumber,
|
||||
telegramFileId: part.telegramFileId,
|
||||
telegramFileUniqueId: part.telegramFileUniqueId,
|
||||
storageChatId: config.storageChatId,
|
||||
storageMessageId: part.storageMessageId,
|
||||
sizeBytes: part.sizeBytes,
|
||||
storedSizeBytes: part.storedSizeBytes,
|
||||
compressionAlgorithm: part.compressionAlgorithm,
|
||||
etag: part.etag,
|
||||
}));
|
||||
|
||||
await insertFileParts(fileParts);
|
||||
return file;
|
||||
};
|
||||
|
||||
export const buildChunkedObjectSources = async (file: File): Promise<ObjectPartSource[]> => {
|
||||
const parts = await listFileParts(file.id);
|
||||
const sources: ObjectPartSource[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
const fileInfo = await botPool.getFileInfo(part.telegramFileId);
|
||||
sources.push({
|
||||
telegramFileId: part.telegramFileId,
|
||||
telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
|
||||
sizeBytes: part.sizeBytes,
|
||||
storedSizeBytes: part.storedSizeBytes,
|
||||
compressionAlgorithm: part.compressionAlgorithm,
|
||||
partNumber: part.partNumber,
|
||||
});
|
||||
}
|
||||
|
||||
return sources;
|
||||
};
|
||||
|
||||
export const createChunkedObjectResponse = async (input: {
|
||||
file: File;
|
||||
range: RangeParseResult;
|
||||
reqId: string;
|
||||
}): Promise<Response> => {
|
||||
const parts = await buildChunkedObjectSources(input.file);
|
||||
if (parts.length === 0) {
|
||||
throw new Error('Chunked object has no parts');
|
||||
}
|
||||
|
||||
return createGetObjectResponse({
|
||||
reqId: input.reqId,
|
||||
contentType: input.file.mimeType,
|
||||
etag: input.file.fileHash || parts.map((p) => p.telegramFileId).join('-'),
|
||||
lastModified:
|
||||
input.file.createdAt instanceof Date ? input.file.createdAt : new Date(input.file.createdAt),
|
||||
totalSize: Number(input.file.sizeBytes),
|
||||
parts,
|
||||
range: input.range,
|
||||
});
|
||||
};
|
||||
@@ -1,237 +0,0 @@
|
||||
import { unlink } from 'node:fs/promises';
|
||||
import logger from './logger';
|
||||
|
||||
export const getErrorMessage = (error: unknown): string => {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
};
|
||||
|
||||
export const cleanupTempFile = async (tempPath: string): Promise<void> => {
|
||||
try {
|
||||
await unlink(tempPath);
|
||||
} catch (err) {
|
||||
logger.warn('Failed to cleanup temp file', { tempPath, error: getErrorMessage(err) });
|
||||
}
|
||||
};
|
||||
|
||||
interface FileMetadata {
|
||||
publicId: string;
|
||||
telegramFileId: string;
|
||||
telegramFileUniqueId: string;
|
||||
storageChatId: number;
|
||||
storageMessageId: number;
|
||||
fileName: string;
|
||||
mimeType: string;
|
||||
sizeBytes: number;
|
||||
fileType: string;
|
||||
uploaderId: number;
|
||||
createdAt: Date | string | number;
|
||||
}
|
||||
|
||||
const FILE_TYPES: Record<string, number> = {
|
||||
document: 2 * 1024 * 1024 * 1024, // 2GB
|
||||
photo: 10 * 1024 * 1024, // 10MB
|
||||
video: 2 * 1024 * 1024 * 1024, // 2GB
|
||||
audio: 200 * 1024 * 1024, // 200MB
|
||||
voice: 200 * 1024 * 1024, // 200MB
|
||||
animation: 2 * 1024 * 1024 * 1024, // 2GB
|
||||
sticker: 10 * 1024 * 1024, // 10MB
|
||||
video_note: 2 * 1024 * 1024 * 1024, // 2GB
|
||||
};
|
||||
|
||||
export const getFileType = (mime: string | null, caption?: string): string => {
|
||||
const mimeUpper = mime?.split('/')[0]?.toLowerCase();
|
||||
const captionLower = caption?.toLowerCase();
|
||||
|
||||
if (mime?.toLowerCase() === 'image/webp' || captionLower?.includes('sticker')) return 'sticker';
|
||||
if (captionLower?.includes('video_note')) return 'video_note';
|
||||
if (mimeUpper === 'video') return 'video';
|
||||
if (mimeUpper === 'audio') return 'audio';
|
||||
if (mimeUpper === 'document') return 'document';
|
||||
if (mimeUpper === 'image') return captionLower?.includes('gif') ? 'animation' : 'photo';
|
||||
if (captionLower?.includes('voice')) return 'voice';
|
||||
if (captionLower?.includes('animation')) return 'animation';
|
||||
|
||||
return mimeUpper === 'application' ? 'application' : 'document';
|
||||
};
|
||||
|
||||
export const checkFileSize = (sizeBytes: number, fileType: string): boolean => {
|
||||
const limit = FILE_TYPES[fileType] || FILE_TYPES.document;
|
||||
return sizeBytes <= limit;
|
||||
};
|
||||
|
||||
export const ensureExtension = (
|
||||
fileName: string,
|
||||
buffer: Buffer,
|
||||
detectedMime?: string,
|
||||
): { fileName: string; mimeType: string } => {
|
||||
const mimeMap: Record<string, string> = {
|
||||
'application/pdf': 'pdf',
|
||||
'image/png': 'png',
|
||||
'image/jpeg': 'jpg',
|
||||
'image/gif': 'gif',
|
||||
'text/plain': 'txt',
|
||||
'application/zip': 'zip',
|
||||
};
|
||||
|
||||
let ext: string | null = null;
|
||||
if (buffer.subarray(0, 4).toString() === '%PDF') {
|
||||
ext = 'pdf';
|
||||
} else if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
|
||||
ext = 'png';
|
||||
} else if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
|
||||
ext = 'jpg';
|
||||
} else if (buffer.subarray(0, 4).toString() === 'GIF8') {
|
||||
ext = 'gif';
|
||||
} else if (detectedMime) {
|
||||
ext = mimeMap[detectedMime.toLowerCase()] || null;
|
||||
}
|
||||
|
||||
let finalFileName = fileName;
|
||||
const hasExtension = fileName.includes('.') && fileName.split('.').pop()!.length >= 2;
|
||||
if (!hasExtension && ext) {
|
||||
finalFileName = `${fileName}.${ext}`;
|
||||
}
|
||||
|
||||
const mimeType = ext
|
||||
? Object.keys(mimeMap).find((k) => mimeMap[k] === ext) ||
|
||||
detectedMime ||
|
||||
'application/octet-stream'
|
||||
: detectedMime || 'application/octet-stream';
|
||||
|
||||
return { fileName: finalFileName, mimeType };
|
||||
};
|
||||
|
||||
type HeaderMapRequest = {
|
||||
headers?:
|
||||
| {
|
||||
get?: (name: string) => string | null;
|
||||
}
|
||||
| Record<string, string>;
|
||||
};
|
||||
|
||||
type FileLike = {
|
||||
fileName?: string;
|
||||
mimeType?: string;
|
||||
};
|
||||
|
||||
type MessageLike = {
|
||||
document?: FileLike;
|
||||
photo?: FileLike[];
|
||||
audio?: FileLike;
|
||||
voice?: FileLike;
|
||||
animation?: FileLike;
|
||||
};
|
||||
|
||||
const getHeader = (request: HeaderMapRequest | null, name: string): string | undefined => {
|
||||
const headers = request?.headers;
|
||||
if (!headers) return undefined;
|
||||
|
||||
const get = 'get' in headers ? headers.get : undefined;
|
||||
if (typeof get === 'function') return get(name) || undefined;
|
||||
|
||||
return (headers as Record<string, string>)[name];
|
||||
};
|
||||
|
||||
export const extractFileName = (msg: MessageLike, request: HeaderMapRequest | null): string => {
|
||||
const headerFileName = getHeader(request, 'x-file-name');
|
||||
if (headerFileName) return headerFileName;
|
||||
|
||||
return (
|
||||
msg.document?.fileName ||
|
||||
msg.photo?.slice(-1)[0]?.fileName ||
|
||||
msg.audio?.fileName ||
|
||||
msg.voice?.fileName ||
|
||||
msg.animation?.fileName ||
|
||||
'file'
|
||||
);
|
||||
};
|
||||
|
||||
export const extractMimeType = (msg: MessageLike, request: HeaderMapRequest | null): string => {
|
||||
const headerMimeType = getHeader(request, 'x-mime-type');
|
||||
if (headerMimeType) return headerMimeType;
|
||||
|
||||
return (
|
||||
msg.document?.mimeType ||
|
||||
msg.photo?.slice(-1)[0]?.mimeType ||
|
||||
msg.audio?.mimeType ||
|
||||
msg.voice?.mimeType ||
|
||||
msg.animation?.mimeType ||
|
||||
'application/octet-stream'
|
||||
);
|
||||
};
|
||||
|
||||
export const computeHash = (buffer: Buffer): string => {
|
||||
const hasher = new Bun.CryptoHasher('sha256');
|
||||
hasher.update(buffer);
|
||||
return hasher.digest('hex');
|
||||
};
|
||||
|
||||
export interface TelegramMessageFile {
|
||||
file_id: string;
|
||||
file_unique_id: string;
|
||||
file_size?: number;
|
||||
mime_type?: string;
|
||||
file_name?: string;
|
||||
}
|
||||
|
||||
export interface TelegramMediaMessage {
|
||||
message_id: number;
|
||||
document?: TelegramMessageFile;
|
||||
photo?: TelegramMessageFile[];
|
||||
video?: TelegramMessageFile;
|
||||
audio?: TelegramMessageFile;
|
||||
voice?: TelegramMessageFile;
|
||||
animation?: TelegramMessageFile;
|
||||
sticker?: TelegramMessageFile;
|
||||
video_note?: TelegramMessageFile;
|
||||
}
|
||||
|
||||
export const extractFileFromMessage = (
|
||||
msg: TelegramMediaMessage,
|
||||
fileType: string,
|
||||
): TelegramMessageFile => {
|
||||
if (fileType === 'photo') return msg.photo?.slice(-1)[0] as TelegramMessageFile;
|
||||
if (fileType === 'sticker') return msg.sticker as TelegramMessageFile;
|
||||
return msg[fileType as keyof TelegramMediaMessage] as TelegramMessageFile;
|
||||
};
|
||||
|
||||
export const detectFileType = (msg: TelegramMediaMessage): string => {
|
||||
if (msg.document) return 'document';
|
||||
if (msg.photo) return 'photo';
|
||||
if (msg.video) return 'video';
|
||||
if (msg.audio) return 'audio';
|
||||
if (msg.voice) return 'voice';
|
||||
if (msg.animation) return 'animation';
|
||||
if (msg.sticker) return 'sticker';
|
||||
if (msg.video_note) return 'video_note';
|
||||
return 'document';
|
||||
};
|
||||
|
||||
export const getFileSizeLimit = (fileType: string): number =>
|
||||
FILE_TYPES[fileType] || FILE_TYPES.document;
|
||||
|
||||
export const formatCreatedAt = (createdAt: Date | string | number): string => {
|
||||
return createdAt instanceof Date ? createdAt.toISOString() : new Date(createdAt).toISOString();
|
||||
};
|
||||
|
||||
export interface UploadResponse {
|
||||
public_id: string;
|
||||
file_name: string;
|
||||
mime_type: string;
|
||||
size_bytes: number;
|
||||
file_type: string;
|
||||
created_at: string;
|
||||
download_url: string;
|
||||
}
|
||||
|
||||
export const buildUploadResponse = (file: FileMetadata, baseUrl: string): UploadResponse => {
|
||||
return {
|
||||
public_id: file.publicId,
|
||||
file_name: file.fileName,
|
||||
mime_type: file.mimeType,
|
||||
size_bytes: file.sizeBytes,
|
||||
file_type: file.fileType,
|
||||
created_at: formatCreatedAt(file.createdAt),
|
||||
download_url: `${baseUrl}/f/${file.publicId}`,
|
||||
};
|
||||
};
|
||||
@@ -1,16 +0,0 @@
|
||||
import { config } from '../env';
|
||||
|
||||
export const extractClientIp = (req: Request): string => {
|
||||
if (!config.trustProxy) return '127.0.0.1';
|
||||
|
||||
const forwardedFor = req.headers.get('x-forwarded-for');
|
||||
if (forwardedFor) {
|
||||
const firstIp = forwardedFor.split(',')[0]?.trim();
|
||||
if (firstIp) return firstIp;
|
||||
}
|
||||
|
||||
const realIp = req.headers.get('x-real-ip')?.trim();
|
||||
if (realIp) return realIp;
|
||||
|
||||
return '127.0.0.1';
|
||||
};
|
||||
@@ -1,25 +0,0 @@
|
||||
import winston from 'winston';
|
||||
|
||||
const logger = winston.createLogger({
|
||||
level: process.env.LOG_LEVEL || 'info',
|
||||
format: winston.format.combine(
|
||||
winston.format.timestamp(),
|
||||
winston.format.errors({ stack: true }),
|
||||
winston.format.json(),
|
||||
),
|
||||
defaultMeta: { service: 'filedrop' },
|
||||
transports: [
|
||||
// Write all logs including error logs to file
|
||||
new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
|
||||
new winston.transports.File({ filename: 'logs/combined.log' }),
|
||||
// Console transport for docker logs / CLI visibility
|
||||
new winston.transports.Console({
|
||||
format:
|
||||
process.env.NODE_ENV !== 'production'
|
||||
? winston.format.combine(winston.format.colorize(), winston.format.simple())
|
||||
: winston.format.json(),
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
export default logger;
|
||||
@@ -1,111 +0,0 @@
|
||||
// No imports needed — logger used only by setInterval which moved to index.ts
|
||||
|
||||
interface Metric {
|
||||
name: string;
|
||||
value: number;
|
||||
timestamp: number;
|
||||
tags?: Record<string, string>;
|
||||
}
|
||||
|
||||
interface MetricsSnapshot {
|
||||
uploadLatency: { p50: number; p95: number; p99: number };
|
||||
uploadThroughput: number;
|
||||
queueSize: number;
|
||||
errorRate: number;
|
||||
cacheHitRate: number;
|
||||
botUtilization: number;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
class MetricsCollector {
|
||||
private metrics: Metric[] = [];
|
||||
private uploadTimes: number[] = [];
|
||||
private errorCount = 0;
|
||||
private totalRequests = 0;
|
||||
private cacheHits = 0;
|
||||
private cacheMisses = 0;
|
||||
private maxMetricsSize = 10000;
|
||||
|
||||
recordUploadTime(durationMs: number): void {
|
||||
this.uploadTimes.push(durationMs);
|
||||
this.totalRequests++;
|
||||
|
||||
// Keep only last 1000 measurements
|
||||
if (this.uploadTimes.length > 1000) {
|
||||
this.uploadTimes.shift();
|
||||
}
|
||||
}
|
||||
|
||||
recordError(): void {
|
||||
this.errorCount++;
|
||||
}
|
||||
|
||||
recordCacheHit(): void {
|
||||
this.cacheHits++;
|
||||
}
|
||||
|
||||
recordCacheMiss(): void {
|
||||
this.cacheMisses++;
|
||||
}
|
||||
|
||||
recordMetric(name: string, value: number, tags?: Record<string, string>): void {
|
||||
this.metrics.push({
|
||||
name,
|
||||
value,
|
||||
timestamp: Date.now(),
|
||||
tags,
|
||||
});
|
||||
|
||||
// Keep metrics bounded
|
||||
if (this.metrics.length > this.maxMetricsSize) {
|
||||
this.metrics = this.metrics.slice(-this.maxMetricsSize);
|
||||
}
|
||||
}
|
||||
|
||||
private calculatePercentile(arr: number[], percentile: number): number {
|
||||
if (arr.length === 0) return 0;
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const index = Math.ceil((percentile / 100) * sorted.length) - 1;
|
||||
return sorted[Math.max(0, index)];
|
||||
}
|
||||
|
||||
getSnapshot(): MetricsSnapshot {
|
||||
const errorRate = this.totalRequests > 0 ? (this.errorCount / this.totalRequests) * 100 : 0;
|
||||
const cacheHitRate =
|
||||
this.cacheHits + this.cacheMisses > 0
|
||||
? (this.cacheHits / (this.cacheHits + this.cacheMisses)) * 100
|
||||
: 0;
|
||||
|
||||
return {
|
||||
uploadLatency: {
|
||||
p50: this.calculatePercentile(this.uploadTimes, 50),
|
||||
p95: this.calculatePercentile(this.uploadTimes, 95),
|
||||
p99: this.calculatePercentile(this.uploadTimes, 99),
|
||||
},
|
||||
uploadThroughput: this.totalRequests > 0 ? this.totalRequests / 60 : 0,
|
||||
queueSize: 0, // Will be updated by queue
|
||||
errorRate,
|
||||
cacheHitRate,
|
||||
botUtilization: 0, // Will be updated by bot tracker
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.uploadTimes = [];
|
||||
this.errorCount = 0;
|
||||
this.totalRequests = 0;
|
||||
this.cacheHits = 0;
|
||||
this.cacheMisses = 0;
|
||||
this.metrics = [];
|
||||
}
|
||||
|
||||
getMetrics(name?: string): Metric[] {
|
||||
if (!name) return this.metrics;
|
||||
return this.metrics.filter((m) => m.name === name);
|
||||
}
|
||||
}
|
||||
|
||||
export const metricsCollector = new MetricsCollector();
|
||||
|
||||
export { MetricsCollector };
|
||||
@@ -1,89 +0,0 @@
|
||||
import { config } from '../env';
|
||||
import { extractClientIp } from './ip';
|
||||
import logger from './logger';
|
||||
|
||||
interface RateLimitEntry {
|
||||
count: number;
|
||||
resetTime: number;
|
||||
}
|
||||
|
||||
const rateLimitStore = new Map<string, RateLimitEntry>();
|
||||
const MAX_STORE_ENTRIES = 50000;
|
||||
|
||||
const evictExpiredEntries = (now = Date.now()): number => {
|
||||
let cleaned = 0;
|
||||
|
||||
for (const [key, entry] of rateLimitStore.entries()) {
|
||||
if (now > entry.resetTime) {
|
||||
rateLimitStore.delete(key);
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
};
|
||||
|
||||
const ensureStoreCapacity = (now: number): void => {
|
||||
if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
|
||||
|
||||
evictExpiredEntries(now);
|
||||
while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
|
||||
const oldestKey = rateLimitStore.keys().next().value;
|
||||
if (!oldestKey) break;
|
||||
rateLimitStore.delete(oldestKey);
|
||||
}
|
||||
};
|
||||
|
||||
export const checkRateLimit = (key: string): boolean => {
|
||||
const now = Date.now();
|
||||
const entry = rateLimitStore.get(key);
|
||||
|
||||
if (!entry || now > entry.resetTime) {
|
||||
ensureStoreCapacity(now);
|
||||
rateLimitStore.set(key, {
|
||||
count: 1,
|
||||
resetTime: now + config.rateLimitWindowMs,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
if (entry.count >= config.rateLimitMaxRequests) {
|
||||
logger.warn('Rate limit exceeded', { key, count: entry.count });
|
||||
return false;
|
||||
}
|
||||
|
||||
entry.count++;
|
||||
return true;
|
||||
};
|
||||
|
||||
export const withRateLimit = <T extends Request>(
|
||||
handler: (req: T) => Promise<Response>,
|
||||
): ((req: T) => Promise<Response>) => {
|
||||
return async (req: T): Promise<Response> => {
|
||||
const ip = extractClientIp(req);
|
||||
if (!checkRateLimit(ip)) {
|
||||
return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
|
||||
}
|
||||
|
||||
return handler(req);
|
||||
};
|
||||
};
|
||||
|
||||
export const cleanupRateLimitCache = (): void => {
|
||||
const cleaned = evictExpiredEntries();
|
||||
|
||||
if (cleaned > 0) {
|
||||
logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
|
||||
}
|
||||
};
|
||||
|
||||
export const getRateLimitStats = () => ({
|
||||
trackedIPs: rateLimitStore.size,
|
||||
windowSize: config.rateLimitWindowMs,
|
||||
maxRequests: config.rateLimitMaxRequests,
|
||||
maxTrackedIPs: MAX_STORE_ENTRIES,
|
||||
});
|
||||
|
||||
export const clearRateLimitCache = (): void => {
|
||||
rateLimitStore.clear();
|
||||
};
|
||||
@@ -1,91 +0,0 @@
|
||||
import logger from './logger';
|
||||
|
||||
interface RetryOptions {
|
||||
maxRetries?: number;
|
||||
initialDelayMs?: number;
|
||||
maxDelayMs?: number;
|
||||
backoffMultiplier?: number;
|
||||
shouldRetry?: (error: unknown) => boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_OPTIONS: Required<RetryOptions> = {
|
||||
maxRetries: 3,
|
||||
initialDelayMs: 100,
|
||||
maxDelayMs: 5000,
|
||||
backoffMultiplier: 2,
|
||||
shouldRetry: (error: unknown) => {
|
||||
const errorStr = error instanceof Error ? error.message : String(error);
|
||||
// Retry on transient errors
|
||||
return (
|
||||
errorStr.includes('ECONNREFUSED') ||
|
||||
errorStr.includes('ETIMEDOUT') ||
|
||||
errorStr.includes('ENOTFOUND') ||
|
||||
errorStr.includes('429') ||
|
||||
errorStr.includes('timeout')
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
export const withRetry = async <T>(
|
||||
fn: () => Promise<T>,
|
||||
options: RetryOptions = {},
|
||||
): Promise<T> => {
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
let lastError: unknown;
|
||||
let delay = opts.initialDelayMs;
|
||||
|
||||
for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error: unknown) {
|
||||
lastError = error;
|
||||
const errorStr = error instanceof Error ? error.message : String(error);
|
||||
|
||||
if (attempt === opts.maxRetries || !opts.shouldRetry(error)) {
|
||||
logger.error('Retry exhausted', {
|
||||
attempt,
|
||||
maxRetries: opts.maxRetries,
|
||||
error: errorStr,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
|
||||
logger.warn('Retrying after error', {
|
||||
attempt,
|
||||
delay,
|
||||
error: errorStr,
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
delay = Math.min(delay * opts.backoffMultiplier, opts.maxDelayMs);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
};
|
||||
|
||||
export const withTimeout = async <T>(
|
||||
fn: () => Promise<T>,
|
||||
timeoutMs: number = 30000,
|
||||
): Promise<T> => {
|
||||
return Promise.race([
|
||||
fn(),
|
||||
new Promise<T>((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`Operation timeout after ${timeoutMs}ms`)), timeoutMs),
|
||||
),
|
||||
]);
|
||||
};
|
||||
|
||||
export const withFallback = async <T>(
|
||||
primary: () => Promise<T>,
|
||||
fallback: () => Promise<T>,
|
||||
): Promise<T> => {
|
||||
try {
|
||||
return await primary();
|
||||
} catch (error: unknown) {
|
||||
logger.warn('Primary operation failed, using fallback', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return fallback();
|
||||
}
|
||||
};
|
||||
@@ -1,463 +0,0 @@
|
||||
import { timingSafeEqual } from 'node:crypto';
|
||||
|
||||
/**
|
||||
* Timing-safe string comparison that prevents timing attacks.
|
||||
*
|
||||
* Uses `crypto.timingSafeEqual` which runs in constant time regardless of
|
||||
* where the strings differ. Returns false for mismatched-length inputs
|
||||
* to avoid leaking length information via early return.
|
||||
*
|
||||
* @param left - The first string to compare.
|
||||
* @param right - The second string to compare.
|
||||
* @returns True if both strings are equal.
|
||||
*/
|
||||
const timingSafeCompare = (left: string, right: string): boolean => {
|
||||
const leftBuffer = Buffer.from(left);
|
||||
const rightBuffer = Buffer.from(right);
|
||||
|
||||
if (leftBuffer.length !== rightBuffer.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return timingSafeEqual(leftBuffer, rightBuffer);
|
||||
};
|
||||
|
||||
export interface SigV4Result {
|
||||
isValid: boolean;
|
||||
credential: {
|
||||
accessKey: string;
|
||||
date: string;
|
||||
region: string;
|
||||
service: string;
|
||||
} | null;
|
||||
errorCode?: string;
|
||||
}
|
||||
|
||||
export interface VerifyPresignedUrlInput {
|
||||
url: string;
|
||||
method: string;
|
||||
headers: Record<string, string>;
|
||||
s3AccessKey: string;
|
||||
s3SecretKey: string;
|
||||
region: string;
|
||||
now?: Date;
|
||||
}
|
||||
|
||||
const SERVICE = 's3';
|
||||
const TERMINATION = 'aws4_request';
|
||||
|
||||
/**
|
||||
* Maximum acceptable clock skew between client and server for header-based
|
||||
* SigV4 authentication. AWS allows 15 minutes.
|
||||
*/
|
||||
const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => {
|
||||
if (data instanceof Uint8Array) return data;
|
||||
if (data instanceof ArrayBuffer) return new Uint8Array(data);
|
||||
return new TextEncoder().encode(data);
|
||||
};
|
||||
|
||||
const sha256Hex = async (data: string | Uint8Array | ArrayBuffer): Promise<string> => {
|
||||
const hashBuffer = await crypto.subtle.digest('SHA-256', buf(data) as never);
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
|
||||
};
|
||||
|
||||
const hmacSha256 = async (key: Uint8Array, message: string): Promise<Uint8Array> => {
|
||||
const cryptoKey = await crypto.subtle.importKey(
|
||||
'raw',
|
||||
key as never,
|
||||
{ name: 'HMAC', hash: 'SHA-256' },
|
||||
false,
|
||||
['sign'],
|
||||
);
|
||||
const result = await crypto.subtle.sign('HMAC', cryptoKey, buf(message) as never);
|
||||
return new Uint8Array(result);
|
||||
};
|
||||
|
||||
const getSigningKey = async (
|
||||
secretKey: string,
|
||||
dateStamp: string,
|
||||
region: string,
|
||||
): Promise<Uint8Array> => {
|
||||
let key = await hmacSha256(buf(`AWS4${secretKey}`), dateStamp);
|
||||
key = await hmacSha256(key, region);
|
||||
key = await hmacSha256(key, SERVICE);
|
||||
return await hmacSha256(key, TERMINATION);
|
||||
};
|
||||
|
||||
const hmacHex = async (key: Uint8Array, message: string): Promise<string> => {
|
||||
const result = await hmacSha256(key, message);
|
||||
return Array.from(result)
|
||||
.map((b) => b.toString(16).padStart(2, '0'))
|
||||
.join('');
|
||||
};
|
||||
|
||||
const parseAuthorizationHeader = (authHeader: string) => {
|
||||
const credentialMatch = authHeader.match(/Credential=([^,]+)/);
|
||||
const signedHeadersMatch = authHeader.match(/SignedHeaders=([^,]+)/);
|
||||
const signatureMatch = authHeader.match(/Signature=([^,]+)/);
|
||||
|
||||
if (!credentialMatch || !signedHeadersMatch || !signatureMatch) return null;
|
||||
|
||||
const credentialParts = credentialMatch[1].split('/');
|
||||
if (credentialParts.length !== 5) return null;
|
||||
|
||||
return {
|
||||
accessKey: credentialParts[0],
|
||||
date: credentialParts[1],
|
||||
region: credentialParts[2],
|
||||
service: credentialParts[3],
|
||||
termination: credentialParts[4],
|
||||
signedHeaders: signedHeadersMatch[1],
|
||||
signature: signatureMatch[1],
|
||||
};
|
||||
};
|
||||
|
||||
const buildCanonicalRequest = (
|
||||
method: string,
|
||||
canonicalUri: string,
|
||||
canonicalQueryString: string,
|
||||
signedHeaders: string,
|
||||
headers: Record<string, string>,
|
||||
hashedPayload: string,
|
||||
): string => {
|
||||
const canonicalHeaders = signedHeaders
|
||||
.split(';')
|
||||
.map((h) => {
|
||||
const value = headers[h.toLowerCase()] || '';
|
||||
return `${h.toLowerCase()}:${value.trim()}\n`;
|
||||
})
|
||||
.join('');
|
||||
|
||||
return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Normalizes a URI per AWS SigV4 requirements plus RFC 3986:
|
||||
*
|
||||
* 1. Decode percent-encoded characters
|
||||
* 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4
|
||||
*
|
||||
* @param uri - The raw URI path to normalize.
|
||||
* @returns The normalized URI path.
|
||||
*/
|
||||
const normalizeUri = (uri: string): string => {
|
||||
if (!uri || uri === '') return '/';
|
||||
|
||||
// AWS SigV4 requires URI-decoded paths in the canonical request
|
||||
// Only `.` and `..` segments are removed per RFC 3986 section 5.2.4
|
||||
// Empty segments (from `//` or trailing `/`) are preserved — they are
|
||||
// part of the URI and the SDK signs them.
|
||||
const decoded = decodeURIComponent(uri);
|
||||
const segments = decoded.split('/');
|
||||
const result: string[] = [];
|
||||
|
||||
for (const segment of segments) {
|
||||
if (segment === '.') continue;
|
||||
if (segment === '..') {
|
||||
result.pop();
|
||||
continue;
|
||||
}
|
||||
result.push(segment);
|
||||
}
|
||||
|
||||
// Join preserves empty first segment (from leading /) automatically
|
||||
return result.join('/') || '/';
|
||||
};
|
||||
|
||||
const awsEncode = (value: string): string =>
|
||||
encodeURIComponent(value).replace(
|
||||
/[!'()*]/g,
|
||||
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
|
||||
);
|
||||
|
||||
export const buildCanonicalQueryString = (
|
||||
searchParams: URLSearchParams,
|
||||
excludeKeys: Set<string> = new Set(),
|
||||
): string => {
|
||||
const pairs: Array<[string, string]> = [];
|
||||
for (const [key, value] of searchParams.entries()) {
|
||||
if (!excludeKeys.has(key)) pairs.push([key, value]);
|
||||
}
|
||||
// AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare
|
||||
pairs.sort(([ak, av], [bk, bv]) => {
|
||||
const a = `${awsEncode(ak)}=${awsEncode(av)}`;
|
||||
const b = `${awsEncode(bk)}=${awsEncode(bv)}`;
|
||||
if (a < b) return -1;
|
||||
if (a > b) return 1;
|
||||
return 0;
|
||||
});
|
||||
return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&');
|
||||
};
|
||||
|
||||
const getHashedPayload = async (body: string | null): Promise<string> => {
|
||||
if (!body || body.length === 0) return await sha256Hex('');
|
||||
return await sha256Hex(body);
|
||||
};
|
||||
|
||||
/**
|
||||
* Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date.
|
||||
*
|
||||
* @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format.
|
||||
* @returns The parsed Date, or null if the format is invalid.
|
||||
*/
|
||||
const parseAmzDateUtc = (amzDate: string): Date | null => {
|
||||
const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/);
|
||||
if (!match) return null;
|
||||
const [, year, month, day, hour, minute, second] = match;
|
||||
return new Date(
|
||||
Date.UTC(
|
||||
Number.parseInt(year, 10),
|
||||
Number.parseInt(month, 10) - 1,
|
||||
Number.parseInt(day, 10),
|
||||
Number.parseInt(hour, 10),
|
||||
Number.parseInt(minute, 10),
|
||||
Number.parseInt(second, 10),
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Validates that `host` is included in the signed headers list.
|
||||
*
|
||||
* AWS SigV4 mandates that `host` is always signed. Reject requests that
|
||||
* omit it to prevent header injection / replay variants.
|
||||
*
|
||||
* @param signedHeaders - The semicolon-separated signed headers string.
|
||||
* @returns True if `host` is present.
|
||||
*/
|
||||
const validateSignedHeaders = (signedHeaders: string): boolean => {
|
||||
return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host');
|
||||
};
|
||||
|
||||
export const verifySignature = async (
|
||||
method: string,
|
||||
url: string,
|
||||
headers: Record<string, string>,
|
||||
body: string | null,
|
||||
s3AccessKey: string,
|
||||
s3SecretKey: string,
|
||||
region: string,
|
||||
): Promise<SigV4Result> => {
|
||||
const authHeader = headers.authorization;
|
||||
if (!authHeader?.startsWith('AWS4-HMAC-SHA256')) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const parsed = parseAuthorizationHeader(authHeader);
|
||||
if (!parsed) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
if (!timingSafeCompare(parsed.accessKey, s3AccessKey)) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
if (!timingSafeCompare(parsed.region, region)) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
// Validate service and termination in credential scope (M2)
|
||||
if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
// Validate host is in signed headers (LOW/host)
|
||||
if (!validateSignedHeaders(parsed.signedHeaders)) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const parsedUrl = new URL(url, 'http://localhost');
|
||||
const canonicalUri = normalizeUri(parsedUrl.pathname);
|
||||
const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams);
|
||||
|
||||
const contentSha256 = headers['x-amz-content-sha256'] || null;
|
||||
if (contentSha256?.startsWith('STREAMING-')) {
|
||||
return { isValid: false, credential: null, errorCode: 'NotImplemented' };
|
||||
}
|
||||
|
||||
// CRITICAL: Use the x-amz-content-sha256 header value in the canonical
|
||||
// request because that's what the client signed. The actual body hash is
|
||||
// verified by verifyBodyHash() after streaming, ensuring integrity without
|
||||
// breaking SigV4.
|
||||
const hashedPayload = contentSha256 || (await getHashedPayload(body));
|
||||
|
||||
const canonicalRequest = buildCanonicalRequest(
|
||||
method,
|
||||
canonicalUri,
|
||||
canonicalQueryString,
|
||||
parsed.signedHeaders,
|
||||
headers,
|
||||
hashedPayload,
|
||||
);
|
||||
|
||||
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
|
||||
|
||||
// M1: Fall back to Date header if x-amz-date is missing
|
||||
const amzDate = headers['x-amz-date'] || headers.date || '';
|
||||
|
||||
// H5: Validate request freshness (clock skew / replay protection)
|
||||
if (amzDate) {
|
||||
const requestDate = parseAmzDateUtc(amzDate);
|
||||
if (requestDate) {
|
||||
const now = Date.now();
|
||||
const skew = Math.abs(now - requestDate.getTime());
|
||||
if (skew > MAX_CLOCK_SKEW_MS) {
|
||||
return { isValid: false, credential: null, errorCode: 'RequestExpired' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const dateStamp = parsed.date;
|
||||
|
||||
// M3: Ensure date in credential scope matches x-amz-date
|
||||
if (amzDate) {
|
||||
const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD"
|
||||
if (amzDateStamp !== dateStamp) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
}
|
||||
|
||||
const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`;
|
||||
|
||||
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
|
||||
|
||||
const signingKey = await getSigningKey(s3SecretKey, dateStamp, region);
|
||||
const expectedSignature = await hmacHex(signingKey, stringToSign);
|
||||
|
||||
if (!timingSafeCompare(expectedSignature, parsed.signature)) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
return {
|
||||
isValid: true,
|
||||
credential: {
|
||||
accessKey: parsed.accessKey,
|
||||
date: parsed.date,
|
||||
region: parsed.region,
|
||||
service: parsed.service,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const verifyPresignedUrl = async ({
|
||||
url,
|
||||
method,
|
||||
headers,
|
||||
s3AccessKey,
|
||||
s3SecretKey,
|
||||
region,
|
||||
now = new Date(),
|
||||
}: VerifyPresignedUrlInput): Promise<SigV4Result> => {
|
||||
const parsedUrl = new URL(url);
|
||||
const searchParams = parsedUrl.searchParams;
|
||||
|
||||
const algorithm = searchParams.get('X-Amz-Algorithm');
|
||||
const credential = searchParams.get('X-Amz-Credential');
|
||||
const signedHeaders = searchParams.get('X-Amz-SignedHeaders');
|
||||
const signature = searchParams.get('X-Amz-Signature');
|
||||
const expiresText = searchParams.get('X-Amz-Expires');
|
||||
const amzDate = searchParams.get('X-Amz-Date');
|
||||
|
||||
if (
|
||||
algorithm !== 'AWS4-HMAC-SHA256' ||
|
||||
!credential ||
|
||||
!signedHeaders ||
|
||||
!signature ||
|
||||
!expiresText ||
|
||||
!amzDate
|
||||
) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const expires = Number.parseInt(expiresText, 10);
|
||||
const signedAt = parseAmzDateUtc(amzDate);
|
||||
if (!Number.isFinite(expires) || expires <= 0 || !signedAt) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
// AWS S3 spec limits presigned URLs to 7 days (604800 seconds)
|
||||
const MAX_PRESIGNED_EXPIRY_SECONDS = 604800;
|
||||
if (
|
||||
now.getTime() > signedAt.getTime() + expires * 1000 ||
|
||||
expires > MAX_PRESIGNED_EXPIRY_SECONDS
|
||||
) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const credParts = credential.split('/');
|
||||
if (credParts.length !== 5) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
const [accessKey, dateStamp, credentialRegion, service, termination] = credParts;
|
||||
if (
|
||||
!timingSafeCompare(accessKey, s3AccessKey) ||
|
||||
!timingSafeCompare(credentialRegion, region) ||
|
||||
service !== SERVICE ||
|
||||
termination !== TERMINATION
|
||||
) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
|
||||
// Validate host is in signed headers for presigned URLs too
|
||||
if (!validateSignedHeaders(signedHeaders)) {
|
||||
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
|
||||
}
|
||||
|
||||
const signedHeaderList = signedHeaders.split(';').filter(Boolean);
|
||||
const canonicalHeaders = signedHeaderList
|
||||
.map((headerName) => {
|
||||
const lower = headerName.toLowerCase();
|
||||
const value = lower === 'host' ? headers.host || parsedUrl.host : headers[lower] || '';
|
||||
return `${lower}:${value.trim()}\n`;
|
||||
})
|
||||
.join('');
|
||||
|
||||
const canonicalRequest = `${method}\n${normalizeUri(parsedUrl.pathname)}\n${buildCanonicalQueryString(searchParams, new Set(['X-Amz-Signature']))}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`;
|
||||
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
|
||||
const credentialScope = `${dateStamp}/${region}/${SERVICE}/${TERMINATION}`;
|
||||
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
|
||||
const expectedSignature = await hmacHex(
|
||||
await getSigningKey(s3SecretKey, dateStamp, region),
|
||||
stringToSign,
|
||||
);
|
||||
|
||||
if (!timingSafeCompare(expectedSignature, signature)) {
|
||||
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
|
||||
}
|
||||
return { isValid: true, credential: { accessKey, date: dateStamp, region, service } };
|
||||
};
|
||||
|
||||
export const isS3Request = (headers: Record<string, string>): boolean => {
|
||||
const auth = headers.authorization || '';
|
||||
return auth.startsWith('AWS4-HMAC-SHA256');
|
||||
};
|
||||
|
||||
/**
|
||||
* Verifies that the actual body SHA-256 matches the `x-amz-content-sha256`
|
||||
* header from the original request.
|
||||
*
|
||||
* This MUST be called AFTER the body has been fully streamed and hashed,
|
||||
* as a second pass after `verifySignature` (which cannot hash a streaming
|
||||
* body without consuming it).
|
||||
*
|
||||
* @param bodySha256 - The SHA-256 hex digest of the actual body content.
|
||||
* @param headers - The original request headers.
|
||||
* @returns An error result on mismatch, or null if the check passes.
|
||||
*/
|
||||
export const verifyBodyHash = (
|
||||
bodySha256: string,
|
||||
headers: Record<string, string>,
|
||||
): SigV4Result | null => {
|
||||
const claimedHash = headers['x-amz-content-sha256'];
|
||||
// If the client sent UNSIGNED-PAYLOAD, skip verification
|
||||
if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) {
|
||||
return null;
|
||||
}
|
||||
if (claimedHash !== bodySha256) {
|
||||
return { isValid: false, credential: null, errorCode: 'BadDigest' };
|
||||
}
|
||||
return null;
|
||||
};
|
||||
@@ -1,52 +0,0 @@
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
export const S3_CORS_HEADERS: Record<string, string> = {
|
||||
'access-control-allow-origin': '*',
|
||||
'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS',
|
||||
'access-control-allow-headers': [
|
||||
'Authorization',
|
||||
'Content-Type',
|
||||
'Content-MD5',
|
||||
'Range',
|
||||
'If-Match',
|
||||
'If-None-Match',
|
||||
'If-Modified-Since',
|
||||
'If-Unmodified-Since',
|
||||
'X-Amz-*',
|
||||
'x-amz-*',
|
||||
].join(', '),
|
||||
'access-control-expose-headers': [
|
||||
'Accept-Ranges',
|
||||
'Content-Length',
|
||||
'Content-Range',
|
||||
'Content-Type',
|
||||
'ETag',
|
||||
'Last-Modified',
|
||||
'x-amz-id-2',
|
||||
'x-amz-request-id',
|
||||
].join(', '),
|
||||
'access-control-max-age': '86400',
|
||||
};
|
||||
|
||||
export const s3Headers = (
|
||||
requestId: string,
|
||||
extraHeaders: Record<string, string> = {},
|
||||
): Record<string, string> => ({
|
||||
...S3_CORS_HEADERS,
|
||||
server: 'AmazonS3',
|
||||
...(requestId
|
||||
? {
|
||||
'x-amz-request-id': requestId,
|
||||
'x-amz-id-2': `${requestId}+${nanoid(16)}`,
|
||||
}
|
||||
: {}),
|
||||
...extraHeaders,
|
||||
});
|
||||
|
||||
export const applyS3Headers = (headers: Headers, requestId: string): Headers => {
|
||||
const result = new Headers(headers);
|
||||
for (const [key, value] of Object.entries(s3Headers(requestId))) {
|
||||
result.set(key, value);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
@@ -1,133 +0,0 @@
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
import { applyS3Headers } from './headers';
|
||||
import { contentRange, type RangeParseResult } from './range';
|
||||
|
||||
export interface ObjectPartSource {
|
||||
telegramFileId: string;
|
||||
telegramUrl: string;
|
||||
sizeBytes: number;
|
||||
partNumber: number;
|
||||
storedSizeBytes?: number;
|
||||
compressionAlgorithm?: 'gzip' | null;
|
||||
}
|
||||
|
||||
export interface ObjectResponseInput {
|
||||
reqId: string;
|
||||
contentType: string;
|
||||
etag: string;
|
||||
lastModified: Date;
|
||||
totalSize: number;
|
||||
parts: ObjectPartSource[];
|
||||
range: RangeParseResult;
|
||||
}
|
||||
|
||||
interface PlannedPart {
|
||||
part: ObjectPartSource;
|
||||
relativeStart: number;
|
||||
relativeEnd: number;
|
||||
}
|
||||
|
||||
const baseHeaders = (input: ObjectResponseInput, contentLength: number): Headers => {
|
||||
const headers = new Headers({
|
||||
'content-type': input.contentType,
|
||||
'content-length': String(contentLength),
|
||||
etag: `"${input.etag}"`,
|
||||
'last-modified': input.lastModified.toUTCString(),
|
||||
'x-amz-request-id': input.reqId,
|
||||
'accept-ranges': 'bytes',
|
||||
'cache-control': 'public, max-age=31536000',
|
||||
});
|
||||
return headers;
|
||||
};
|
||||
|
||||
const planParts = (parts: ObjectPartSource[], start: number, end: number): PlannedPart[] => {
|
||||
const planned: PlannedPart[] = [];
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
const partStart = offset;
|
||||
const partEnd = offset + part.sizeBytes - 1;
|
||||
offset += part.sizeBytes;
|
||||
if (end < partStart || start > partEnd) continue;
|
||||
planned.push({
|
||||
part,
|
||||
relativeStart: Math.max(start, partStart) - partStart,
|
||||
relativeEnd: Math.min(end, partEnd) - partStart,
|
||||
});
|
||||
}
|
||||
return planned;
|
||||
};
|
||||
|
||||
const streamFromBytes = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
|
||||
new Response(bytes).body!;
|
||||
|
||||
const TELEGRAM_FETCH_TIMEOUT_MS = 30_000;
|
||||
|
||||
const fetchWholePartBytes = async (telegramUrl: string): Promise<Uint8Array> => {
|
||||
const res = await fetch(telegramUrl, { signal: AbortSignal.timeout(TELEGRAM_FETCH_TIMEOUT_MS) });
|
||||
if (!res.ok) throw new Error(`Telegram fetch failed: ${res.status}`);
|
||||
return new Uint8Array(await res.arrayBuffer());
|
||||
};
|
||||
|
||||
const fetchPartBody = async (planned: PlannedPart): Promise<ReadableStream<Uint8Array>> => {
|
||||
const wantsWholePart =
|
||||
planned.relativeStart === 0 && planned.relativeEnd === planned.part.sizeBytes - 1;
|
||||
|
||||
if (planned.part.compressionAlgorithm === 'gzip') {
|
||||
const storedBytes = await fetchWholePartBytes(planned.part.telegramUrl);
|
||||
const bytes = gunzipSync(storedBytes);
|
||||
return streamFromBytes(bytes.subarray(planned.relativeStart, planned.relativeEnd + 1));
|
||||
}
|
||||
|
||||
const rangeHeader = `bytes=${planned.relativeStart}-${planned.relativeEnd}`;
|
||||
const fetchOpts: RequestInit = { signal: AbortSignal.timeout(TELEGRAM_FETCH_TIMEOUT_MS) };
|
||||
if (!wantsWholePart) {
|
||||
fetchOpts.headers = { range: rangeHeader };
|
||||
}
|
||||
const res = await fetch(planned.part.telegramUrl, fetchOpts);
|
||||
if (!res.ok) throw new Error(`Telegram fetch failed: ${res.status}`);
|
||||
if (wantsWholePart || res.status === 206) return res.body!;
|
||||
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
return streamFromBytes(bytes.slice(planned.relativeStart, planned.relativeEnd + 1));
|
||||
};
|
||||
|
||||
const concatPartStreams = (plannedParts: PlannedPart[]): ReadableStream<Uint8Array> =>
|
||||
new ReadableStream<Uint8Array>({
|
||||
async start(controller) {
|
||||
try {
|
||||
for (const planned of plannedParts) {
|
||||
const stream = await fetchPartBody(planned);
|
||||
const reader = stream.getReader();
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
if (value) controller.enqueue(value);
|
||||
}
|
||||
}
|
||||
controller.close();
|
||||
} catch (error) {
|
||||
controller.error(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
export const createGetObjectResponse = async (input: ObjectResponseInput): Promise<Response> => {
|
||||
if (input.range.type === 'invalid') {
|
||||
throw new Error('createGetObjectResponse received invalid range');
|
||||
}
|
||||
|
||||
const start = input.range.type === 'valid' ? input.range.start : 0;
|
||||
const end = input.range.type === 'valid' ? input.range.end : input.totalSize - 1;
|
||||
const plannedParts = planParts(input.parts, start, end);
|
||||
const contentLength = end >= start ? end - start + 1 : 0;
|
||||
const headers = applyS3Headers(baseHeaders(input, contentLength), input.reqId);
|
||||
|
||||
if (input.range.type === 'valid') {
|
||||
headers.set('content-range', contentRange(start, end, input.totalSize));
|
||||
}
|
||||
|
||||
return new Response(concatPartStreams(plannedParts), {
|
||||
status: input.range.type === 'valid' ? 206 : 200,
|
||||
headers,
|
||||
});
|
||||
};
|
||||
@@ -1,46 +0,0 @@
|
||||
export type RangeParseResult =
|
||||
| { type: 'none' }
|
||||
| { type: 'valid'; start: number; end: number }
|
||||
| { type: 'invalid' };
|
||||
|
||||
const DECIMAL = /^\d+$/;
|
||||
|
||||
export const parseRangeHeader = (rangeHeader: string | null, size: number): RangeParseResult => {
|
||||
if (!rangeHeader) return { type: 'none' };
|
||||
if (!Number.isSafeInteger(size) || size < 0) return { type: 'invalid' };
|
||||
if (!rangeHeader.startsWith('bytes=')) return { type: 'invalid' };
|
||||
|
||||
const spec = rangeHeader.slice('bytes='.length).trim();
|
||||
if (spec.includes(',')) return { type: 'invalid' };
|
||||
|
||||
const dash = spec.indexOf('-');
|
||||
if (dash === -1) return { type: 'invalid' };
|
||||
|
||||
const startText = spec.slice(0, dash).trim();
|
||||
const endText = spec.slice(dash + 1).trim();
|
||||
if (!startText && !endText) return { type: 'invalid' };
|
||||
if (size === 0) return { type: 'invalid' };
|
||||
|
||||
if (!startText) {
|
||||
if (!DECIMAL.test(endText)) return { type: 'invalid' };
|
||||
const suffixLength = Number.parseInt(endText, 10);
|
||||
if (suffixLength <= 0) return { type: 'invalid' };
|
||||
return { type: 'valid', start: Math.max(size - suffixLength, 0), end: size - 1 };
|
||||
}
|
||||
|
||||
if (!DECIMAL.test(startText)) return { type: 'invalid' };
|
||||
const start = Number.parseInt(startText, 10);
|
||||
if (start >= size) return { type: 'invalid' };
|
||||
|
||||
if (!endText) return { type: 'valid', start, end: size - 1 };
|
||||
if (!DECIMAL.test(endText)) return { type: 'invalid' };
|
||||
|
||||
const requestedEnd = Number.parseInt(endText, 10);
|
||||
if (requestedEnd < start) return { type: 'invalid' };
|
||||
return { type: 'valid', start, end: Math.min(requestedEnd, size - 1) };
|
||||
};
|
||||
|
||||
export const contentRange = (start: number, end: number, size: number): string =>
|
||||
`bytes ${start}-${end}/${size}`;
|
||||
|
||||
export const unsatisfiedContentRange = (size: number): string => `bytes */${size}`;
|
||||
@@ -1,27 +0,0 @@
|
||||
const stripPort = (host: string): string => {
|
||||
// Handle IPv6: [::1]:8080 -> [::1]
|
||||
if (host.startsWith('[')) {
|
||||
const closeBracket = host.indexOf(']');
|
||||
return host.slice(0, closeBracket + 1).toLowerCase();
|
||||
}
|
||||
return host.split(':')[0].toLowerCase().replace(/\.$/, '');
|
||||
};
|
||||
|
||||
const isValidBucketLabel = (bucket: string): boolean =>
|
||||
/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) &&
|
||||
!bucket.includes('..') &&
|
||||
!bucket.includes('.-') &&
|
||||
!bucket.includes('-.');
|
||||
|
||||
export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => {
|
||||
const normalizedHost = stripPort(host);
|
||||
for (const domain of domains) {
|
||||
const normalizedDomain = stripPort(domain);
|
||||
if (!normalizedDomain || normalizedHost === normalizedDomain) continue;
|
||||
if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue;
|
||||
|
||||
const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1));
|
||||
return isValidBucketLabel(bucket) ? bucket : null;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
@@ -1,311 +0,0 @@
|
||||
import { s3Headers } from './headers';
|
||||
|
||||
const escapeXml = (str: string): string =>
|
||||
str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
|
||||
const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||
|
||||
const encodeKey = (value: string, encodingType: string | null = null): string =>
|
||||
encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value);
|
||||
|
||||
// ─────── Bucket operations ───────
|
||||
|
||||
export const listBucketsXml = (
|
||||
buckets: { name: string; createdAt: Date }[],
|
||||
_requestId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Buckets>
|
||||
${buckets
|
||||
.map(
|
||||
(b) => `<Bucket>
|
||||
<Name>${escapeXml(b.name)}</Name>
|
||||
<CreationDate>${isoDate(b.createdAt)}</CreationDate>
|
||||
</Bucket>`,
|
||||
)
|
||||
.join('')}
|
||||
</Buckets>
|
||||
</ListAllMyBucketsResult>`;
|
||||
|
||||
export const bucketVersioningConfigurationXml =
|
||||
(): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`;
|
||||
|
||||
// ─────── Object listing ───────
|
||||
|
||||
export const listBucketResultXml = (
|
||||
bucketName: string,
|
||||
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
|
||||
prefixes: string[],
|
||||
isTruncated: boolean,
|
||||
marker: string | null,
|
||||
maxKeys: number,
|
||||
prefix: string,
|
||||
delimiter: string | null,
|
||||
nextMarker: string | null,
|
||||
_requestId: string,
|
||||
encodingType: string | null = null,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>${escapeXml(bucketName)}</Name>
|
||||
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
|
||||
<Marker>${encodeKey(marker || '', encodingType)}</Marker>
|
||||
<MaxKeys>${maxKeys}</MaxKeys>
|
||||
<Delimiter>${encodeKey(delimiter || '', encodingType)}</Delimiter>
|
||||
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
|
||||
<IsTruncated>${isTruncated}</IsTruncated>
|
||||
${objects
|
||||
.map(
|
||||
(o) => `<Contents>
|
||||
<Key>${encodeKey(o.key, encodingType)}</Key>
|
||||
<LastModified>${isoDate(o.lastModified)}</LastModified>
|
||||
<ETag>"${o.etag}"</ETag>
|
||||
<Size>${o.sizeBytes}</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>`,
|
||||
)
|
||||
.join('')}
|
||||
${prefixes
|
||||
.map(
|
||||
(p) => `<CommonPrefixes>
|
||||
<Prefix>${encodeKey(p, encodingType)}</Prefix>
|
||||
</CommonPrefixes>`,
|
||||
)
|
||||
.join('')}
|
||||
${nextMarker ? `<NextMarker>${encodeKey(nextMarker, encodingType)}</NextMarker>` : ''}
|
||||
</ListBucketResult>`;
|
||||
|
||||
export const listBucketV2ResultXml = (
|
||||
bucketName: string,
|
||||
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
|
||||
prefixes: string[],
|
||||
isTruncated: boolean,
|
||||
maxKeys: number,
|
||||
prefix: string,
|
||||
delimiter: string | null,
|
||||
continuationToken: string | null,
|
||||
nextContinuationToken: string | null,
|
||||
keyCount: number,
|
||||
_requestId: string,
|
||||
encodingType: string | null = null,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>${escapeXml(bucketName)}</Name>
|
||||
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
|
||||
<MaxKeys>${maxKeys}</MaxKeys>
|
||||
<KeyCount>${keyCount}</KeyCount>
|
||||
${delimiter ? `<Delimiter>${encodeKey(delimiter, encodingType)}</Delimiter>` : ''}
|
||||
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
|
||||
${continuationToken ? `<ContinuationToken>${encodeKey(continuationToken, encodingType)}</ContinuationToken>` : ''}
|
||||
<IsTruncated>${isTruncated}</IsTruncated>
|
||||
${objects
|
||||
.map(
|
||||
(o) => `<Contents>
|
||||
<Key>${encodeKey(o.key, encodingType)}</Key>
|
||||
<LastModified>${isoDate(o.lastModified)}</LastModified>
|
||||
<ETag>"${o.etag}"</ETag>
|
||||
<Size>${o.sizeBytes}</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>`,
|
||||
)
|
||||
.join('')}
|
||||
${prefixes
|
||||
.map(
|
||||
(p) => `<CommonPrefixes>
|
||||
<Prefix>${encodeKey(p, encodingType)}</Prefix>
|
||||
</CommonPrefixes>`,
|
||||
)
|
||||
.join('')}
|
||||
${nextContinuationToken ? `<NextContinuationToken>${encodeKey(nextContinuationToken, encodingType)}</NextContinuationToken>` : ''}
|
||||
</ListBucketResultV2>`;
|
||||
|
||||
// ─────── Multipart ───────
|
||||
|
||||
export const initiateMultipartUploadXml = (
|
||||
bucketName: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||
<Key>${escapeXml(key)}</Key>
|
||||
<UploadId>${uploadId}</UploadId>
|
||||
</InitiateMultipartUploadResult>`;
|
||||
|
||||
export const listPartsXml = (
|
||||
bucketName: string,
|
||||
key: string,
|
||||
uploadId: string,
|
||||
parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[],
|
||||
maxParts: number,
|
||||
isTruncated: boolean,
|
||||
_requestId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||
<Key>${escapeXml(key)}</Key>
|
||||
<UploadId>${uploadId}</UploadId>
|
||||
<MaxParts>${maxParts}</MaxParts>
|
||||
<IsTruncated>${isTruncated}</IsTruncated>
|
||||
${parts
|
||||
.map(
|
||||
(p) => `<Part>
|
||||
<PartNumber>${p.partNumber}</PartNumber>
|
||||
<LastModified>${isoDate(p.createdAt)}</LastModified>
|
||||
<ETag>"${p.etag}"</ETag>
|
||||
<Size>${p.sizeBytes}</Size>
|
||||
</Part>`,
|
||||
)
|
||||
.join('')}
|
||||
</ListPartsResult>`;
|
||||
|
||||
export const listMultipartUploadsXml = (
|
||||
bucketName: string,
|
||||
uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[],
|
||||
maxUploads: number,
|
||||
isTruncated: boolean,
|
||||
nextKeyMarker: string | null,
|
||||
_requestId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||
<KeyMarker></KeyMarker>
|
||||
<UploadIdMarker></UploadIdMarker>
|
||||
${nextKeyMarker ? `<NextKeyMarker>${escapeXml(nextKeyMarker)}</NextKeyMarker>` : ''}
|
||||
<MaxUploads>${maxUploads}</MaxUploads>
|
||||
<IsTruncated>${isTruncated}</IsTruncated>
|
||||
${uploads
|
||||
.map(
|
||||
(u) => `<Upload>
|
||||
<Key>${escapeXml(u.key)}</Key>
|
||||
<UploadId>${u.uploadId}</UploadId>
|
||||
<Initiator><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Initiator>
|
||||
<Owner><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Owner>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
<Initiated>${isoDate(u.initiatedAt)}</Initiated>
|
||||
</Upload>`,
|
||||
)
|
||||
.join('')}
|
||||
</ListMultipartUploadsResult>`;
|
||||
|
||||
export const completeMultipartUploadXml = (
|
||||
bucketName: string,
|
||||
key: string,
|
||||
etag: string,
|
||||
location: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Location>${escapeXml(location)}</Location>
|
||||
<Bucket>${escapeXml(bucketName)}</Bucket>
|
||||
<Key>${escapeXml(key)}</Key>
|
||||
<ETag>"${etag}"</ETag>
|
||||
</CompleteMultipartUploadResult>`;
|
||||
|
||||
// ─────── Delete result ───────
|
||||
|
||||
export const deleteResultXml = (
|
||||
deleted: string[],
|
||||
errors: { key: string; code: string; message: string }[],
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
${deleted
|
||||
.map(
|
||||
(key) => `<Deleted>
|
||||
<Key>${escapeXml(key)}</Key>
|
||||
</Deleted>`,
|
||||
)
|
||||
.join('')}
|
||||
${errors
|
||||
.map(
|
||||
(e) => `<Error>
|
||||
<Key>${escapeXml(e.key)}</Key>
|
||||
<Code>${e.code}</Code>
|
||||
<Message>${escapeXml(e.message)}</Message>
|
||||
</Error>`,
|
||||
)
|
||||
.join('')}
|
||||
</DeleteResult>`;
|
||||
|
||||
// ─────── Copy ───────
|
||||
|
||||
export const copyObjectResultXml = (
|
||||
etag: string,
|
||||
lastModified: Date,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<ETag>"${etag}"</ETag>
|
||||
<LastModified>${isoDate(lastModified)}</LastModified>
|
||||
</CopyObjectResult>`;
|
||||
|
||||
// ─────── Error ───────
|
||||
|
||||
export const s3ErrorXml = (
|
||||
code: string,
|
||||
message: string,
|
||||
resource: string,
|
||||
requestId: string,
|
||||
): string => `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Error>
|
||||
<Code>${code}</Code>
|
||||
<Message>${escapeXml(message)}</Message>
|
||||
<Resource>${escapeXml(resource)}</Resource>
|
||||
<RequestId>${requestId}</RequestId>
|
||||
<HostId>${requestId}</HostId>
|
||||
</Error>`;
|
||||
|
||||
export const s3ErrorResponse = (
|
||||
code: string,
|
||||
message: string,
|
||||
resource: string,
|
||||
status: number,
|
||||
requestId: string = '',
|
||||
extraHeaders: Record<string, string> = {},
|
||||
): Response =>
|
||||
new Response(s3ErrorXml(code, message, resource, requestId), {
|
||||
status,
|
||||
headers: s3Headers(requestId, {
|
||||
'content-type': 'application/xml',
|
||||
...extraHeaders,
|
||||
}),
|
||||
});
|
||||
|
||||
// ─────── DeleteObjects XML parser ───────
|
||||
|
||||
export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => {
|
||||
// H9: Use non-greedy match to handle keys containing < character
|
||||
const keys = Array.from(body.matchAll(/<Key>([\s\S]*?)<\/Key>/g), (match) => match[1]);
|
||||
// Handle whitespace inside <Quiet> element + namespace prefix support
|
||||
const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body);
|
||||
return { keys, quiet };
|
||||
};
|
||||
|
||||
// ─────── CompleteMultipartUpload XML parser ───────
|
||||
|
||||
export interface CompletePart {
|
||||
partNumber: number;
|
||||
etag: string;
|
||||
}
|
||||
|
||||
export const parseCompleteMultipartBody = (body: string): CompletePart[] => {
|
||||
const parts: CompletePart[] = [];
|
||||
const partRegex = /<Part>[\s\S]*?<\/Part>/g;
|
||||
const partMatch = body.match(partRegex) || [];
|
||||
|
||||
for (const partXml of partMatch) {
|
||||
const numMatch = partXml.match(/<PartNumber>(\d+)<\/PartNumber>/);
|
||||
const etagMatch = partXml.match(/<ETag>"?([^"<\s]+)"?<\/ETag>/);
|
||||
if (numMatch && etagMatch) {
|
||||
parts.push({
|
||||
partNumber: Number.parseInt(numMatch[1], 10),
|
||||
etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return parts;
|
||||
};
|
||||
@@ -1,152 +0,0 @@
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { db, files as fileSchema } from '../db';
|
||||
import type { NewFile } from '../db/schema';
|
||||
import { config } from '../env';
|
||||
import { botPool } from '../infrastructure/telegram/bot-pool';
|
||||
import { cleanupTempFile } from './file';
|
||||
import { createZip, type ZipEntry } from './zip';
|
||||
|
||||
export type PreparedUpload = {
|
||||
tempPath: string;
|
||||
fileHash: string;
|
||||
sizeBytes: number;
|
||||
signatureBuffer: Buffer;
|
||||
};
|
||||
|
||||
export type UploadedFile = NewFile & {
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
};
|
||||
|
||||
export type BatchUploadItem = {
|
||||
prepared: PreparedUpload;
|
||||
fileName: string;
|
||||
mimeType: string;
|
||||
fileType: string;
|
||||
};
|
||||
|
||||
type PendingUpload = BatchUploadItem & {
|
||||
resolve: (file: UploadedFile) => void;
|
||||
reject: (error: unknown) => void;
|
||||
};
|
||||
|
||||
const BATCH_WINDOW_MS = 2000;
|
||||
|
||||
let pendingUploads: PendingUpload[] = [];
|
||||
let flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
const buildUploadedFile = (
|
||||
item: BatchUploadItem,
|
||||
entry: ZipEntry,
|
||||
archive: {
|
||||
telegramFileId: string;
|
||||
telegramFileUniqueId: string;
|
||||
storageMessageId: number;
|
||||
fileName: string;
|
||||
sizeBytes: number;
|
||||
},
|
||||
): UploadedFile => ({
|
||||
publicId: nanoid(),
|
||||
telegramFileId: archive.telegramFileId,
|
||||
telegramFileUniqueId: archive.telegramFileUniqueId,
|
||||
storageChatId: config.storageChatId,
|
||||
storageMessageId: archive.storageMessageId,
|
||||
fileName: item.fileName,
|
||||
mimeType: item.mimeType || 'application/octet-stream',
|
||||
sizeBytes: item.prepared.sizeBytes,
|
||||
fileType: item.fileType,
|
||||
uploaderId: 0,
|
||||
fileHash: item.prepared.fileHash,
|
||||
archiveTelegramFileId: archive.telegramFileId,
|
||||
archiveStorageMessageId: archive.storageMessageId,
|
||||
archiveFileName: archive.fileName,
|
||||
archiveEntryName: entry.entryName,
|
||||
archiveMimeType: 'application/zip',
|
||||
archiveSizeBytes: archive.sizeBytes,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
|
||||
const flushUploads = async (): Promise<void> => {
|
||||
if (flushTimer) {
|
||||
clearTimeout(flushTimer);
|
||||
flushTimer = null;
|
||||
}
|
||||
|
||||
const batch = pendingUploads;
|
||||
pendingUploads = [];
|
||||
if (batch.length === 0) return;
|
||||
|
||||
let zipTempPath: string | null = null;
|
||||
|
||||
try {
|
||||
const zip = await createZip(
|
||||
batch.map((item) => ({ tempPath: item.prepared.tempPath, fileName: item.fileName })),
|
||||
);
|
||||
zipTempPath = zip.tempPath;
|
||||
const archiveFileName = `filedrop-${nanoid()}.zip`;
|
||||
const archiveResult = await botPool.forwardToStorage(
|
||||
createReadStream(zip.tempPath),
|
||||
archiveFileName,
|
||||
'document',
|
||||
);
|
||||
|
||||
const uploadedFiles = batch.map((item, index) =>
|
||||
buildUploadedFile(item, zip.entries[index], {
|
||||
telegramFileId: archiveResult.telegramFileId,
|
||||
telegramFileUniqueId: archiveResult.telegramFileUniqueId,
|
||||
storageMessageId: archiveResult.storageMessageId,
|
||||
fileName: archiveFileName,
|
||||
sizeBytes: zip.sizeBytes,
|
||||
}),
|
||||
);
|
||||
|
||||
await db.insert(fileSchema).values(uploadedFiles);
|
||||
|
||||
for (let i = 0; i < batch.length; i++) {
|
||||
batch[i].resolve(uploadedFiles[i]);
|
||||
}
|
||||
} catch (error) {
|
||||
for (const item of batch) {
|
||||
item.reject(error);
|
||||
}
|
||||
} finally {
|
||||
await Promise.all(batch.map((item) => cleanupTempFile(item.prepared.tempPath)));
|
||||
if (zipTempPath) await cleanupTempFile(zipTempPath);
|
||||
// Reschedule timer if new items arrived during async processing
|
||||
if (pendingUploads.length > 0 && !flushTimer) {
|
||||
flushTimer = setTimeout(() => {
|
||||
void flushUploads();
|
||||
}, BATCH_WINDOW_MS);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const getPendingSize = (): number =>
|
||||
pendingUploads.reduce((total, item) => total + item.prepared.sizeBytes, 0);
|
||||
|
||||
export const enqueuePreparedUpload = (item: BatchUploadItem): Promise<UploadedFile> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
pendingUploads.push({ ...item, resolve, reject });
|
||||
|
||||
if (!flushTimer) {
|
||||
flushTimer = setTimeout(() => {
|
||||
void flushUploads();
|
||||
}, BATCH_WINDOW_MS);
|
||||
}
|
||||
|
||||
if (
|
||||
pendingUploads.length >= config.batchMaxItems ||
|
||||
getPendingSize() >= config.batchMaxSizeBytes
|
||||
) {
|
||||
void flushUploads();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
export const flushPendingUploads = async (): Promise<void> => {
|
||||
await flushUploads();
|
||||
};
|
||||
|
||||
export const getPendingUploadCount = (): number => pendingUploads.length;
|
||||
@@ -1,292 +0,0 @@
|
||||
import { once } from 'node:events';
|
||||
import { createReadStream, createWriteStream } from 'node:fs';
|
||||
import { open, stat, unlink } from 'node:fs/promises';
|
||||
import { basename } from 'node:path';
|
||||
import { finished } from 'node:stream/promises';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
export type ZipInputFile = {
|
||||
tempPath: string;
|
||||
fileName: string;
|
||||
};
|
||||
|
||||
export type ZipEntry = {
|
||||
fileName: string;
|
||||
entryName: string;
|
||||
crc32: number;
|
||||
compressedSize: number;
|
||||
uncompressedSize: number;
|
||||
localHeaderOffset: number;
|
||||
};
|
||||
|
||||
export type CreatedZip = {
|
||||
tempPath: string;
|
||||
sizeBytes: number;
|
||||
fileHash: string;
|
||||
entries: ZipEntry[];
|
||||
};
|
||||
|
||||
const CRC32_TABLE = new Uint32Array(256).map((_, index) => {
|
||||
let value = index;
|
||||
for (let bit = 0; bit < 8; bit++) {
|
||||
value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
|
||||
}
|
||||
return value >>> 0;
|
||||
});
|
||||
|
||||
const updateCrc32 = (crc: number, chunk: Buffer): number => {
|
||||
let value = crc;
|
||||
for (const byte of chunk) {
|
||||
value = CRC32_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8);
|
||||
}
|
||||
return value >>> 0;
|
||||
};
|
||||
|
||||
const dosDateTime = (date = new Date()): { date: number; time: number } => {
|
||||
const year = Math.max(date.getFullYear(), 1980);
|
||||
return {
|
||||
time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2),
|
||||
date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
|
||||
};
|
||||
};
|
||||
|
||||
const writeUInt16 = (value: number): Buffer<ArrayBuffer> => {
|
||||
const buffer = Buffer.allocUnsafe(2);
|
||||
buffer.writeUInt16LE(value & 0xffff, 0);
|
||||
return buffer;
|
||||
};
|
||||
|
||||
const writeUInt32 = (value: number): Buffer<ArrayBuffer> => {
|
||||
const buffer = Buffer.allocUnsafe(4);
|
||||
buffer.writeUInt32LE(value >>> 0, 0);
|
||||
return buffer;
|
||||
};
|
||||
|
||||
const writeChunk = async (
|
||||
writer: ReturnType<typeof createWriteStream>,
|
||||
chunk: Buffer,
|
||||
): Promise<void> => {
|
||||
if (!writer.write(chunk)) {
|
||||
await once(writer, 'drain');
|
||||
}
|
||||
};
|
||||
|
||||
const finishWriter = async (writer: ReturnType<typeof createWriteStream>): Promise<void> => {
|
||||
writer.end();
|
||||
await finished(writer);
|
||||
};
|
||||
|
||||
export const sanitizeZipEntryName = (fileName: string, usedNames = new Set<string>()): string => {
|
||||
const cleaned = basename(fileName)
|
||||
.replace(/[\\/]+/g, '_')
|
||||
.replace(/\.\.+/g, '.')
|
||||
.trim();
|
||||
const fallback = cleaned && cleaned !== '.' && cleaned !== '..' ? cleaned : 'file';
|
||||
const dotIndex = fallback.lastIndexOf('.');
|
||||
const baseName = dotIndex > 0 ? fallback.slice(0, dotIndex) : fallback;
|
||||
const extension = dotIndex > 0 ? fallback.slice(dotIndex) : '';
|
||||
let candidate = fallback;
|
||||
let counter = 1;
|
||||
|
||||
while (usedNames.has(candidate)) {
|
||||
candidate = `${baseName}-${counter}${extension}`;
|
||||
counter++;
|
||||
}
|
||||
|
||||
usedNames.add(candidate);
|
||||
return candidate;
|
||||
};
|
||||
|
||||
const calculateFileCrc32 = async (tempPath: string): Promise<number> => {
|
||||
let crc = 0xffffffff;
|
||||
const reader = createReadStream(tempPath);
|
||||
for await (const chunk of reader) {
|
||||
crc = updateCrc32(crc, chunk as Buffer);
|
||||
}
|
||||
return (crc ^ 0xffffffff) >>> 0;
|
||||
};
|
||||
|
||||
export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
|
||||
const tempPath = `/tmp/filedrop-${nanoid()}.zip`;
|
||||
const writer = createWriteStream(tempPath);
|
||||
const hasher = new Bun.CryptoHasher('sha256');
|
||||
const entries: ZipEntry[] = [];
|
||||
const usedNames = new Set<string>();
|
||||
let offset = 0;
|
||||
|
||||
const writeHashed = async (chunk: Buffer): Promise<void> => {
|
||||
hasher.update(chunk);
|
||||
await writeChunk(writer, chunk);
|
||||
offset += chunk.byteLength;
|
||||
};
|
||||
|
||||
try {
|
||||
for (const file of files) {
|
||||
const entryName = sanitizeZipEntryName(file.fileName, usedNames);
|
||||
const nameBuffer = Buffer.from(entryName);
|
||||
const fileStats = await stat(file.tempPath);
|
||||
const { date, time } = dosDateTime();
|
||||
const localHeaderOffset = offset;
|
||||
const crc32 = await calculateFileCrc32(file.tempPath);
|
||||
|
||||
const localHeader = Buffer.concat([
|
||||
writeUInt32(0x04034b50),
|
||||
writeUInt16(20),
|
||||
writeUInt16(0),
|
||||
writeUInt16(0),
|
||||
writeUInt16(time),
|
||||
writeUInt16(date),
|
||||
writeUInt32(crc32),
|
||||
writeUInt32(fileStats.size),
|
||||
writeUInt32(fileStats.size),
|
||||
writeUInt16(nameBuffer.byteLength),
|
||||
writeUInt16(0),
|
||||
nameBuffer,
|
||||
]);
|
||||
|
||||
await writeHashed(localHeader);
|
||||
const reader = createReadStream(file.tempPath);
|
||||
for await (const chunk of reader) {
|
||||
await writeHashed(chunk as Buffer);
|
||||
}
|
||||
|
||||
entries.push({
|
||||
fileName: file.fileName,
|
||||
entryName,
|
||||
crc32,
|
||||
compressedSize: fileStats.size,
|
||||
uncompressedSize: fileStats.size,
|
||||
localHeaderOffset,
|
||||
});
|
||||
}
|
||||
|
||||
const centralDirectoryOffset = offset;
|
||||
for (const entry of entries) {
|
||||
const nameBuffer = Buffer.from(entry.entryName);
|
||||
const { date, time } = dosDateTime();
|
||||
await writeHashed(
|
||||
Buffer.concat([
|
||||
writeUInt32(0x02014b50),
|
||||
writeUInt16(20),
|
||||
writeUInt16(20),
|
||||
writeUInt16(0),
|
||||
writeUInt16(0),
|
||||
writeUInt16(time),
|
||||
writeUInt16(date),
|
||||
writeUInt32(entry.crc32),
|
||||
writeUInt32(entry.compressedSize),
|
||||
writeUInt32(entry.uncompressedSize),
|
||||
writeUInt16(nameBuffer.byteLength),
|
||||
writeUInt16(0),
|
||||
writeUInt16(0),
|
||||
writeUInt16(0),
|
||||
writeUInt16(0),
|
||||
writeUInt32(0),
|
||||
writeUInt32(entry.localHeaderOffset),
|
||||
nameBuffer,
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
const centralDirectorySize = offset - centralDirectoryOffset;
|
||||
await writeHashed(
|
||||
Buffer.concat([
|
||||
writeUInt32(0x06054b50),
|
||||
writeUInt16(0),
|
||||
writeUInt16(0),
|
||||
writeUInt16(entries.length),
|
||||
writeUInt16(entries.length),
|
||||
writeUInt32(centralDirectorySize),
|
||||
writeUInt32(centralDirectoryOffset),
|
||||
writeUInt16(0),
|
||||
]),
|
||||
);
|
||||
|
||||
await finishWriter(writer);
|
||||
|
||||
return {
|
||||
tempPath,
|
||||
sizeBytes: offset,
|
||||
fileHash: hasher.digest('hex'),
|
||||
entries,
|
||||
};
|
||||
} catch (error) {
|
||||
writer.destroy();
|
||||
await unlink(tempPath).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const extractZipEntry = async (
|
||||
zipBuffer: Buffer,
|
||||
entryName: string,
|
||||
): Promise<Buffer | null> => {
|
||||
let offset = 0;
|
||||
|
||||
while (offset + 30 <= zipBuffer.byteLength) {
|
||||
const signature = zipBuffer.readUInt32LE(offset);
|
||||
if (signature !== 0x04034b50) break;
|
||||
|
||||
const compressionMethod = zipBuffer.readUInt16LE(offset + 8);
|
||||
const compressedSize = zipBuffer.readUInt32LE(offset + 18);
|
||||
const fileNameLength = zipBuffer.readUInt16LE(offset + 26);
|
||||
const extraLength = zipBuffer.readUInt16LE(offset + 28);
|
||||
const nameStart = offset + 30;
|
||||
const nameEnd = nameStart + fileNameLength;
|
||||
const dataStart = nameEnd + extraLength;
|
||||
const dataEnd = dataStart + compressedSize;
|
||||
const currentName = zipBuffer.subarray(nameStart, nameEnd).toString();
|
||||
|
||||
if (currentName === entryName) {
|
||||
if (compressionMethod !== 0) return null;
|
||||
return zipBuffer.subarray(dataStart, dataEnd);
|
||||
}
|
||||
|
||||
offset = dataEnd;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
export type LocatedZipEntry = {
|
||||
start: number;
|
||||
length: number;
|
||||
};
|
||||
|
||||
export const locateZipEntry = async (
|
||||
zipPath: string,
|
||||
entryName: string,
|
||||
): Promise<LocatedZipEntry | null> => {
|
||||
const handle = await open(zipPath, 'r');
|
||||
let offset = 0;
|
||||
|
||||
try {
|
||||
const header = Buffer.alloc(30);
|
||||
|
||||
while (true) {
|
||||
const { bytesRead } = await handle.read(header, 0, header.byteLength, offset);
|
||||
if (bytesRead < header.byteLength) return null;
|
||||
|
||||
const signature = header.readUInt32LE(0);
|
||||
if (signature !== 0x04034b50) return null;
|
||||
|
||||
const compressionMethod = header.readUInt16LE(8);
|
||||
const compressedSize = header.readUInt32LE(18);
|
||||
const fileNameLength = header.readUInt16LE(26);
|
||||
const extraLength = header.readUInt16LE(28);
|
||||
const nameBuffer = Buffer.alloc(fileNameLength);
|
||||
const nameOffset = offset + 30;
|
||||
await handle.read(nameBuffer, 0, fileNameLength, nameOffset);
|
||||
|
||||
const dataStart = nameOffset + fileNameLength + extraLength;
|
||||
if (nameBuffer.toString() === entryName) {
|
||||
if (compressionMethod !== 0) return null;
|
||||
return { start: dataStart, length: compressedSize };
|
||||
}
|
||||
|
||||
offset = dataStart + compressedSize;
|
||||
}
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user