refactor: full DDD + Clean Architecture refactor
Deploy FileDrop / deploy (push) Successful in 48s

- Hapus src/utils/ (17 files) + src/db/ (8 files) dead code
- Absorb 7 re-export stubs → real impl di lokasi DDD
- Buat src/infrastructure/di.ts (DI container)
- Rewrite 5 controllers pakai repository/DI
- Fix shared/utils imports, env.ts, routes, index.ts
- Update package.json build path migrate
- Lint clean, build clean
This commit is contained in:
Claude
2026-07-29 15:41:50 +07:00
parent 3580001b8a
commit 4c216b1d9f
44 changed files with 1223 additions and 3364 deletions
+1 -1
View File
@@ -5,7 +5,7 @@
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "bun --hot src/index.ts", "dev": "bun --hot src/index.ts",
"build": "bun build src/index.ts --target=bun --outfile=dist/index.js && bun build src/db/migrate.ts --target=bun --outfile=dist/migrate.js", "build": "bun build src/index.ts --target=bun --outfile=dist/index.js && bun build src/infrastructure/persistence/drizzle/migrate.ts --target=bun --outfile=dist/migrate.js",
"start": "NODE_ENV=production bun dist/index.js", "start": "NODE_ENV=production bun dist/index.js",
"db:migrate": "bun dist/migrate.js", "db:migrate": "bun dist/migrate.js",
"test": "bun test --preload ./test/helpers/setup-env.ts test/rateLimit.test.ts && bun test --preload ./test/helpers/setup-env.ts test/file.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegram.test.ts && bun test --preload ./test/helpers/setup-env.ts test/upload.test.ts && bun test --preload ./test/helpers/setup-env.ts test/files.test.ts && bun test --preload ./test/helpers/setup-env.ts test/health.test.ts && bun test --preload ./test/helpers/setup-env.ts test/db.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bot.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bootstrap.test.ts && bun test --preload ./test/helpers/setup-env.ts test/swagger.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth-routes.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-operations.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-bucket-config.test.ts && bun test --preload ./test/helpers/setup-env.ts test/web-api.test.ts && bun test --preload ./test/helpers/setup-env.ts test/env.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegramQueue.test.ts", "test": "bun test --preload ./test/helpers/setup-env.ts test/rateLimit.test.ts && bun test --preload ./test/helpers/setup-env.ts test/file.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegram.test.ts && bun test --preload ./test/helpers/setup-env.ts test/upload.test.ts && bun test --preload ./test/helpers/setup-env.ts test/files.test.ts && bun test --preload ./test/helpers/setup-env.ts test/health.test.ts && bun test --preload ./test/helpers/setup-env.ts test/db.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bot.test.ts && bun test --preload ./test/helpers/setup-env.ts test/bootstrap.test.ts && bun test --preload ./test/helpers/setup-env.ts test/swagger.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/auth-routes.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-auth.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-operations.test.ts && bun test --preload ./test/helpers/setup-env.ts test/s3-bucket-config.test.ts && bun test --preload ./test/helpers/setup-env.ts test/web-api.test.ts && bun test --preload ./test/helpers/setup-env.ts test/env.test.ts && bun test --preload ./test/helpers/setup-env.ts test/telegramQueue.test.ts",
-81
View File
@@ -1,81 +0,0 @@
import { sql } from 'drizzle-orm';
import { db } from './index';
export interface Bucket {
id: string;
name: string;
createdAt: Date;
updatedAt: Date;
}
type QueryRow = Record<string, unknown>;
type QueryResult = QueryRow[];
export const createBucket = async (name: string): Promise<Bucket> => {
const result = (await db.execute(
sql`INSERT INTO buckets (name) VALUES (${name}) RETURNING id, name, created_at, updated_at`,
)) as unknown as QueryResult;
const row = result[0]!;
return {
id: row.id as string,
name: row.name as string,
createdAt: new Date(row.created_at as string),
updatedAt: new Date(row.updated_at as string),
};
};
export const findBucketByName = async (name: string): Promise<Bucket | null> => {
const result = (await db.execute(
sql`SELECT id, name, created_at, updated_at FROM buckets WHERE name = ${name}`,
)) as unknown as QueryResult;
if (result.length === 0) return null;
const row = result[0]!;
return {
id: row.id as string,
name: row.name as string,
createdAt: new Date(row.created_at as string),
updatedAt: new Date(row.updated_at as string),
};
};
export const listBuckets = async (): Promise<Bucket[]> => {
const result = (await db.execute(
sql`SELECT id, name, created_at, updated_at FROM buckets ORDER BY name`,
)) as unknown as QueryResult;
return result.map((row) => ({
id: row.id as string,
name: row.name as string,
createdAt: new Date(row.created_at as string),
updatedAt: new Date(row.updated_at as string),
}));
};
export const deleteBucket = async (name: string): Promise<boolean> => {
// Cascade-delete rows that hold FK references to the bucket
await db
.execute(
sql`DELETE FROM multipart_parts WHERE upload_id IN (SELECT upload_id FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name}))`,
)
.catch(() => {});
await db
.execute(
sql`DELETE FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
)
.catch(() => {});
await db
.execute(
sql`DELETE FROM files WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
)
.catch(() => {});
const result = (await db.execute(
sql`DELETE FROM buckets WHERE name = ${name}`,
)) as unknown as QueryResult;
return result.length > 0;
};
export const bucketExists = async (name: string): Promise<boolean> => {
const result = (await db.execute(
sql`SELECT 1 FROM buckets WHERE name = ${name}`,
)) as unknown as QueryResult;
return result.length > 0;
};
-97
View File
@@ -1,97 +0,0 @@
import { sql } from 'drizzle-orm';
import { db } from './index';
export type CompressionAlgorithm = 'gzip' | null;
export interface FilePart {
id: number;
fileId: string;
partNumber: number;
telegramFileId: string;
telegramFileUniqueId: string;
storageChatId: number;
storageMessageId: number;
sizeBytes: number;
storedSizeBytes: number;
compressionAlgorithm: CompressionAlgorithm;
etag: string;
createdAt: Date;
}
export type NewFilePartInput = Omit<FilePart, 'id' | 'createdAt'>;
const toNumber = (value: unknown): number => Number(value ?? 0);
const mapRowToFilePart = (row: Record<string, unknown>): FilePart => ({
id: toNumber(row.id),
fileId: row.file_id as string,
partNumber: toNumber(row.part_number),
telegramFileId: row.telegram_file_id as string,
telegramFileUniqueId: row.telegram_file_unique_id as string,
storageChatId: toNumber(row.storage_chat_id),
storageMessageId: toNumber(row.storage_message_id),
sizeBytes: toNumber(row.size_bytes),
storedSizeBytes: toNumber(row.stored_size_bytes),
compressionAlgorithm: (row.compression_algorithm as CompressionAlgorithm) || null,
etag: row.etag as string,
createdAt: new Date(row.created_at as string),
});
export const insertFileParts = async (parts: NewFilePartInput[]): Promise<void> => {
for (const part of parts) {
await db.execute(
sql`INSERT INTO file_parts (
file_id,
part_number,
telegram_file_id,
telegram_file_unique_id,
storage_chat_id,
storage_message_id,
size_bytes,
stored_size_bytes,
compression_algorithm,
etag
) VALUES (
${part.fileId}::uuid,
${part.partNumber},
${part.telegramFileId},
${part.telegramFileUniqueId},
${part.storageChatId},
${part.storageMessageId},
${part.sizeBytes},
${part.storedSizeBytes},
${part.compressionAlgorithm},
${part.etag}
)`,
);
}
};
export const listFileParts = async (fileId: string): Promise<FilePart[]> => {
const result = (await db.execute(
sql`SELECT id,
file_id,
part_number,
telegram_file_id,
telegram_file_unique_id,
storage_chat_id,
storage_message_id,
size_bytes,
stored_size_bytes,
compression_algorithm,
etag,
created_at
FROM file_parts
WHERE file_id = ${fileId}::uuid
ORDER BY part_number`,
)) as unknown as Record<string, unknown>[];
return result.map(mapRowToFilePart);
};
export const countFileParts = async (fileId: string): Promise<number> => {
const result = (await db.execute(
sql`SELECT COUNT(*) AS count FROM file_parts WHERE file_id = ${fileId}::uuid`,
)) as unknown as Record<string, unknown>[];
return toNumber(result[0]?.count);
};
-143
View File
@@ -1,143 +0,0 @@
import { and, eq, sql } from 'drizzle-orm';
import { db, files as fileSchema } from './index';
import type { File } from './schema';
export interface S3FileRecord extends File {
bucketId: string;
s3Key: string;
}
export const findFileByBucketAndKey = async (
bucketId: string,
s3Key: string,
): Promise<File | null> => {
const result = await db
.select()
.from(fileSchema)
.where(
and(
eq(fileSchema.bucketId, bucketId),
eq(fileSchema.s3Key, s3Key),
eq(fileSchema.isDeleted, false),
),
)
.limit(1);
return result[0] || null;
};
const mapDbRowToS3Record = (row: Record<string, unknown>): S3FileRecord => {
return {
id: row.id as string,
publicId: row.public_id as string,
telegramFileId: row.telegram_file_id as string,
telegramFileUniqueId: row.telegram_file_unique_id as string,
storageChatId: toNumber(row.storage_chat_id),
storageMessageId: toNumber(row.storage_message_id),
fileName: row.file_name as string,
mimeType: row.mime_type as string,
sizeBytes: toNumber(row.size_bytes),
fileType: row.file_type as string,
uploaderId: toNumber(row.uploader_id),
fileHash: row.file_hash as string | null,
archiveTelegramFileId: row.archive_telegram_file_id as string | null,
archiveStorageMessageId:
row.archive_storage_message_id === null ? null : toNumber(row.archive_storage_message_id),
archiveFileName: row.archive_file_name as string | null,
archiveEntryName: row.archive_entry_name as string | null,
archiveMimeType: row.archive_mime_type as string | null,
archiveSizeBytes: row.archive_size_bytes === null ? null : toNumber(row.archive_size_bytes),
bucketId: row.bucket_id as string,
s3Key: row.s3_key as string,
storageBackend: (row.storage_backend as string) || 'telegram',
isDeleted: row.is_deleted as boolean,
multipartUploadId: row.multipart_upload_id as string | null,
partCount:
row.part_count === null || row.part_count === undefined ? null : toNumber(row.part_count),
createdAt: new Date(row.created_at as string),
updatedAt: new Date(row.updated_at as string),
};
};
const escapeLike = (s: string): string => s.replace(/[%_\\]/g, '\\$&');
const toNumber = (value: unknown): number => Number(value ?? 0);
export const listObjectsByPrefix = async (
bucketId: string,
prefix: string,
delimiter: string | null,
maxKeys: number,
startAfter: string | null,
): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> => {
let query = prefix
? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${escapeLike(prefix)}%`}`
: sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`;
if (startAfter) {
query = sql`${query} AND s3_key > ${startAfter}`;
}
query = sql`${query} ORDER BY s3_key LIMIT ${maxKeys + 1}`;
const rawResult = (await db.execute(query)) as unknown as Record<string, unknown>[];
if (delimiter === '/') {
const prefixSet = new Set<string>();
const objects: S3FileRecord[] = [];
for (const row of rawResult) {
const s3Key = row.s3_key as string;
const relativeKey = s3Key.substring(prefix.length);
const slashIndex = relativeKey.indexOf('/');
if (slashIndex >= 0) {
const folderPrefix = prefix + relativeKey.substring(0, slashIndex + 1);
if (folderPrefix !== prefix) {
prefixSet.add(folderPrefix);
}
} else {
objects.push(mapDbRowToS3Record(row));
}
}
return {
objects: objects.slice(0, maxKeys),
prefixes: Array.from(prefixSet).sort(),
};
}
return {
objects: rawResult.slice(0, maxKeys).map(mapDbRowToS3Record),
prefixes: [],
};
};
export const softDeleteFile = async (bucketId: string, s3Key: string): Promise<boolean> => {
const result = (await db.execute(
sql`UPDATE files SET is_deleted = true WHERE bucket_id = ${bucketId}::uuid AND s3_key = ${s3Key} RETURNING id`,
)) as unknown as Record<string, unknown>[];
return result.length > 0;
};
export const softDeleteFilesBatch = async (bucketId: string, keys: string[]): Promise<number> => {
let deleted = 0;
for (const key of keys) {
const ok = await softDeleteFile(bucketId, key);
if (ok) deleted++;
}
return deleted;
};
export const countBucketObjects = async (bucketId: string): Promise<number> => {
const result = (await db.execute(
sql`SELECT count(*) as count FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`,
)) as unknown as Record<string, unknown>[];
return Number(result[0]?.count || 0);
};
export const findOrphanFilesByBucket = async (bucketId: string): Promise<File[]> => {
return await db
.select()
.from(fileSchema)
.where(and(eq(fileSchema.bucketId, bucketId), eq(fileSchema.isDeleted, true)))
.limit(100);
};
-26
View File
@@ -1,26 +0,0 @@
import { eq } from 'drizzle-orm';
import { db, files as fileSchema } from './index';
import type { File } from './schema';
export const findFileByHash = async (hash: string): Promise<File | null> => {
const result = await db.select().from(fileSchema).where(eq(fileSchema.fileHash, hash)).limit(1);
return result[0] || null;
};
export const findFileByPublicId = async (publicId: string): Promise<File | null> => {
const result = await db
.select()
.from(fileSchema)
.where(eq(fileSchema.publicId, publicId))
.limit(1);
return result[0] || null;
};
export const findFileByUniqueId = async (telegramFileUniqueId: string): Promise<File | null> => {
const result = await db
.select()
.from(fileSchema)
.where(eq(fileSchema.telegramFileUniqueId, telegramFileUniqueId))
.limit(1);
return result[0] || null;
};
-13
View File
@@ -1,13 +0,0 @@
import { drizzle } from 'drizzle-orm/postgres-js';
import postgres from 'postgres';
import { fileParts, files } from './schema';
const client = postgres(process.env.DATABASE_URL!, {
max: 10,
idle_timeout: 20,
connect_timeout: 10,
});
export const db = drizzle(client, { schema: { fileParts, files } });
export { fileParts, files };
export default db;
-53
View File
@@ -1,53 +0,0 @@
import postgres from 'postgres';
import { config } from '../env';
import { getErrorMessage } from '../utils/file';
import logger from '../utils/logger';
/**
* Run raw SQL migration from schema.sql.
* Safe to call multiple times — all statements use IF NOT EXISTS.
*/
export const runMigration = async (): Promise<void> => {
// In compiled dist: import.meta.dir = .../dist/
// In source via bun --hot: import.meta.dir = .../src/db/
const dir = import.meta.dir || '';
const candidates = [
`${dir}/../../schema.sql`, // from dist/
`${dir}/../schema.sql`, // from src/ (bun --hot src/index.ts)
`${dir}/../schema.sql`, // from src/db/ (bun --hot src/db/migrate.ts)
`${dir}/schema.sql`, // from src/ (bun run db:migrate)
];
let schemaSql: string | null = null;
for (const p of candidates) {
const file = Bun.file(p);
const exists = await file.exists();
if (exists) {
schemaSql = await file.text();
break;
}
}
if (!schemaSql) {
logger.error(`Migration failed: schema.sql not found (tried ${candidates.join(', ')})`);
process.exitCode = 1;
return;
}
const sql = postgres(config.databaseUrl, { max: 1 });
try {
await sql.unsafe(schemaSql);
logger.info('Database migration completed');
} catch (error: unknown) {
logger.error('Database migration failed', { error: getErrorMessage(error) });
process.exitCode = 1;
} finally {
await sql.end();
}
};
// When run directly: `bun src/db/migrate.ts` or `bun dist/migrate.js`
if (import.meta.path === Bun.main) {
await runMigration();
}
-137
View File
@@ -1,137 +0,0 @@
import { sql } from 'drizzle-orm';
import { nanoid } from 'nanoid';
import { db } from './index';
export interface MultipartUpload {
uploadId: string;
bucketId: string;
s3Key: string;
initiatedAt: Date;
status: string;
initiatedBy: string;
contentType: string | null;
}
export interface MultipartPart {
id: number;
uploadId: string;
partNumber: number;
telegramFileId: string;
telegramFileUniqueId: string;
storageMessageId: number;
sizeBytes: number;
etag: string;
createdAt: Date;
}
export const createMultipartUpload = async (
bucketId: string,
s3Key: string,
initiatedBy: string,
contentType?: string | null,
): Promise<string> => {
const uploadId = nanoid(32);
await db.execute(
contentType
? sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by, content_type) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy}, ${contentType})`
: sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy})`,
);
return uploadId;
};
export const findMultipartUpload = async (uploadId: string): Promise<MultipartUpload | null> => {
const result = (await db.execute(
sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, content_type FROM multipart_uploads WHERE upload_id = ${uploadId} AND status = 'in_progress'`,
)) as unknown as Record<string, unknown>[];
if (result.length === 0) return null;
const r = result[0]!;
return {
uploadId: r.upload_id as string,
bucketId: r.bucket_id as string,
s3Key: r.s3_key as string,
initiatedAt: new Date(r.initiated_at as string),
status: r.status as string,
initiatedBy: '',
contentType: (r.content_type as string | null) || null,
};
};
export const completeMultipartUpload = async (uploadId: string): Promise<void> => {
await db.execute(
sql`UPDATE multipart_uploads SET status = 'completed' WHERE upload_id = ${uploadId}`,
);
};
export const abortMultipartUpload = async (uploadId: string): Promise<void> => {
// H7: FK cascade only fires on DELETE, not UPDATE. Delete parts explicitly
// before updating the upload status.
await db.execute(sql`DELETE FROM multipart_parts WHERE upload_id = ${uploadId}`);
await db.execute(
sql`UPDATE multipart_uploads SET status = 'aborted' WHERE upload_id = ${uploadId}`,
);
};
export const insertMultipartPart = async (
part: Omit<MultipartPart, 'id' | 'createdAt'>,
): Promise<void> => {
await db.execute(
sql`INSERT INTO multipart_parts (upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag)
VALUES (${part.uploadId}, ${part.partNumber}, ${part.telegramFileId}, ${part.telegramFileUniqueId}, ${part.storageMessageId}, ${part.sizeBytes}, ${part.etag})`,
);
};
export const listMultipartParts = async (uploadId: string): Promise<MultipartPart[]> => {
const result = (await db.execute(
sql`SELECT id, upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag, created_at
FROM multipart_parts WHERE upload_id = ${uploadId} ORDER BY part_number`,
)) as unknown as Record<string, unknown>[];
return result.map((r) => ({
id: r.id as number,
uploadId: r.upload_id as string,
partNumber: r.part_number as number,
telegramFileId: r.telegram_file_id as string,
telegramFileUniqueId: r.telegram_file_unique_id as string,
storageMessageId: r.storage_message_id as number,
sizeBytes: Number(r.size_bytes),
etag: r.etag as string,
createdAt: new Date(r.created_at as string),
}));
};
const mapRowToMultipartUpload = (r: Record<string, unknown>): MultipartUpload => ({
uploadId: r.upload_id as string,
bucketId: r.bucket_id as string,
s3Key: r.s3_key as string,
initiatedAt: new Date(r.initiated_at as string),
status: r.status as string,
initiatedBy: (r.initiated_by as string | null) || '',
contentType: (r.content_type as string | null) || null,
});
export const listMultipartUploadsByBucket = async (
bucketId: string,
maxUploads: number,
keyMarker: string | null,
): Promise<{ uploads: MultipartUpload[]; isTruncated: boolean; nextKeyMarker: string | null }> => {
const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000);
const result = (await db.execute(
keyMarker
? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
FROM multipart_uploads
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker}
ORDER BY s3_key, initiated_at
LIMIT ${limit + 1}`
: sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
FROM multipart_uploads
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress'
ORDER BY s3_key, initiated_at
LIMIT ${limit + 1}`,
)) as unknown as Record<string, unknown>[];
const uploads = result.slice(0, limit).map(mapRowToMultipartUpload);
return {
uploads,
isTruncated: result.length > limit,
nextKeyMarker: result.length > limit ? uploads.at(-1)?.s3Key || null : null,
};
};
-71
View File
@@ -1,71 +0,0 @@
import { eq, type InferInsertModel, type InferSelectModel } from 'drizzle-orm';
import {
bigint,
boolean,
integer,
pgTable,
serial,
text,
timestamp,
uniqueIndex,
uuid,
} from 'drizzle-orm/pg-core';
export const files = pgTable(
'files',
{
id: uuid('id').primaryKey().defaultRandom(),
publicId: text('public_id').unique().notNull(),
telegramFileId: text('telegram_file_id').notNull(),
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
fileName: text('file_name').notNull(),
mimeType: text('mime_type').notNull(),
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
fileType: text('file_type').notNull(),
uploaderId: bigint('uploader_id', { mode: 'number' }).notNull(),
fileHash: text('file_hash'),
archiveTelegramFileId: text('archive_telegram_file_id'),
archiveStorageMessageId: bigint('archive_storage_message_id', { mode: 'number' }),
archiveFileName: text('archive_file_name'),
archiveEntryName: text('archive_entry_name'),
archiveMimeType: text('archive_mime_type'),
archiveSizeBytes: bigint('archive_size_bytes', { mode: 'number' }),
bucketId: text('bucket_id'),
s3Key: text('s3_key'),
storageBackend: text('storage_backend').default('telegram'),
isDeleted: boolean('is_deleted').default(false),
multipartUploadId: text('multipart_upload_id'),
partCount: integer('part_count'),
createdAt: timestamp('created_at').defaultNow().notNull(),
updatedAt: timestamp('updated_at').defaultNow().notNull(),
},
(table) => ({
// H2: Prevent TOCTOU race on concurrent PUT — only one active (non-deleted)
// object per (bucket_id, s3_key) pair.
activeObjectIdx: uniqueIndex('active_object_idx')
.on(table.bucketId, table.s3Key)
.where(eq(table.isDeleted, false)),
}),
);
export const fileParts = pgTable('file_parts', {
id: serial('id').primaryKey(),
fileId: uuid('file_id').notNull(),
partNumber: integer('part_number').notNull(),
telegramFileId: text('telegram_file_id').notNull(),
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
storedSizeBytes: bigint('stored_size_bytes', { mode: 'number' }).notNull(),
compressionAlgorithm: text('compression_algorithm'),
etag: text('etag').notNull(),
createdAt: timestamp('created_at').defaultNow().notNull(),
});
export type File = InferSelectModel<typeof files>;
export type NewFile = InferInsertModel<typeof files>;
export type FilePart = InferSelectModel<typeof fileParts>;
export type NewFilePart = InferInsertModel<typeof fileParts>;
+1 -1
View File
@@ -1,4 +1,4 @@
import logger from './utils/logger'; import logger from './shared/logger/index';
interface AppConfig { interface AppConfig {
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */ /** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
+1 -1
View File
@@ -12,7 +12,7 @@ import { metricsCollector } from './shared/metrics/index';
// ─── Auto-run migration at startup ────────────────────────────────────────── // ─── Auto-run migration at startup ──────────────────────────────────────────
try { try {
const { runMigration } = await import('./db/migrate'); const { runMigration } = await import('./infrastructure/persistence/drizzle/migrate');
await runMigration(); await runMigration();
} catch { } catch {
logger.warn('Auto-migration skipped (non-fatal)'); logger.warn('Auto-migration skipped (non-fatal)');
+51
View File
@@ -0,0 +1,51 @@
/**
* Dependency Injection container.
*
* Wires up singleton instances of all repositories and application services,
* making them available to controllers and other adapters without requiring
* a full DI framework.
*
* @module infrastructure/di
*/
import type { IBucketRepository } from '../domain/ports/bucket-repository';
import type { IFilePartRepository } from '../domain/ports/file-part-repository';
import type { IFileRepository } from '../domain/ports/file-repository';
import type { IMultipartRepository } from '../domain/ports/multipart-repository';
import type { ITelegramService } from '../domain/ports/telegram-service';
import { DrizzleBucketRepository } from './persistence/repositories/bucket-repository';
import { DrizzleFilePartRepository } from './persistence/repositories/file-part-repository';
import { DrizzleFileRepository } from './persistence/repositories/file-repository';
import { DrizzleMultipartRepository } from './persistence/repositories/multipart-repository';
import { botPool } from './telegram/bot-pool';
import { ChunkedStorage } from './telegram/chunked-storage';
import { UploadBatcher } from './telegram/upload-batcher';
// ─── Repository Singletons ──────────────────────────────────────────
/** Singleton IFileRepository instance backed by Drizzle ORM. */
export const fileRepository: IFileRepository = new DrizzleFileRepository();
/** Singleton IBucketRepository instance backed by Drizzle ORM. */
export const bucketRepository: IBucketRepository = new DrizzleBucketRepository();
/** Singleton IFilePartRepository instance backed by Drizzle ORM. */
export const filePartRepository: IFilePartRepository = new DrizzleFilePartRepository();
/** Singleton IMultipartRepository instance backed by Drizzle ORM. */
export const multipartRepository: IMultipartRepository = new DrizzleMultipartRepository();
/** Singleton ITelegramService instance backed by the bot pool. */
export const telegramService: ITelegramService = botPool;
// ─── Service Singletons ─────────────────────────────────────────────
/** Singleton ChunkedStorage for large file chunked uploads. */
export const chunkedStorage = new ChunkedStorage(
fileRepository,
filePartRepository,
telegramService,
);
/** Singleton UploadBatcher for batched small-file uploads. */
export const uploadBatcher = new UploadBatcher(fileRepository, telegramService);
@@ -11,7 +11,7 @@ import {
createSessionCookie, createSessionCookie,
getAuthSession, getAuthSession,
isAuthEnabled, isAuthEnabled,
} from '../../../utils/auth'; } from '../middleware/auth';
/** /**
* Helper that builds a JSON Response with optional extra headers. * Helper that builds a JSON Response with optional extra headers.
@@ -2,11 +2,11 @@ import { createReadStream } from 'node:fs';
import { nanoid } from 'nanoid'; import { nanoid } from 'nanoid';
import type { TelegramFileInfo } from '../../../domain/ports/telegram-service'; import type { TelegramFileInfo } from '../../../domain/ports/telegram-service';
import { fileInfoCache } from '../../../infrastructure/cache/index'; import { fileInfoCache } from '../../../infrastructure/cache/index';
import { chunkedStorage } from '../../../infrastructure/di';
import { botPool } from '../../../infrastructure/telegram/bot-pool'; import { botPool } from '../../../infrastructure/telegram/bot-pool';
import logger from '../../../shared/logger/index'; import logger from '../../../shared/logger/index';
import { cleanupTempFile, formatCreatedAt, getErrorMessage } from '../../../shared/utils/file'; import { cleanupTempFile, formatCreatedAt, getErrorMessage } from '../../../shared/utils/file';
import { createChunkedObjectResponse } from '../../../utils/chunked-storage'; import { locateZipEntry } from '../../../shared/utils/zip';
import { locateZipEntry } from '../../../utils/zip';
/** /**
* Extended Request type that includes route parameter access. * Extended Request type that includes route parameter access.
@@ -115,7 +115,7 @@ export const handleFileRedirect = async (req: RequestWithParams): Promise<Respon
return fail(501, 'Archive entry extraction is not supported for chunked files'); return fail(501, 'Archive entry extraction is not supported for chunked files');
} }
const range = { type: 'none' as const }; const range = { type: 'none' as const };
return createChunkedObjectResponse({ file, range, reqId: '' }); return chunkedStorage.createChunkedObjectResponse({ file, range, reqId: '' });
} }
const archiveEntryName = file.archiveEntryName; const archiveEntryName = file.archiveEntryName;
@@ -1,36 +1,22 @@
import { createReadStream } from 'node:fs'; import { createReadStream } from 'node:fs';
import { nanoid } from 'nanoid'; import { nanoid } from 'nanoid';
import { config } from '../../../config/index'; import { config } from '../../../config/index';
import { createBucket, deleteBucket, findBucketByName, listBuckets } from '../../../db/buckets'; import type { File as FileEntity } from '../../../domain/entities/file';
import {
countBucketObjects,
findFileByBucketAndKey,
listObjectsByPrefix,
softDeleteFile,
} from '../../../db/files-ext';
import { db, files as fileSchema } from '../../../db/index';
import {
abortMultipartUpload,
completeMultipartUpload,
createMultipartUpload,
findMultipartUpload,
insertMultipartPart,
listMultipartParts,
listMultipartUploadsByBucket,
} from '../../../db/multipart';
import type { File } from '../../../db/schema';
import type { ForwardResult } from '../../../domain/ports/telegram-service'; import type { ForwardResult } from '../../../domain/ports/telegram-service';
import {
bucketRepository,
chunkedStorage,
fileRepository,
multipartRepository,
} from '../../../infrastructure/di';
import { db, files as fileSchema } from '../../../infrastructure/persistence/drizzle/index';
import { botPool } from '../../../infrastructure/telegram/bot-pool'; import { botPool } from '../../../infrastructure/telegram/bot-pool';
import logger from '../../../shared/logger/index'; import logger from '../../../shared/logger/index';
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file'; import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
import { import { verifyBodyHash, verifyPresignedUrl, verifySignature } from '../../s3/auth';
createChunkedObjectResponse, import { S3_CORS_HEADERS, s3Headers } from '../../s3/headers';
storeFileInTelegramChunks, import { createGetObjectResponse, type ObjectPartSource } from '../../s3/object-stream';
} from '../../../utils/chunked-storage'; import { parseRangeHeader, unsatisfiedContentRange } from '../../s3/range';
import { verifyBodyHash, verifyPresignedUrl, verifySignature } from '../../../utils/s3/auth';
import { S3_CORS_HEADERS, s3Headers } from '../../../utils/s3/headers';
import { createGetObjectResponse, type ObjectPartSource } from '../../../utils/s3/object-stream';
import { parseRangeHeader, unsatisfiedContentRange } from '../../../utils/s3/range';
import { import {
bucketVersioningConfigurationXml, bucketVersioningConfigurationXml,
completeMultipartUploadXml, completeMultipartUploadXml,
@@ -45,7 +31,7 @@ import {
parseCompleteMultipartBody, parseCompleteMultipartBody,
parseDeleteObjectsBody, parseDeleteObjectsBody,
s3ErrorResponse, s3ErrorResponse,
} from '../../../utils/s3/xml'; } from '../../s3/xml';
/** /**
* The default S3 region returned when no region is explicitly configured. * The default S3 region returned when no region is explicitly configured.
@@ -309,7 +295,7 @@ export const handleS3Request = async (
* @returns An S3 XML response with the bucket list. * @returns An S3 XML response with the bucket list.
*/ */
const handleListBuckets = async (reqId: string): Promise<Response> => { const handleListBuckets = async (reqId: string): Promise<Response> => {
const buckets = await listBuckets(); const buckets = await bucketRepository.list();
const xml = listBucketsXml(buckets, reqId); const xml = listBucketsXml(buckets, reqId);
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
}; };
@@ -339,7 +325,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
reqId, reqId,
); );
} }
const existing = await findBucketByName(bucketName); const existing = await bucketRepository.findByName(bucketName);
if (existing) { if (existing) {
return s3ErrorResponse( return s3ErrorResponse(
'BucketAlreadyExists', 'BucketAlreadyExists',
@@ -349,7 +335,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
reqId, reqId,
); );
} }
await createBucket(bucketName); await bucketRepository.create(bucketName);
return s3Response(null, 200, reqId); return s3Response(null, 200, reqId);
}; };
@@ -361,7 +347,7 @@ const handleCreateBucket = async (bucketName: string, reqId: string): Promise<Re
* @returns A 200 response when the bucket exists, or an S3 XML error. * @returns A 200 response when the bucket exists, or an S3 XML error.
*/ */
const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Response> => { const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Response> => {
const bucket = await findBucketByName(bucketName); const bucket = await bucketRepository.findByName(bucketName);
if (!bucket) { if (!bucket) {
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -384,7 +370,7 @@ const handleHeadBucket = async (bucketName: string, reqId: string): Promise<Resp
* @returns A 204 response on success, or an S3 XML error. * @returns A 204 response on success, or an S3 XML error.
*/ */
const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Response> => { const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Response> => {
const bucket = await findBucketByName(bucketName); const bucket = await bucketRepository.findByName(bucketName);
if (!bucket) { if (!bucket) {
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -394,7 +380,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
reqId, reqId,
); );
} }
const objCount = await countBucketObjects(bucket.id); const objCount = await fileRepository.countByBucket(bucket.id);
if (objCount > 0) { if (objCount > 0) {
return s3ErrorResponse( return s3ErrorResponse(
'BucketNotEmpty', 'BucketNotEmpty',
@@ -404,7 +390,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
reqId, reqId,
); );
} }
await deleteBucket(bucketName); await bucketRepository.delete(bucketName);
return s3Response(null, 204, reqId); return s3Response(null, 204, reqId);
}; };
@@ -417,7 +403,7 @@ const handleDeleteBucket = async (bucketName: string, reqId: string): Promise<Re
* @returns An S3 XML response with the versioning configuration. * @returns An S3 XML response with the versioning configuration.
*/ */
const handleGetBucketVersioning = async (bucketName: string, reqId: string): Promise<Response> => { const handleGetBucketVersioning = async (bucketName: string, reqId: string): Promise<Response> => {
const bucket = await findBucketByName(bucketName); const bucket = await bucketRepository.findByName(bucketName);
if (!bucket) { if (!bucket) {
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -456,7 +442,7 @@ const handleGetObject = async (
headers: Record<string, string>, headers: Record<string, string>,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -466,7 +452,7 @@ const handleGetObject = async (
reqId, reqId,
); );
const file = await findFileByBucketAndKey(bucketRecord.id, key); const file = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
if (!file) if (!file)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchKey', 'NoSuchKey',
@@ -551,7 +537,7 @@ const handleGetObject = async (
); );
} }
try { try {
return await createChunkedObjectResponse({ file, range, reqId }); return await chunkedStorage.createChunkedObjectResponse({ file, range, reqId });
} catch (error) { } catch (error) {
logger.warn('Chunked object content fetch failed', { key, error: getErrorMessage(error) }); logger.warn('Chunked object content fetch failed', { key, error: getErrorMessage(error) });
return s3ErrorResponse( return s3ErrorResponse(
@@ -638,14 +624,14 @@ const handleGetObject = async (
* @returns An S3 response streaming the assembled object content. * @returns An S3 response streaming the assembled object content.
*/ */
const handleGetMultipartObject = async ( const handleGetMultipartObject = async (
file: File, file: FileEntity,
bucket: string, bucket: string,
key: string, key: string,
headers: Record<string, string>, headers: Record<string, string>,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const uploadId = file.multipartUploadId!; const uploadId = file.multipartUploadId!;
const parts = await listMultipartParts(uploadId); const parts = await multipartRepository.listParts(uploadId);
if (parts.length === 0) { if (parts.length === 0) {
return s3ErrorResponse( return s3ErrorResponse(
@@ -724,7 +710,7 @@ const handleHeadObject = async (
headers: Record<string, string>, headers: Record<string, string>,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -734,7 +720,7 @@ const handleHeadObject = async (
reqId, reqId,
); );
const file = await findFileByBucketAndKey(bucketRecord.id, key); const file = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
if (!file) if (!file)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchKey', 'NoSuchKey',
@@ -930,7 +916,7 @@ const handlePutObject = async (
req: Request, req: Request,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -1011,7 +997,7 @@ const handlePutObject = async (
// Idempotent PUT: if the object already exists, skip upload // Idempotent PUT: if the object already exists, skip upload
try { try {
const existing = await findFileByBucketAndKey(bucketRecord.id, key); const existing = await fileRepository.findByBucketAndKey(bucketRecord.id, key);
if (existing) { if (existing) {
await cleanupTempFile(streamed.tempPath); await cleanupTempFile(streamed.tempPath);
return s3Response(null, 200, reqId, { etag: `"${streamed.fileHash}"` }); return s3Response(null, 200, reqId, { etag: `"${streamed.fileHash}"` });
@@ -1057,7 +1043,7 @@ const storeFileFromTemp = async (
const partFileNamePrefix = `s3-${bucketRecord.name}-${key.replace(/\//g, '_')}`; const partFileNamePrefix = `s3-${bucketRecord.name}-${key.replace(/\//g, '_')}`;
if (streamed.sizeBytes > config.telegramChunkSizeBytes) { if (streamed.sizeBytes > config.telegramChunkSizeBytes) {
const file = await storeFileInTelegramChunks({ const file = await chunkedStorage.storeFileInTelegramChunks({
tempPath: streamed.tempPath, tempPath: streamed.tempPath,
partFileNamePrefix, partFileNamePrefix,
fileName: finalFileName, fileName: finalFileName,
@@ -1138,7 +1124,7 @@ const handleCopyObject = async (
const sourceBucket = parts[0]; const sourceBucket = parts[0];
const sourceKey = parts.slice(1).join('/'); const sourceKey = parts.slice(1).join('/');
const sourceBucketRecord = await findBucketByName(sourceBucket); const sourceBucketRecord = await bucketRepository.findByName(sourceBucket);
if (!sourceBucketRecord) if (!sourceBucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -1148,7 +1134,7 @@ const handleCopyObject = async (
reqId, reqId,
); );
const sourceFile = await findFileByBucketAndKey(sourceBucketRecord.id, sourceKey); const sourceFile = await fileRepository.findByBucketAndKey(sourceBucketRecord.id, sourceKey);
if (!sourceFile) if (!sourceFile)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchKey', 'NoSuchKey',
@@ -1232,7 +1218,7 @@ const handleDeleteObject = async (
key: string, key: string,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -1242,7 +1228,7 @@ const handleDeleteObject = async (
reqId, reqId,
); );
await softDeleteFile(bucketRecord.id, key); await fileRepository.softDelete(bucketRecord.id, key);
return s3Response(null, 204, reqId); return s3Response(null, 204, reqId);
}; };
@@ -1262,7 +1248,7 @@ const handleDeleteObjects = async (
body: string, body: string,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -1288,7 +1274,7 @@ const handleDeleteObjects = async (
const deletedKeys: string[] = []; const deletedKeys: string[] = [];
const errors: Array<{ key: string; code: string; message: string }> = []; const errors: Array<{ key: string; code: string; message: string }> = [];
for (const key of keys) { for (const key of keys) {
const ok = await softDeleteFile(bucketRecord.id, key); const ok = await fileRepository.softDelete(bucketRecord.id, key);
if (ok) { if (ok) {
deletedKeys.push(key); deletedKeys.push(key);
} else { } else {
@@ -1316,7 +1302,7 @@ const handleListObjectsV1 = async (
searchParams: URLSearchParams, searchParams: URLSearchParams,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -1335,7 +1321,7 @@ const handleListObjectsV1 = async (
const marker = searchParams.get('marker') || null; const marker = searchParams.get('marker') || null;
const encodingType = searchParams.get('encoding-type') || null; const encodingType = searchParams.get('encoding-type') || null;
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix( const { objects, prefixes: commonPrefixes } = await fileRepository.listByPrefix(
bucketRecord.id, bucketRecord.id,
prefix, prefix,
delimiter, delimiter,
@@ -1386,7 +1372,7 @@ const handleListObjectsV2 = async (
searchParams: URLSearchParams, searchParams: URLSearchParams,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -1403,7 +1389,7 @@ const handleListObjectsV2 = async (
const startAfter = searchParams.get('start-after') || null; const startAfter = searchParams.get('start-after') || null;
const encodingType = searchParams.get('encoding-type') || null; const encodingType = searchParams.get('encoding-type') || null;
const { objects, prefixes: commonPrefixes } = await listObjectsByPrefix( const { objects, prefixes: commonPrefixes } = await fileRepository.listByPrefix(
bucketRecord.id, bucketRecord.id,
prefix, prefix,
delimiter, delimiter,
@@ -1459,7 +1445,7 @@ const handleCreateMultipartUpload = async (
headers: Record<string, string>, headers: Record<string, string>,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -1470,7 +1456,7 @@ const handleCreateMultipartUpload = async (
); );
const contentType = headers['content-type'] || null; const contentType = headers['content-type'] || null;
const uploadId = await createMultipartUpload(bucketRecord.id, key, 's3', contentType); const uploadId = await multipartRepository.create(bucketRecord.id, key, 's3', contentType);
const xml = initiateMultipartUploadXml(bucket, key, uploadId); const xml = initiateMultipartUploadXml(bucket, key, uploadId);
return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' }); return s3Response(xml, 200, reqId, { 'content-type': 'application/xml' });
@@ -1514,7 +1500,7 @@ const handleUploadPart = async (
); );
} }
const multipart = await findMultipartUpload(uploadId); const multipart = await multipartRepository.findById(uploadId);
if (!multipart || multipart.s3Key !== key) { if (!multipart || multipart.s3Key !== key) {
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchUpload', 'NoSuchUpload',
@@ -1583,7 +1569,7 @@ const handleUploadPart = async (
await cleanupTempFile(tempPath); await cleanupTempFile(tempPath);
const etag = hasher.digest('hex'); const etag = hasher.digest('hex');
await insertMultipartPart({ await multipartRepository.insertPart({
uploadId, uploadId,
partNumber, partNumber,
telegramFileId: forwardResult.telegramFileId, telegramFileId: forwardResult.telegramFileId,
@@ -1617,7 +1603,7 @@ const handleCompleteMultipartUpload = async (
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const uploadId = searchParams.get('uploadId')!; const uploadId = searchParams.get('uploadId')!;
const multipart = await findMultipartUpload(uploadId); const multipart = await multipartRepository.findById(uploadId);
// H5: Verify both upload exists AND key matches (consistent with handleUploadPart) // H5: Verify both upload exists AND key matches (consistent with handleUploadPart)
if (!multipart || multipart.s3Key !== key) { if (!multipart || multipart.s3Key !== key) {
return s3ErrorResponse( return s3ErrorResponse(
@@ -1630,7 +1616,7 @@ const handleCompleteMultipartUpload = async (
} }
const parts = parseCompleteMultipartBody(body); const parts = parseCompleteMultipartBody(body);
const storedParts = await listMultipartParts(uploadId); const storedParts = await multipartRepository.listParts(uploadId);
// Validate ascending part order // Validate ascending part order
const partNumbers = parts.map((p) => p.partNumber); const partNumbers = parts.map((p) => p.partNumber);
@@ -1702,7 +1688,7 @@ const handleCompleteMultipartUpload = async (
updatedAt: new Date(), updatedAt: new Date(),
}); });
await completeMultipartUpload(uploadId); await multipartRepository.complete(uploadId);
const location = `${config.baseUrl}/${bucket}/${key}`; const location = `${config.baseUrl}/${bucket}/${key}`;
const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location); const xml = completeMultipartUploadXml(bucket, key, combinedEtag, location);
@@ -1723,7 +1709,7 @@ const handleListMultipartUploads = async (
searchParams: URLSearchParams, searchParams: URLSearchParams,
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const bucketRecord = await findBucketByName(bucket); const bucketRecord = await bucketRepository.findByName(bucket);
if (!bucketRecord) if (!bucketRecord)
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchBucket', 'NoSuchBucket',
@@ -1735,7 +1721,7 @@ const handleListMultipartUploads = async (
const maxUploads = Math.min(Number.parseInt(searchParams.get('max-uploads') || '1000', 10), 1000); const maxUploads = Math.min(Number.parseInt(searchParams.get('max-uploads') || '1000', 10), 1000);
const keyMarker = searchParams.get('key-marker') || null; const keyMarker = searchParams.get('key-marker') || null;
const { uploads, isTruncated, nextKeyMarker } = await listMultipartUploadsByBucket( const { uploads, isTruncated, nextKeyMarker } = await multipartRepository.listByBucket(
bucketRecord.id, bucketRecord.id,
maxUploads, maxUploads,
keyMarker, keyMarker,
@@ -1774,7 +1760,7 @@ const handleAbortMultipartUpload = async (
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const uploadId = searchParams.get('uploadId')!; const uploadId = searchParams.get('uploadId')!;
const multipart = await findMultipartUpload(uploadId); const multipart = await multipartRepository.findById(uploadId);
if (!multipart) { if (!multipart) {
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchUpload', 'NoSuchUpload',
@@ -1785,7 +1771,7 @@ const handleAbortMultipartUpload = async (
); );
} }
await abortMultipartUpload(uploadId); await multipartRepository.abort(uploadId);
return s3Response(null, 204, reqId); return s3Response(null, 204, reqId);
}; };
@@ -1807,7 +1793,7 @@ const handleListParts = async (
reqId: string, reqId: string,
): Promise<Response> => { ): Promise<Response> => {
const uploadId = searchParams.get('uploadId')!; const uploadId = searchParams.get('uploadId')!;
const multipart = await findMultipartUpload(uploadId); const multipart = await multipartRepository.findById(uploadId);
if (!multipart) { if (!multipart) {
return s3ErrorResponse( return s3ErrorResponse(
'NoSuchUpload', 'NoSuchUpload',
@@ -1818,7 +1804,7 @@ const handleListParts = async (
); );
} }
const parts = await listMultipartParts(uploadId); const parts = await multipartRepository.listParts(uploadId);
const maxParts = Math.min(Number.parseInt(searchParams.get('max-parts') || '1000', 10), 1000); const maxParts = Math.min(Number.parseInt(searchParams.get('max-parts') || '1000', 10), 1000);
const xml = listPartsXml( const xml = listPartsXml(
@@ -1,7 +1,8 @@
import { createWriteStream } from 'node:fs'; import { createWriteStream } from 'node:fs';
import { nanoid } from 'nanoid'; import { nanoid } from 'nanoid';
import { config } from '../../../config/index'; import { config } from '../../../config/index';
import { findFileByHash } from '../../../db/files'; import { chunkedStorage, fileRepository, uploadBatcher } from '../../../infrastructure/di';
import type { PreparedUpload } from '../../../infrastructure/telegram/upload-batcher';
import logger from '../../../shared/logger/index'; import logger from '../../../shared/logger/index';
import { metricsCollector } from '../../../shared/metrics/index'; import { metricsCollector } from '../../../shared/metrics/index';
import { import {
@@ -14,8 +15,6 @@ import {
getErrorMessage, getErrorMessage,
getFileType, getFileType,
} from '../../../shared/utils/file'; } from '../../../shared/utils/file';
import { storeFileInTelegramChunks } from '../../../utils/chunked-storage';
import { enqueuePreparedUpload, type PreparedUpload } from '../../../utils/uploadBatcher';
/** /**
* Maximum allowed size (in bytes) for a base64 JSON upload. * Maximum allowed size (in bytes) for a base64 JSON upload.
@@ -223,7 +222,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
const prepared = await streamFileToTemp(file, config.maxRequestBodyBytes); const prepared = await streamFileToTemp(file, config.maxRequestBodyBytes);
const existingFile = await findFileByHash(prepared.fileHash); const existingFile = await fileRepository.findByHash(prepared.fileHash);
if (existingFile) { if (existingFile) {
await cleanupTempFile(prepared.tempPath); await cleanupTempFile(prepared.tempPath);
return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 }); return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 });
@@ -243,7 +242,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
} }
if (prepared.sizeBytes > config.telegramChunkSizeBytes) { if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
const uploadedFile = await storeFileInTelegramChunks({ const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
tempPath: prepared.tempPath, tempPath: prepared.tempPath,
partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'upload'}`, partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'upload'}`,
fileName: finalFileName, fileName: finalFileName,
@@ -256,7 +255,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 }); return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 });
} }
const uploaded = await enqueuePreparedUpload({ const uploaded = await uploadBatcher.enqueuePreparedUpload({
prepared, prepared,
fileName: finalFileName, fileName: finalFileName,
mimeType, mimeType,
@@ -317,7 +316,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
const fileBytes = Buffer.from(base64Data, 'base64'); const fileBytes = Buffer.from(base64Data, 'base64');
const hash = computeHash(fileBytes); const hash = computeHash(fileBytes);
const existingFile = await findFileByHash(hash); const existingFile = await fileRepository.findByHash(hash);
if (existingFile) { if (existingFile) {
return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 }); return Response.json(buildUploadResponse(existingFile, config.baseUrl), { status: 200 });
} }
@@ -334,7 +333,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
const prepared = await writeBufferToTemp(fileBytes, hash); const prepared = await writeBufferToTemp(fileBytes, hash);
if (prepared.sizeBytes > config.telegramChunkSizeBytes) { if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
const uploadedFile = await storeFileInTelegramChunks({ const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
tempPath: prepared.tempPath, tempPath: prepared.tempPath,
partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'json'}`, partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'json'}`,
fileName: finalFileName, fileName: finalFileName,
@@ -347,7 +346,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 }); return Response.json(buildUploadResponse(uploadedFile, config.baseUrl), { status: 200 });
} }
const uploaded = await enqueuePreparedUpload({ const uploaded = await uploadBatcher.enqueuePreparedUpload({
prepared, prepared,
fileName: finalFileName, fileName: finalFileName,
mimeType, mimeType,
@@ -1,21 +1,11 @@
import { createReadStream } from 'node:fs'; import { createReadStream } from 'node:fs';
import { nanoid } from 'nanoid'; import { nanoid } from 'nanoid';
import { config } from '../../../config/index'; import { config } from '../../../config/index';
import { createBucket, deleteBucket, findBucketByName, listBuckets } from '../../../db/buckets'; import { bucketRepository, chunkedStorage, fileRepository } from '../../../infrastructure/di';
import { import { db, files as fileSchema } from '../../../infrastructure/persistence/drizzle/index';
countBucketObjects,
findFileByBucketAndKey,
listObjectsByPrefix,
softDeleteFile,
} from '../../../db/files-ext';
import { db, files as fileSchema } from '../../../db/index';
import { botPool } from '../../../infrastructure/telegram/bot-pool'; import { botPool } from '../../../infrastructure/telegram/bot-pool';
import logger from '../../../shared/logger/index'; import logger from '../../../shared/logger/index';
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file'; import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
import {
createChunkedObjectResponse,
storeFileInTelegramChunks,
} from '../../../utils/chunked-storage';
/** /**
* Route parameters extracted from the URL path. * Route parameters extracted from the URL path.
@@ -48,13 +38,13 @@ const jsonError = (error: string, status: number): Response => Response.json({ e
* @returns A JSON response with the bucket list. * @returns A JSON response with the bucket list.
*/ */
export const handleListBucketsV1 = async (): Promise<Response> => { export const handleListBucketsV1 = async (): Promise<Response> => {
const buckets = await listBuckets(); const buckets = await bucketRepository.list();
const result = await Promise.all( const result = await Promise.all(
buckets.map(async (b) => ({ buckets.map(async (b) => ({
id: b.id, id: b.id,
name: b.name, name: b.name,
createdAt: b.createdAt.toISOString(), createdAt: b.createdAt.toISOString(),
objectCount: await countBucketObjects(b.id), objectCount: await fileRepository.countByBucket(b.id),
})), })),
); );
return json({ buckets: result }); return json({ buckets: result });
@@ -73,9 +63,9 @@ export const handleCreateBucketV1 = async (req: Request): Promise<Response> => {
if (!body.name || !/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(body.name)) { if (!body.name || !/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(body.name)) {
return jsonError('Invalid bucket name. Use lowercase, 3-63 chars, no underscore', 400); return jsonError('Invalid bucket name. Use lowercase, 3-63 chars, no underscore', 400);
} }
const existing = await findBucketByName(body.name); const existing = await bucketRepository.findByName(body.name);
if (existing) return jsonError('Bucket already exists', 409); if (existing) return jsonError('Bucket already exists', 409);
const bucket = await createBucket(body.name); const bucket = await bucketRepository.create(body.name);
return json({ id: bucket.id, name: bucket.name }, 201); return json({ id: bucket.id, name: bucket.name }, 201);
}; };
@@ -92,11 +82,11 @@ export const handleDeleteBucketV1 = async (
_req: Request, _req: Request,
params: RouteParams, params: RouteParams,
): Promise<Response> => { ): Promise<Response> => {
const bucket = await findBucketByName(params.bucket!); const bucket = await bucketRepository.findByName(params.bucket!);
if (!bucket) return jsonError('Bucket not found', 404); if (!bucket) return jsonError('Bucket not found', 404);
const count = await countBucketObjects(bucket.id); const count = await fileRepository.countByBucket(bucket.id);
if (count > 0) return jsonError('Bucket is not empty', 409); if (count > 0) return jsonError('Bucket is not empty', 409);
await deleteBucket(params.bucket!); await bucketRepository.delete(params.bucket!);
return json({ success: true }); return json({ success: true });
}; };
@@ -110,7 +100,7 @@ export const handleDeleteBucketV1 = async (
* @returns A JSON response with the object list. * @returns A JSON response with the object list.
*/ */
export const handleListObjectsV1 = async (req: Request, params: RouteParams): Promise<Response> => { export const handleListObjectsV1 = async (req: Request, params: RouteParams): Promise<Response> => {
const bucket = await findBucketByName(params.bucket!); const bucket = await bucketRepository.findByName(params.bucket!);
if (!bucket) return jsonError('Bucket not found', 404); if (!bucket) return jsonError('Bucket not found', 404);
const url = new URL(req.url); const url = new URL(req.url);
@@ -119,7 +109,7 @@ export const handleListObjectsV1 = async (req: Request, params: RouteParams): Pr
const maxKeys = Number.parseInt(url.searchParams.get('max-keys') || '1000', 10); const maxKeys = Number.parseInt(url.searchParams.get('max-keys') || '1000', 10);
const continuationToken = url.searchParams.get('continuation-token') || null; const continuationToken = url.searchParams.get('continuation-token') || null;
const { objects, prefixes } = await listObjectsByPrefix( const { objects, prefixes } = await fileRepository.listByPrefix(
bucket.id, bucket.id,
prefix, prefix,
delimiter, delimiter,
@@ -162,7 +152,7 @@ export const handleUploadObjectV1 = async (
req: Request, req: Request,
params: RouteParams, params: RouteParams,
): Promise<Response> => { ): Promise<Response> => {
const bucket = await findBucketByName(params.bucket!); const bucket = await bucketRepository.findByName(params.bucket!);
if (!bucket) return jsonError('Bucket not found', 404); if (!bucket) return jsonError('Bucket not found', 404);
const formData = await req.formData(); const formData = await req.formData();
@@ -217,7 +207,7 @@ export const handleUploadObjectV1 = async (
const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`; const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`;
if (sizeBytes > config.telegramChunkSizeBytes) { if (sizeBytes > config.telegramChunkSizeBytes) {
const uploadedFile = await storeFileInTelegramChunks({ const uploadedFile = await chunkedStorage.storeFileInTelegramChunks({
tempPath, tempPath,
partFileNamePrefix, partFileNamePrefix,
fileName: finalFileName, fileName: finalFileName,
@@ -287,9 +277,9 @@ export const handleDeleteObjectV1 = async (
_req: Request, _req: Request,
params: RouteParams, params: RouteParams,
): Promise<Response> => { ): Promise<Response> => {
const bucket = await findBucketByName(params.bucket!); const bucket = await bucketRepository.findByName(params.bucket!);
if (!bucket) return jsonError('Bucket not found', 404); if (!bucket) return jsonError('Bucket not found', 404);
await softDeleteFile(bucket.id, params.key!); await fileRepository.softDelete(bucket.id, params.key!);
return json({ success: true }); return json({ success: true });
}; };
@@ -307,15 +297,15 @@ export const handleDownloadObjectV1 = async (
_req: Request, _req: Request,
params: RouteParams, params: RouteParams,
): Promise<Response> => { ): Promise<Response> => {
const bucket = await findBucketByName(params.bucket!); const bucket = await bucketRepository.findByName(params.bucket!);
if (!bucket) return jsonError('Bucket not found', 404); if (!bucket) return jsonError('Bucket not found', 404);
const file = await findFileByBucketAndKey(bucket.id, params.key!); const file = await fileRepository.findByBucketAndKey(bucket.id, params.key!);
if (!file) return jsonError('Object not found', 404); if (!file) return jsonError('Object not found', 404);
if (file.storageBackend === 'chunked') { if (file.storageBackend === 'chunked') {
const range = { type: 'none' as const }; const range = { type: 'none' as const };
return createChunkedObjectResponse({ file, range, reqId: '' }); return chunkedStorage.createChunkedObjectResponse({ file, range, reqId: '' });
} }
const fileInfo = await botPool.getFileInfo(file.telegramFileId); const fileInfo = await botPool.getFileInfo(file.telegramFileId);
@@ -348,12 +338,12 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro
} }
const destBucketName = body.destBucket || params.bucket!; const destBucketName = body.destBucket || params.bucket!;
const sourceBucket = await findBucketByName(params.bucket!); const sourceBucket = await bucketRepository.findByName(params.bucket!);
const destBucket = await findBucketByName(destBucketName); const destBucket = await bucketRepository.findByName(destBucketName);
if (!sourceBucket || !destBucket) return jsonError('Bucket not found', 404); if (!sourceBucket || !destBucket) return jsonError('Bucket not found', 404);
const sourceFile = await findFileByBucketAndKey(sourceBucket.id, body.sourceKey); const sourceFile = await fileRepository.findByBucketAndKey(sourceBucket.id, body.sourceKey);
if (!sourceFile) return jsonError('Source object not found', 404); if (!sourceFile) return jsonError('Source object not found', 404);
if (sourceFile.storageBackend === 'chunked') { if (sourceFile.storageBackend === 'chunked') {
+89 -7
View File
@@ -1,7 +1,89 @@
export { import { config } from '../../../config/index';
checkRateLimit, import logger from '../../../shared/logger/index';
cleanupRateLimitCache, import { extractClientIp } from '../../../shared/utils/ip';
clearRateLimitCache,
getRateLimitStats, interface RateLimitEntry {
withRateLimit, count: number;
} from '../../../utils/rateLimit'; resetTime: number;
}
const rateLimitStore = new Map<string, RateLimitEntry>();
const MAX_STORE_ENTRIES = 50000;
const evictExpiredEntries = (now = Date.now()): number => {
let cleaned = 0;
for (const [key, entry] of rateLimitStore.entries()) {
if (now > entry.resetTime) {
rateLimitStore.delete(key);
cleaned++;
}
}
return cleaned;
};
const ensureStoreCapacity = (now: number): void => {
if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
evictExpiredEntries(now);
while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
const oldestKey = rateLimitStore.keys().next().value;
if (!oldestKey) break;
rateLimitStore.delete(oldestKey);
}
};
export const checkRateLimit = (key: string): boolean => {
const now = Date.now();
const entry = rateLimitStore.get(key);
if (!entry || now > entry.resetTime) {
ensureStoreCapacity(now);
rateLimitStore.set(key, {
count: 1,
resetTime: now + config.rateLimitWindowMs,
});
return true;
}
if (entry.count >= config.rateLimitMaxRequests) {
logger.warn('Rate limit exceeded', { key, count: entry.count });
return false;
}
entry.count++;
return true;
};
export const withRateLimit = <T extends Request>(
handler: (req: T) => Promise<Response>,
): ((req: T) => Promise<Response>) => {
return async (req: T): Promise<Response> => {
const ip = extractClientIp(req);
if (!checkRateLimit(ip)) {
return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
}
return handler(req);
};
};
export const cleanupRateLimitCache = (): void => {
const cleaned = evictExpiredEntries();
if (cleaned > 0) {
logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
}
};
export const getRateLimitStats = () => ({
trackedIPs: rateLimitStore.size,
windowSize: config.rateLimitWindowMs,
maxRequests: config.rateLimitMaxRequests,
maxTrackedIPs: MAX_STORE_ENTRIES,
});
export const clearRateLimitCache = (): void => {
rateLimitStore.clear();
};
+1 -1
View File
@@ -1,7 +1,7 @@
import { config } from '../../../config/index'; import { config } from '../../../config/index';
import { handleSwaggerHtml, handleSwaggerJson } from '../../../routes/swagger'; import { handleSwaggerHtml, handleSwaggerJson } from '../../../routes/swagger';
import { extractS3BucketFromHost } from '../../../utils/s3/virtual-host';
import { isS3Request } from '../../s3/auth'; import { isS3Request } from '../../s3/auth';
import { extractS3BucketFromHost } from '../../s3/virtual-host';
import { handleLogin, handleLogout, handleMe } from '../controllers/auth-controller'; import { handleLogin, handleLogout, handleMe } from '../controllers/auth-controller';
import { handleFileInfo, handleFileRedirect } from '../controllers/file-controller'; import { handleFileInfo, handleFileRedirect } from '../controllers/file-controller';
import { handleHealth } from '../controllers/health-controller'; import { handleHealth } from '../controllers/health-controller';
+463 -7
View File
@@ -1,7 +1,463 @@
export type { SigV4Result, VerifyPresignedUrlInput } from '../../utils/s3/auth'; import { timingSafeEqual } from 'node:crypto';
export {
buildCanonicalQueryString, /**
isS3Request, * Timing-safe string comparison that prevents timing attacks.
verifyPresignedUrl, *
verifySignature, * Uses `crypto.timingSafeEqual` which runs in constant time regardless of
} from '../../utils/s3/auth'; * where the strings differ. Returns false for mismatched-length inputs
* to avoid leaking length information via early return.
*
* @param left - The first string to compare.
* @param right - The second string to compare.
* @returns True if both strings are equal.
*/
const timingSafeCompare = (left: string, right: string): boolean => {
const leftBuffer = Buffer.from(left);
const rightBuffer = Buffer.from(right);
if (leftBuffer.length !== rightBuffer.length) {
return false;
}
return timingSafeEqual(leftBuffer, rightBuffer);
};
export interface SigV4Result {
isValid: boolean;
credential: {
accessKey: string;
date: string;
region: string;
service: string;
} | null;
errorCode?: string;
}
export interface VerifyPresignedUrlInput {
url: string;
method: string;
headers: Record<string, string>;
s3AccessKey: string;
s3SecretKey: string;
region: string;
now?: Date;
}
const SERVICE = 's3';
const TERMINATION = 'aws4_request';
/**
* Maximum acceptable clock skew between client and server for header-based
* SigV4 authentication. AWS allows 15 minutes.
*/
const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => {
if (data instanceof Uint8Array) return data;
if (data instanceof ArrayBuffer) return new Uint8Array(data);
return new TextEncoder().encode(data);
};
const sha256Hex = async (data: string | Uint8Array | ArrayBuffer): Promise<string> => {
const hashBuffer = await crypto.subtle.digest('SHA-256', buf(data) as never);
const hashArray = Array.from(new Uint8Array(hashBuffer));
return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
};
const hmacSha256 = async (key: Uint8Array, message: string): Promise<Uint8Array> => {
const cryptoKey = await crypto.subtle.importKey(
'raw',
key as never,
{ name: 'HMAC', hash: 'SHA-256' },
false,
['sign'],
);
const result = await crypto.subtle.sign('HMAC', cryptoKey, buf(message) as never);
return new Uint8Array(result);
};
const getSigningKey = async (
secretKey: string,
dateStamp: string,
region: string,
): Promise<Uint8Array> => {
let key = await hmacSha256(buf(`AWS4${secretKey}`), dateStamp);
key = await hmacSha256(key, region);
key = await hmacSha256(key, SERVICE);
return await hmacSha256(key, TERMINATION);
};
const hmacHex = async (key: Uint8Array, message: string): Promise<string> => {
const result = await hmacSha256(key, message);
return Array.from(result)
.map((b) => b.toString(16).padStart(2, '0'))
.join('');
};
const parseAuthorizationHeader = (authHeader: string) => {
const credentialMatch = authHeader.match(/Credential=([^,]+)/);
const signedHeadersMatch = authHeader.match(/SignedHeaders=([^,]+)/);
const signatureMatch = authHeader.match(/Signature=([^,]+)/);
if (!credentialMatch || !signedHeadersMatch || !signatureMatch) return null;
const credentialParts = credentialMatch[1].split('/');
if (credentialParts.length !== 5) return null;
return {
accessKey: credentialParts[0],
date: credentialParts[1],
region: credentialParts[2],
service: credentialParts[3],
termination: credentialParts[4],
signedHeaders: signedHeadersMatch[1],
signature: signatureMatch[1],
};
};
const buildCanonicalRequest = (
method: string,
canonicalUri: string,
canonicalQueryString: string,
signedHeaders: string,
headers: Record<string, string>,
hashedPayload: string,
): string => {
const canonicalHeaders = signedHeaders
.split(';')
.map((h) => {
const value = headers[h.toLowerCase()] || '';
return `${h.toLowerCase()}:${value.trim()}\n`;
})
.join('');
return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`;
};
/**
* Normalizes a URI per AWS SigV4 requirements plus RFC 3986:
*
* 1. Decode percent-encoded characters
* 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4
*
* @param uri - The raw URI path to normalize.
* @returns The normalized URI path.
*/
const normalizeUri = (uri: string): string => {
if (!uri || uri === '') return '/';
// AWS SigV4 requires URI-decoded paths in the canonical request
// Only `.` and `..` segments are removed per RFC 3986 section 5.2.4
// Empty segments (from `//` or trailing `/`) are preserved — they are
// part of the URI and the SDK signs them.
const decoded = decodeURIComponent(uri);
const segments = decoded.split('/');
const result: string[] = [];
for (const segment of segments) {
if (segment === '.') continue;
if (segment === '..') {
result.pop();
continue;
}
result.push(segment);
}
// Join preserves empty first segment (from leading /) automatically
return result.join('/') || '/';
};
const awsEncode = (value: string): string =>
encodeURIComponent(value).replace(
/[!'()*]/g,
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
);
export const buildCanonicalQueryString = (
searchParams: URLSearchParams,
excludeKeys: Set<string> = new Set(),
): string => {
const pairs: Array<[string, string]> = [];
for (const [key, value] of searchParams.entries()) {
if (!excludeKeys.has(key)) pairs.push([key, value]);
}
// AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare
pairs.sort(([ak, av], [bk, bv]) => {
const a = `${awsEncode(ak)}=${awsEncode(av)}`;
const b = `${awsEncode(bk)}=${awsEncode(bv)}`;
if (a < b) return -1;
if (a > b) return 1;
return 0;
});
return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&');
};
const getHashedPayload = async (body: string | null): Promise<string> => {
if (!body || body.length === 0) return await sha256Hex('');
return await sha256Hex(body);
};
/**
* Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date.
*
* @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format.
* @returns The parsed Date, or null if the format is invalid.
*/
const parseAmzDateUtc = (amzDate: string): Date | null => {
const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/);
if (!match) return null;
const [, year, month, day, hour, minute, second] = match;
return new Date(
Date.UTC(
Number.parseInt(year, 10),
Number.parseInt(month, 10) - 1,
Number.parseInt(day, 10),
Number.parseInt(hour, 10),
Number.parseInt(minute, 10),
Number.parseInt(second, 10),
),
);
};
/**
* Validates that `host` is included in the signed headers list.
*
* AWS SigV4 mandates that `host` is always signed. Reject requests that
* omit it to prevent header injection / replay variants.
*
* @param signedHeaders - The semicolon-separated signed headers string.
* @returns True if `host` is present.
*/
const validateSignedHeaders = (signedHeaders: string): boolean => {
return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host');
};
export const verifySignature = async (
method: string,
url: string,
headers: Record<string, string>,
body: string | null,
s3AccessKey: string,
s3SecretKey: string,
region: string,
): Promise<SigV4Result> => {
const authHeader = headers.authorization;
if (!authHeader?.startsWith('AWS4-HMAC-SHA256')) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const parsed = parseAuthorizationHeader(authHeader);
if (!parsed) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
if (!timingSafeCompare(parsed.accessKey, s3AccessKey)) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
if (!timingSafeCompare(parsed.region, region)) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
// Validate service and termination in credential scope (M2)
if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
// Validate host is in signed headers (LOW/host)
if (!validateSignedHeaders(parsed.signedHeaders)) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const parsedUrl = new URL(url, 'http://localhost');
const canonicalUri = normalizeUri(parsedUrl.pathname);
const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams);
const contentSha256 = headers['x-amz-content-sha256'] || null;
if (contentSha256?.startsWith('STREAMING-')) {
return { isValid: false, credential: null, errorCode: 'NotImplemented' };
}
// CRITICAL: Use the x-amz-content-sha256 header value in the canonical
// request because that's what the client signed. The actual body hash is
// verified by verifyBodyHash() after streaming, ensuring integrity without
// breaking SigV4.
const hashedPayload = contentSha256 || (await getHashedPayload(body));
const canonicalRequest = buildCanonicalRequest(
method,
canonicalUri,
canonicalQueryString,
parsed.signedHeaders,
headers,
hashedPayload,
);
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
// M1: Fall back to Date header if x-amz-date is missing
const amzDate = headers['x-amz-date'] || headers.date || '';
// H5: Validate request freshness (clock skew / replay protection)
if (amzDate) {
const requestDate = parseAmzDateUtc(amzDate);
if (requestDate) {
const now = Date.now();
const skew = Math.abs(now - requestDate.getTime());
if (skew > MAX_CLOCK_SKEW_MS) {
return { isValid: false, credential: null, errorCode: 'RequestExpired' };
}
}
}
const dateStamp = parsed.date;
// M3: Ensure date in credential scope matches x-amz-date
if (amzDate) {
const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD"
if (amzDateStamp !== dateStamp) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
}
const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`;
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
const signingKey = await getSigningKey(s3SecretKey, dateStamp, region);
const expectedSignature = await hmacHex(signingKey, stringToSign);
if (!timingSafeCompare(expectedSignature, parsed.signature)) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
return {
isValid: true,
credential: {
accessKey: parsed.accessKey,
date: parsed.date,
region: parsed.region,
service: parsed.service,
},
};
};
export const verifyPresignedUrl = async ({
url,
method,
headers,
s3AccessKey,
s3SecretKey,
region,
now = new Date(),
}: VerifyPresignedUrlInput): Promise<SigV4Result> => {
const parsedUrl = new URL(url);
const searchParams = parsedUrl.searchParams;
const algorithm = searchParams.get('X-Amz-Algorithm');
const credential = searchParams.get('X-Amz-Credential');
const signedHeaders = searchParams.get('X-Amz-SignedHeaders');
const signature = searchParams.get('X-Amz-Signature');
const expiresText = searchParams.get('X-Amz-Expires');
const amzDate = searchParams.get('X-Amz-Date');
if (
algorithm !== 'AWS4-HMAC-SHA256' ||
!credential ||
!signedHeaders ||
!signature ||
!expiresText ||
!amzDate
) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const expires = Number.parseInt(expiresText, 10);
const signedAt = parseAmzDateUtc(amzDate);
if (!Number.isFinite(expires) || expires <= 0 || !signedAt) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
// AWS S3 spec limits presigned URLs to 7 days (604800 seconds)
const MAX_PRESIGNED_EXPIRY_SECONDS = 604800;
if (
now.getTime() > signedAt.getTime() + expires * 1000 ||
expires > MAX_PRESIGNED_EXPIRY_SECONDS
) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const credParts = credential.split('/');
if (credParts.length !== 5) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const [accessKey, dateStamp, credentialRegion, service, termination] = credParts;
if (
!timingSafeCompare(accessKey, s3AccessKey) ||
!timingSafeCompare(credentialRegion, region) ||
service !== SERVICE ||
termination !== TERMINATION
) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
// Validate host is in signed headers for presigned URLs too
if (!validateSignedHeaders(signedHeaders)) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const signedHeaderList = signedHeaders.split(';').filter(Boolean);
const canonicalHeaders = signedHeaderList
.map((headerName) => {
const lower = headerName.toLowerCase();
const value = lower === 'host' ? headers.host || parsedUrl.host : headers[lower] || '';
return `${lower}:${value.trim()}\n`;
})
.join('');
const canonicalRequest = `${method}\n${normalizeUri(parsedUrl.pathname)}\n${buildCanonicalQueryString(searchParams, new Set(['X-Amz-Signature']))}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`;
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
const credentialScope = `${dateStamp}/${region}/${SERVICE}/${TERMINATION}`;
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
const expectedSignature = await hmacHex(
await getSigningKey(s3SecretKey, dateStamp, region),
stringToSign,
);
if (!timingSafeCompare(expectedSignature, signature)) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
return { isValid: true, credential: { accessKey, date: dateStamp, region, service } };
};
export const isS3Request = (headers: Record<string, string>): boolean => {
const auth = headers.authorization || '';
return auth.startsWith('AWS4-HMAC-SHA256');
};
/**
* Verifies that the actual body SHA-256 matches the `x-amz-content-sha256`
* header from the original request.
*
* This MUST be called AFTER the body has been fully streamed and hashed,
* as a second pass after `verifySignature` (which cannot hash a streaming
* body without consuming it).
*
* @param bodySha256 - The SHA-256 hex digest of the actual body content.
* @param headers - The original request headers.
* @returns An error result on mismatch, or null if the check passes.
*/
export const verifyBodyHash = (
bodySha256: string,
headers: Record<string, string>,
): SigV4Result | null => {
const claimedHash = headers['x-amz-content-sha256'];
// If the client sent UNSIGNED-PAYLOAD, skip verification
if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) {
return null;
}
if (claimedHash !== bodySha256) {
return { isValid: false, credential: null, errorCode: 'BadDigest' };
}
return null;
};
+52 -10
View File
@@ -1,10 +1,52 @@
/** import { nanoid } from 'nanoid';
* Re-export from the canonical headers implementation.
* export const S3_CORS_HEADERS: Record<string, string> = {
* @module 'access-control-allow-origin': '*',
*/ 'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS',
export { 'access-control-allow-headers': [
applyS3Headers, 'Authorization',
S3_CORS_HEADERS, 'Content-Type',
s3Headers, 'Content-MD5',
} from '../../utils/s3/headers'; 'Range',
'If-Match',
'If-None-Match',
'If-Modified-Since',
'If-Unmodified-Since',
'X-Amz-*',
'x-amz-*',
].join(', '),
'access-control-expose-headers': [
'Accept-Ranges',
'Content-Length',
'Content-Range',
'Content-Type',
'ETag',
'Last-Modified',
'x-amz-id-2',
'x-amz-request-id',
].join(', '),
'access-control-max-age': '86400',
};
export const s3Headers = (
requestId: string,
extraHeaders: Record<string, string> = {},
): Record<string, string> => ({
...S3_CORS_HEADERS,
server: 'AmazonS3',
...(requestId
? {
'x-amz-request-id': requestId,
'x-amz-id-2': `${requestId}+${nanoid(16)}`,
}
: {}),
...extraHeaders,
});
export const applyS3Headers = (headers: Headers, requestId: string): Headers => {
const result = new Headers(headers);
for (const [key, value] of Object.entries(s3Headers(requestId))) {
result.set(key, value);
}
return result;
};
+27 -1
View File
@@ -1 +1,27 @@
export { extractS3BucketFromHost } from '../../utils/s3/virtual-host'; const stripPort = (host: string): string => {
// Handle IPv6: [::1]:8080 -> [::1]
if (host.startsWith('[')) {
const closeBracket = host.indexOf(']');
return host.slice(0, closeBracket + 1).toLowerCase();
}
return host.split(':')[0].toLowerCase().replace(/\.$/, '');
};
const isValidBucketLabel = (bucket: string): boolean =>
/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) &&
!bucket.includes('..') &&
!bucket.includes('.-') &&
!bucket.includes('-.');
export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => {
const normalizedHost = stripPort(host);
for (const domain of domains) {
const normalizedDomain = stripPort(domain);
if (!normalizedDomain || normalizedHost === normalizedDomain) continue;
if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue;
const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1));
return isValidBucketLabel(bucket) ? bucket : null;
}
return null;
};
+311 -22
View File
@@ -1,22 +1,311 @@
/** import { s3Headers } from './headers';
* Re-export from the canonical XML implementation.
* const escapeXml = (str: string): string =>
* @module str
*/ .replace(/&/g, '&amp;')
export { .replace(/</g, '&lt;')
bucketVersioningConfigurationXml, .replace(/>/g, '&gt;')
type CompletePart, .replace(/"/g, '&quot;')
completeMultipartUploadXml, .replace(/'/g, '&apos;');
copyObjectResultXml,
deleteResultXml, const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z');
initiateMultipartUploadXml,
listBucketResultXml, const encodeKey = (value: string, encodingType: string | null = null): string =>
listBucketsXml, encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value);
listBucketV2ResultXml,
listMultipartUploadsXml, // ─────── Bucket operations ───────
listPartsXml,
parseCompleteMultipartBody, export const listBucketsXml = (
parseDeleteObjectsBody, buckets: { name: string; createdAt: Date }[],
s3ErrorResponse, _requestId: string,
s3ErrorXml, ): string => `<?xml version="1.0" encoding="UTF-8"?>
} from '../../utils/s3/xml'; <ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Buckets>
${buckets
.map(
(b) => `<Bucket>
<Name>${escapeXml(b.name)}</Name>
<CreationDate>${isoDate(b.createdAt)}</CreationDate>
</Bucket>`,
)
.join('')}
</Buckets>
</ListAllMyBucketsResult>`;
export const bucketVersioningConfigurationXml =
(): string => `<?xml version="1.0" encoding="UTF-8"?>
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`;
// ─────── Object listing ───────
export const listBucketResultXml = (
bucketName: string,
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
prefixes: string[],
isTruncated: boolean,
marker: string | null,
maxKeys: number,
prefix: string,
delimiter: string | null,
nextMarker: string | null,
_requestId: string,
encodingType: string | null = null,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>${escapeXml(bucketName)}</Name>
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
<Marker>${encodeKey(marker || '', encodingType)}</Marker>
<MaxKeys>${maxKeys}</MaxKeys>
<Delimiter>${encodeKey(delimiter || '', encodingType)}</Delimiter>
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
<IsTruncated>${isTruncated}</IsTruncated>
${objects
.map(
(o) => `<Contents>
<Key>${encodeKey(o.key, encodingType)}</Key>
<LastModified>${isoDate(o.lastModified)}</LastModified>
<ETag>"${o.etag}"</ETag>
<Size>${o.sizeBytes}</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>`,
)
.join('')}
${prefixes
.map(
(p) => `<CommonPrefixes>
<Prefix>${encodeKey(p, encodingType)}</Prefix>
</CommonPrefixes>`,
)
.join('')}
${nextMarker ? `<NextMarker>${encodeKey(nextMarker, encodingType)}</NextMarker>` : ''}
</ListBucketResult>`;
export const listBucketV2ResultXml = (
bucketName: string,
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
prefixes: string[],
isTruncated: boolean,
maxKeys: number,
prefix: string,
delimiter: string | null,
continuationToken: string | null,
nextContinuationToken: string | null,
keyCount: number,
_requestId: string,
encodingType: string | null = null,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>${escapeXml(bucketName)}</Name>
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
<MaxKeys>${maxKeys}</MaxKeys>
<KeyCount>${keyCount}</KeyCount>
${delimiter ? `<Delimiter>${encodeKey(delimiter, encodingType)}</Delimiter>` : ''}
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
${continuationToken ? `<ContinuationToken>${encodeKey(continuationToken, encodingType)}</ContinuationToken>` : ''}
<IsTruncated>${isTruncated}</IsTruncated>
${objects
.map(
(o) => `<Contents>
<Key>${encodeKey(o.key, encodingType)}</Key>
<LastModified>${isoDate(o.lastModified)}</LastModified>
<ETag>"${o.etag}"</ETag>
<Size>${o.sizeBytes}</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>`,
)
.join('')}
${prefixes
.map(
(p) => `<CommonPrefixes>
<Prefix>${encodeKey(p, encodingType)}</Prefix>
</CommonPrefixes>`,
)
.join('')}
${nextContinuationToken ? `<NextContinuationToken>${encodeKey(nextContinuationToken, encodingType)}</NextContinuationToken>` : ''}
</ListBucketResultV2>`;
// ─────── Multipart ───────
export const initiateMultipartUploadXml = (
bucketName: string,
key: string,
uploadId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>${escapeXml(bucketName)}</Bucket>
<Key>${escapeXml(key)}</Key>
<UploadId>${uploadId}</UploadId>
</InitiateMultipartUploadResult>`;
export const listPartsXml = (
bucketName: string,
key: string,
uploadId: string,
parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[],
maxParts: number,
isTruncated: boolean,
_requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>${escapeXml(bucketName)}</Bucket>
<Key>${escapeXml(key)}</Key>
<UploadId>${uploadId}</UploadId>
<MaxParts>${maxParts}</MaxParts>
<IsTruncated>${isTruncated}</IsTruncated>
${parts
.map(
(p) => `<Part>
<PartNumber>${p.partNumber}</PartNumber>
<LastModified>${isoDate(p.createdAt)}</LastModified>
<ETag>"${p.etag}"</ETag>
<Size>${p.sizeBytes}</Size>
</Part>`,
)
.join('')}
</ListPartsResult>`;
export const listMultipartUploadsXml = (
bucketName: string,
uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[],
maxUploads: number,
isTruncated: boolean,
nextKeyMarker: string | null,
_requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>${escapeXml(bucketName)}</Bucket>
<KeyMarker></KeyMarker>
<UploadIdMarker></UploadIdMarker>
${nextKeyMarker ? `<NextKeyMarker>${escapeXml(nextKeyMarker)}</NextKeyMarker>` : ''}
<MaxUploads>${maxUploads}</MaxUploads>
<IsTruncated>${isTruncated}</IsTruncated>
${uploads
.map(
(u) => `<Upload>
<Key>${escapeXml(u.key)}</Key>
<UploadId>${u.uploadId}</UploadId>
<Initiator><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Initiator>
<Owner><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Owner>
<StorageClass>STANDARD</StorageClass>
<Initiated>${isoDate(u.initiatedAt)}</Initiated>
</Upload>`,
)
.join('')}
</ListMultipartUploadsResult>`;
export const completeMultipartUploadXml = (
bucketName: string,
key: string,
etag: string,
location: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Location>${escapeXml(location)}</Location>
<Bucket>${escapeXml(bucketName)}</Bucket>
<Key>${escapeXml(key)}</Key>
<ETag>"${etag}"</ETag>
</CompleteMultipartUploadResult>`;
// ─────── Delete result ───────
export const deleteResultXml = (
deleted: string[],
errors: { key: string; code: string; message: string }[],
): string => `<?xml version="1.0" encoding="UTF-8"?>
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
${deleted
.map(
(key) => `<Deleted>
<Key>${escapeXml(key)}</Key>
</Deleted>`,
)
.join('')}
${errors
.map(
(e) => `<Error>
<Key>${escapeXml(e.key)}</Key>
<Code>${e.code}</Code>
<Message>${escapeXml(e.message)}</Message>
</Error>`,
)
.join('')}
</DeleteResult>`;
// ─────── Copy ───────
export const copyObjectResultXml = (
etag: string,
lastModified: Date,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<ETag>"${etag}"</ETag>
<LastModified>${isoDate(lastModified)}</LastModified>
</CopyObjectResult>`;
// ─────── Error ───────
export const s3ErrorXml = (
code: string,
message: string,
resource: string,
requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>${code}</Code>
<Message>${escapeXml(message)}</Message>
<Resource>${escapeXml(resource)}</Resource>
<RequestId>${requestId}</RequestId>
<HostId>${requestId}</HostId>
</Error>`;
export const s3ErrorResponse = (
code: string,
message: string,
resource: string,
status: number,
requestId: string = '',
extraHeaders: Record<string, string> = {},
): Response =>
new Response(s3ErrorXml(code, message, resource, requestId), {
status,
headers: s3Headers(requestId, {
'content-type': 'application/xml',
...extraHeaders,
}),
});
// ─────── DeleteObjects XML parser ───────
export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => {
// H9: Use non-greedy match to handle keys containing < character
const keys = Array.from(body.matchAll(/<Key>([\s\S]*?)<\/Key>/g), (match) => match[1]);
// Handle whitespace inside <Quiet> element + namespace prefix support
const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body);
return { keys, quiet };
};
// ─────── CompleteMultipartUpload XML parser ───────
export interface CompletePart {
partNumber: number;
etag: string;
}
export const parseCompleteMultipartBody = (body: string): CompletePart[] => {
const parts: CompletePart[] = [];
const partRegex = /<Part>[\s\S]*?<\/Part>/g;
const partMatch = body.match(partRegex) || [];
for (const partXml of partMatch) {
const numMatch = partXml.match(/<PartNumber>(\d+)<\/PartNumber>/);
const etagMatch = partXml.match(/<ETag>"?([^"<\s]+)"?<\/ETag>/);
if (numMatch && etagMatch) {
parts.push({
partNumber: Number.parseInt(numMatch[1], 10),
etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''),
});
}
}
return parts;
};
+26 -4
View File
@@ -1,4 +1,26 @@
import _logger from '../../utils/logger'; import winston from 'winston';
export default _logger;
export type { Logger } from 'winston'; const logger = winston.createLogger({
export { _logger as logger }; level: process.env.LOG_LEVEL || 'info',
format: winston.format.combine(
winston.format.timestamp(),
winston.format.errors({ stack: true }),
winston.format.json(),
),
defaultMeta: { service: 'filedrop' },
transports: [
// Write all logs including error logs to file
new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'logs/combined.log' }),
// Console transport for docker logs / CLI visibility
new winston.transports.Console({
format:
process.env.NODE_ENV !== 'production'
? winston.format.combine(winston.format.colorize(), winston.format.simple())
: winston.format.json(),
}),
],
});
export default logger;
export { logger };
+111 -1
View File
@@ -1 +1,111 @@
export { MetricsCollector, metricsCollector } from '../../utils/metrics'; // No imports needed — logger used only by setInterval which moved to index.ts
interface Metric {
name: string;
value: number;
timestamp: number;
tags?: Record<string, string>;
}
interface MetricsSnapshot {
uploadLatency: { p50: number; p95: number; p99: number };
uploadThroughput: number;
queueSize: number;
errorRate: number;
cacheHitRate: number;
botUtilization: number;
timestamp: number;
}
class MetricsCollector {
private metrics: Metric[] = [];
private uploadTimes: number[] = [];
private errorCount = 0;
private totalRequests = 0;
private cacheHits = 0;
private cacheMisses = 0;
private maxMetricsSize = 10000;
recordUploadTime(durationMs: number): void {
this.uploadTimes.push(durationMs);
this.totalRequests++;
// Keep only last 1000 measurements
if (this.uploadTimes.length > 1000) {
this.uploadTimes.shift();
}
}
recordError(): void {
this.errorCount++;
}
recordCacheHit(): void {
this.cacheHits++;
}
recordCacheMiss(): void {
this.cacheMisses++;
}
recordMetric(name: string, value: number, tags?: Record<string, string>): void {
this.metrics.push({
name,
value,
timestamp: Date.now(),
tags,
});
// Keep metrics bounded
if (this.metrics.length > this.maxMetricsSize) {
this.metrics = this.metrics.slice(-this.maxMetricsSize);
}
}
private calculatePercentile(arr: number[], percentile: number): number {
if (arr.length === 0) return 0;
const sorted = [...arr].sort((a, b) => a - b);
const index = Math.ceil((percentile / 100) * sorted.length) - 1;
return sorted[Math.max(0, index)];
}
getSnapshot(): MetricsSnapshot {
const errorRate = this.totalRequests > 0 ? (this.errorCount / this.totalRequests) * 100 : 0;
const cacheHitRate =
this.cacheHits + this.cacheMisses > 0
? (this.cacheHits / (this.cacheHits + this.cacheMisses)) * 100
: 0;
return {
uploadLatency: {
p50: this.calculatePercentile(this.uploadTimes, 50),
p95: this.calculatePercentile(this.uploadTimes, 95),
p99: this.calculatePercentile(this.uploadTimes, 99),
},
uploadThroughput: this.totalRequests > 0 ? this.totalRequests / 60 : 0,
queueSize: 0, // Will be updated by queue
errorRate,
cacheHitRate,
botUtilization: 0, // Will be updated by bot tracker
timestamp: Date.now(),
};
}
reset(): void {
this.uploadTimes = [];
this.errorCount = 0;
this.totalRequests = 0;
this.cacheHits = 0;
this.cacheMisses = 0;
this.metrics = [];
}
getMetrics(name?: string): Metric[] {
if (!name) return this.metrics;
return this.metrics.filter((m) => m.name === name);
}
}
export const metricsCollector = new MetricsCollector();
export { MetricsCollector };
+1 -1
View File
@@ -1,5 +1,5 @@
import { unlink } from 'node:fs/promises'; import { unlink } from 'node:fs/promises';
import logger from '../../utils/logger'; import logger from '../logger/index';
/** /**
* Safely extracts an error message from an unknown value. * Safely extracts an error message from an unknown value.
+1 -1
View File
@@ -1,4 +1,4 @@
import logger from '../../utils/logger'; import logger from '../logger/index';
/** Configuration options for retry behaviour. */ /** Configuration options for retry behaviour. */
interface RetryOptions { interface RetryOptions {
-205
View File
@@ -1,205 +0,0 @@
import { createHmac, timingSafeEqual } from 'node:crypto';
import { config } from '../env';
const ADMIN_USERNAME = 'admin';
const SIGNATURE_SEPARATOR = '.';
type Handler = (req: Request) => Response | Promise<Response>;
export interface AuthSession {
username: string;
expiresAt: Date | null;
method: 'cookie' | 'bearer';
}
interface CookieOptions {
secret?: string;
cookieName?: string;
maxAgeMs?: number;
}
interface SessionPayload {
u: string;
e: number;
}
const getSecret = (secret?: string): string => secret ?? config.adminApiToken;
const getCookieName = (cookieName?: string): string => cookieName ?? config.sessionCookieName;
const getMaxAgeMs = (maxAgeMs?: number): number => maxAgeMs ?? config.sessionMaxAgeMs;
const encodePayload = (value: string): string => Buffer.from(value, 'utf8').toString('base64url');
const decodePayload = (value: string): string | null => {
try {
return Buffer.from(value, 'base64url').toString('utf8');
} catch {
return null;
}
};
export const isAuthEnabled = (secret = config.adminApiToken): boolean => secret.length > 0;
export const timingSafeCompare = (left: string, right: string): boolean => {
const leftBuffer = Buffer.from(left);
const rightBuffer = Buffer.from(right);
if (leftBuffer.length !== rightBuffer.length) {
return false;
}
return timingSafeEqual(leftBuffer, rightBuffer);
};
export const signCookiePayload = (payload: string, secret: string): string =>
createHmac('sha256', secret).update(payload).digest('base64url');
export const verifyCookieSignature = (cookieValue: string, secret: string): string | null => {
const separatorIndex = cookieValue.lastIndexOf(SIGNATURE_SEPARATOR);
if (separatorIndex <= 0 || separatorIndex === cookieValue.length - 1) {
return null;
}
const payload = cookieValue.slice(0, separatorIndex);
const signature = cookieValue.slice(separatorIndex + 1);
const expectedSignature = signCookiePayload(payload, secret);
if (!timingSafeCompare(signature, expectedSignature)) {
return null;
}
return payload;
};
const cookieAttributes = (maxAgeSeconds: number): string =>
[`Max-Age=${maxAgeSeconds}`, 'Path=/', 'HttpOnly', 'SameSite=Lax', 'Secure'].join('; ');
export const createSessionCookie = (
username = ADMIN_USERNAME,
options: CookieOptions = {},
): string => {
const secret = getSecret(options.secret);
const cookieName = getCookieName(options.cookieName);
const maxAgeMs = getMaxAgeMs(options.maxAgeMs);
const expiresAt = Date.now() + maxAgeMs;
const payload = encodePayload(
JSON.stringify({ u: username, e: expiresAt } satisfies SessionPayload),
);
const signature = signCookiePayload(payload, secret);
const maxAgeSeconds = Math.max(1, Math.floor(maxAgeMs / 1000));
return `${cookieName}=${payload}${SIGNATURE_SEPARATOR}${signature}; ${cookieAttributes(maxAgeSeconds)}`;
};
export const clearSessionCookie = (cookieName = config.sessionCookieName): string =>
`${cookieName}=; ${cookieAttributes(0)}`;
const findCookieValue = (cookieHeader: string | null, cookieName: string): string | null => {
if (!cookieHeader) return null;
for (const rawCookie of cookieHeader.split(';')) {
const cookie = rawCookie.trim();
const equalsIndex = cookie.indexOf('=');
if (equalsIndex <= 0) continue;
const name = cookie.slice(0, equalsIndex);
if (name === cookieName) {
return cookie.slice(equalsIndex + 1);
}
}
return null;
};
export const parseSessionFromCookie = (
cookieHeader: string | null,
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
): AuthSession | null => {
const secret = getSecret(options.secret);
const cookieName = getCookieName(options.cookieName);
if (!isAuthEnabled(secret)) return null;
const cookieValue = findCookieValue(cookieHeader, cookieName);
if (!cookieValue) return null;
const encodedPayload = verifyCookieSignature(cookieValue, secret);
if (!encodedPayload) return null;
const rawPayload = decodePayload(encodedPayload);
if (!rawPayload) return null;
try {
const payload = JSON.parse(rawPayload) as Partial<SessionPayload>;
if (payload.u !== ADMIN_USERNAME || typeof payload.e !== 'number') return null;
if (!Number.isFinite(payload.e) || payload.e <= Date.now()) return null;
return {
username: payload.u,
expiresAt: new Date(payload.e),
method: 'cookie',
};
} catch {
return null;
}
};
export const checkBearerToken = (
authorizationHeader: string | null,
secret = config.adminApiToken,
): boolean => {
if (!isAuthEnabled(secret) || !authorizationHeader) return false;
const [scheme, ...rest] = authorizationHeader.split(' ');
if (scheme !== 'Bearer' || rest.length === 0) return false;
const token = rest.join(' ').trim();
return token.length > 0 && timingSafeCompare(token, secret);
};
export const getAuthSession = (
req: Request,
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
): AuthSession | null => {
const secret = getSecret(options.secret);
if (!isAuthEnabled(secret)) {
return {
username: ADMIN_USERNAME,
expiresAt: null,
method: 'bearer',
};
}
const cookieSession = parseSessionFromCookie(req.headers.get('cookie'), options);
if (cookieSession) return cookieSession;
if (checkBearerToken(req.headers.get('authorization'), secret)) {
return {
username: ADMIN_USERNAME,
expiresAt: null,
method: 'bearer',
};
}
return null;
};
export const unauthorizedResponse = (): Response =>
Response.json({ error: 'Unauthorized' }, { status: 401 });
export const requireAuth = (
handler: Handler,
options: Pick<CookieOptions, 'secret' | 'cookieName'> = {},
): ((req: Request) => Promise<Response>) => {
return async (req: Request): Promise<Response> => {
const secret = getSecret(options.secret);
if (!isAuthEnabled(secret)) {
return handler(req);
}
const session = getAuthSession(req, options);
if (!session) {
return unauthorizedResponse();
}
return handler(req);
};
};
-74
View File
@@ -1,74 +0,0 @@
// Simple in-memory cache with TTL support
interface CacheEntry<T> {
value: T;
expiresAt: number;
}
class Cache<T> {
private store = new Map<string, CacheEntry<T>>();
private ttlMs: number;
constructor(ttlSeconds: number = 3600) {
this.ttlMs = ttlSeconds * 1000;
}
set(key: string, value: T): void {
this.store.set(key, {
value,
expiresAt: Date.now() + this.ttlMs,
});
}
get(key: string): T | null {
const entry = this.store.get(key);
if (!entry) return null;
if (Date.now() > entry.expiresAt) {
this.store.delete(key);
return null;
}
return entry.value;
}
has(key: string): boolean {
return this.get(key) !== null;
}
delete(key: string): void {
this.store.delete(key);
}
clear(): void {
this.store.clear();
}
size(): number {
return this.store.size;
}
// Cleanup expired entries
cleanup(): number {
let removed = 0;
const now = Date.now();
for (const [key, entry] of this.store.entries()) {
if (now > entry.expiresAt) {
this.store.delete(key);
removed++;
}
}
return removed;
}
}
// File info cache (1 hour TTL)
export const fileInfoCache = new Cache<{
file_size: number;
mime_type: string;
file_path: string;
bot_token: string;
}>(3600);
export { Cache };
-249
View File
@@ -1,249 +0,0 @@
import { randomUUID } from 'node:crypto';
import { createReadStream } from 'node:fs';
import { gzipSync } from 'node:zlib';
import { nanoid } from 'nanoid';
import { db, files as fileSchema } from '../db';
import { insertFileParts, listFileParts, type NewFilePartInput } from '../db/file-parts';
import type { File } from '../db/schema';
import { config } from '../env';
import { botPool } from '../infrastructure/telegram/bot-pool';
import { computeHash } from './file';
import { createGetObjectResponse, type ObjectPartSource } from './s3/object-stream';
import type { RangeParseResult } from './s3/range';
export type ChunkCompressionAlgorithm = 'gzip' | null;
export interface ChunkedUploadPart {
partNumber: number;
telegramFileId: string;
telegramFileUniqueId: string;
storageMessageId: number;
sizeBytes: number;
storedSizeBytes: number;
compressionAlgorithm: ChunkCompressionAlgorithm;
etag: string;
}
export interface ChunkedUploadResult {
parts: ChunkedUploadPart[];
fileHash: string;
totalSizeBytes: number;
}
export interface ChunkedFileInput {
tempPath: string;
partFileNamePrefix: string;
fileName: string;
mimeType: string;
sizeBytes: number;
fileType: string;
uploaderId: number;
bucketId?: string | null;
s3Key?: string | null;
}
const asSafeChunkSize = (chunkSizeBytes: number): number => {
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
throw new Error('Invalid Telegram chunk size');
}
return chunkSizeBytes;
};
const maybeCompressChunk = (
chunk: Buffer,
compress: boolean,
compressionMinSizeBytes: number,
): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => {
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
return { bytes: chunk, compressionAlgorithm: null };
}
const gzipped = gzipSync(chunk);
if (gzipped.byteLength >= chunk.byteLength) {
return { bytes: chunk, compressionAlgorithm: null };
}
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
};
export const uploadFileInTelegramChunks = async (input: {
tempPath: string;
partFileNamePrefix: string;
chunkSizeBytes: number;
compress: boolean;
compressionMinSizeBytes: number;
}): Promise<ChunkedUploadResult> => {
const chunkSizeBytes = asSafeChunkSize(input.chunkSizeBytes);
const hasher = new Bun.CryptoHasher('sha256');
const parts: ChunkedUploadPart[] = [];
let totalSizeBytes = 0;
let partNumber = 0;
const stream = createReadStream(input.tempPath, { highWaterMark: chunkSizeBytes });
// Concurrent upload set: tracks in-flight uploads and limits how many
// chunks are being uploaded at once from this single file.
const inFlight = new Set<Promise<void>>();
for await (const data of stream) {
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data as Uint8Array);
if (chunk.byteLength === 0) continue;
partNumber += 1;
totalSizeBytes += chunk.byteLength;
hasher.update(chunk);
const { bytes, compressionAlgorithm } = maybeCompressChunk(
chunk,
input.compress,
input.compressionMinSizeBytes,
);
const currentPart = partNumber;
// Fire upload concurrently — don't await inside the read loop
const uploadPromise = botPool
.forwardToStorage(bytes, `${input.partFileNamePrefix}.part-${currentPart}`, 'document')
.then((forwardResult) => {
parts.push({
partNumber: currentPart,
telegramFileId: forwardResult.telegramFileId,
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
storageMessageId: forwardResult.storageMessageId,
sizeBytes: chunk.byteLength,
storedSizeBytes: bytes.byteLength,
compressionAlgorithm,
etag: computeHash(chunk),
});
});
// Clean up from in-flight set when done (regardless of success/failure)
const trackPromise = uploadPromise.finally(() => {
inFlight.delete(trackPromise);
});
inFlight.add(trackPromise);
// Backpressure: if too many chunks are in-flight, wait for one to
// finish before reading more — prevents unbounded memory growth.
if (inFlight.size >= botPool.getEffectiveConcurrency() * 2) {
await Promise.race(inFlight);
// Yield microtask to let .finally() run and remove from inFlight
await new Promise((resolve) => setTimeout(resolve, 0));
}
}
// Wait for all remaining uploads to finish
await Promise.all(inFlight);
return {
parts,
fileHash: hasher.digest('hex'),
totalSizeBytes,
};
};
export const storeFileInTelegramChunks = async (input: ChunkedFileInput): Promise<File> => {
const upload = await uploadFileInTelegramChunks({
tempPath: input.tempPath,
partFileNamePrefix: input.partFileNamePrefix,
chunkSizeBytes: config.telegramChunkSizeBytes,
compress: config.compressChunkedUploads,
compressionMinSizeBytes: config.chunkCompressionMinSizeBytes,
});
const firstPart = upload.parts[0];
if (!firstPart) {
throw new Error('Chunked upload produced no parts');
}
const now = new Date();
const fileId = randomUUID();
const publicId = nanoid();
const file: File = {
id: fileId,
publicId,
telegramFileId: firstPart.telegramFileId,
telegramFileUniqueId: firstPart.telegramFileUniqueId,
storageChatId: config.storageChatId,
storageMessageId: firstPart.storageMessageId,
fileName: input.fileName,
mimeType: input.mimeType,
sizeBytes: upload.totalSizeBytes,
fileType: input.fileType,
uploaderId: input.uploaderId,
fileHash: upload.fileHash,
archiveTelegramFileId: null,
archiveStorageMessageId: null,
archiveFileName: null,
archiveEntryName: null,
archiveMimeType: null,
archiveSizeBytes: null,
bucketId: input.bucketId ?? null,
s3Key: input.s3Key ?? null,
storageBackend: 'chunked',
isDeleted: false,
multipartUploadId: null,
partCount: upload.parts.length,
createdAt: now,
updatedAt: now,
};
await db.insert(fileSchema).values(file);
const fileParts: NewFilePartInput[] = upload.parts.map((part) => ({
fileId,
partNumber: part.partNumber,
telegramFileId: part.telegramFileId,
telegramFileUniqueId: part.telegramFileUniqueId,
storageChatId: config.storageChatId,
storageMessageId: part.storageMessageId,
sizeBytes: part.sizeBytes,
storedSizeBytes: part.storedSizeBytes,
compressionAlgorithm: part.compressionAlgorithm,
etag: part.etag,
}));
await insertFileParts(fileParts);
return file;
};
export const buildChunkedObjectSources = async (file: File): Promise<ObjectPartSource[]> => {
const parts = await listFileParts(file.id);
const sources: ObjectPartSource[] = [];
for (const part of parts) {
const fileInfo = await botPool.getFileInfo(part.telegramFileId);
sources.push({
telegramFileId: part.telegramFileId,
telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
sizeBytes: part.sizeBytes,
storedSizeBytes: part.storedSizeBytes,
compressionAlgorithm: part.compressionAlgorithm,
partNumber: part.partNumber,
});
}
return sources;
};
export const createChunkedObjectResponse = async (input: {
file: File;
range: RangeParseResult;
reqId: string;
}): Promise<Response> => {
const parts = await buildChunkedObjectSources(input.file);
if (parts.length === 0) {
throw new Error('Chunked object has no parts');
}
return createGetObjectResponse({
reqId: input.reqId,
contentType: input.file.mimeType,
etag: input.file.fileHash || parts.map((p) => p.telegramFileId).join('-'),
lastModified:
input.file.createdAt instanceof Date ? input.file.createdAt : new Date(input.file.createdAt),
totalSize: Number(input.file.sizeBytes),
parts,
range: input.range,
});
};
-237
View File
@@ -1,237 +0,0 @@
import { unlink } from 'node:fs/promises';
import logger from './logger';
export const getErrorMessage = (error: unknown): string => {
return error instanceof Error ? error.message : String(error);
};
export const cleanupTempFile = async (tempPath: string): Promise<void> => {
try {
await unlink(tempPath);
} catch (err) {
logger.warn('Failed to cleanup temp file', { tempPath, error: getErrorMessage(err) });
}
};
interface FileMetadata {
publicId: string;
telegramFileId: string;
telegramFileUniqueId: string;
storageChatId: number;
storageMessageId: number;
fileName: string;
mimeType: string;
sizeBytes: number;
fileType: string;
uploaderId: number;
createdAt: Date | string | number;
}
const FILE_TYPES: Record<string, number> = {
document: 2 * 1024 * 1024 * 1024, // 2GB
photo: 10 * 1024 * 1024, // 10MB
video: 2 * 1024 * 1024 * 1024, // 2GB
audio: 200 * 1024 * 1024, // 200MB
voice: 200 * 1024 * 1024, // 200MB
animation: 2 * 1024 * 1024 * 1024, // 2GB
sticker: 10 * 1024 * 1024, // 10MB
video_note: 2 * 1024 * 1024 * 1024, // 2GB
};
export const getFileType = (mime: string | null, caption?: string): string => {
const mimeUpper = mime?.split('/')[0]?.toLowerCase();
const captionLower = caption?.toLowerCase();
if (mime?.toLowerCase() === 'image/webp' || captionLower?.includes('sticker')) return 'sticker';
if (captionLower?.includes('video_note')) return 'video_note';
if (mimeUpper === 'video') return 'video';
if (mimeUpper === 'audio') return 'audio';
if (mimeUpper === 'document') return 'document';
if (mimeUpper === 'image') return captionLower?.includes('gif') ? 'animation' : 'photo';
if (captionLower?.includes('voice')) return 'voice';
if (captionLower?.includes('animation')) return 'animation';
return mimeUpper === 'application' ? 'application' : 'document';
};
export const checkFileSize = (sizeBytes: number, fileType: string): boolean => {
const limit = FILE_TYPES[fileType] || FILE_TYPES.document;
return sizeBytes <= limit;
};
export const ensureExtension = (
fileName: string,
buffer: Buffer,
detectedMime?: string,
): { fileName: string; mimeType: string } => {
const mimeMap: Record<string, string> = {
'application/pdf': 'pdf',
'image/png': 'png',
'image/jpeg': 'jpg',
'image/gif': 'gif',
'text/plain': 'txt',
'application/zip': 'zip',
};
let ext: string | null = null;
if (buffer.subarray(0, 4).toString() === '%PDF') {
ext = 'pdf';
} else if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4e && buffer[3] === 0x47) {
ext = 'png';
} else if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
ext = 'jpg';
} else if (buffer.subarray(0, 4).toString() === 'GIF8') {
ext = 'gif';
} else if (detectedMime) {
ext = mimeMap[detectedMime.toLowerCase()] || null;
}
let finalFileName = fileName;
const hasExtension = fileName.includes('.') && fileName.split('.').pop()!.length >= 2;
if (!hasExtension && ext) {
finalFileName = `${fileName}.${ext}`;
}
const mimeType = ext
? Object.keys(mimeMap).find((k) => mimeMap[k] === ext) ||
detectedMime ||
'application/octet-stream'
: detectedMime || 'application/octet-stream';
return { fileName: finalFileName, mimeType };
};
type HeaderMapRequest = {
headers?:
| {
get?: (name: string) => string | null;
}
| Record<string, string>;
};
type FileLike = {
fileName?: string;
mimeType?: string;
};
type MessageLike = {
document?: FileLike;
photo?: FileLike[];
audio?: FileLike;
voice?: FileLike;
animation?: FileLike;
};
const getHeader = (request: HeaderMapRequest | null, name: string): string | undefined => {
const headers = request?.headers;
if (!headers) return undefined;
const get = 'get' in headers ? headers.get : undefined;
if (typeof get === 'function') return get(name) || undefined;
return (headers as Record<string, string>)[name];
};
export const extractFileName = (msg: MessageLike, request: HeaderMapRequest | null): string => {
const headerFileName = getHeader(request, 'x-file-name');
if (headerFileName) return headerFileName;
return (
msg.document?.fileName ||
msg.photo?.slice(-1)[0]?.fileName ||
msg.audio?.fileName ||
msg.voice?.fileName ||
msg.animation?.fileName ||
'file'
);
};
export const extractMimeType = (msg: MessageLike, request: HeaderMapRequest | null): string => {
const headerMimeType = getHeader(request, 'x-mime-type');
if (headerMimeType) return headerMimeType;
return (
msg.document?.mimeType ||
msg.photo?.slice(-1)[0]?.mimeType ||
msg.audio?.mimeType ||
msg.voice?.mimeType ||
msg.animation?.mimeType ||
'application/octet-stream'
);
};
export const computeHash = (buffer: Buffer): string => {
const hasher = new Bun.CryptoHasher('sha256');
hasher.update(buffer);
return hasher.digest('hex');
};
export interface TelegramMessageFile {
file_id: string;
file_unique_id: string;
file_size?: number;
mime_type?: string;
file_name?: string;
}
export interface TelegramMediaMessage {
message_id: number;
document?: TelegramMessageFile;
photo?: TelegramMessageFile[];
video?: TelegramMessageFile;
audio?: TelegramMessageFile;
voice?: TelegramMessageFile;
animation?: TelegramMessageFile;
sticker?: TelegramMessageFile;
video_note?: TelegramMessageFile;
}
export const extractFileFromMessage = (
msg: TelegramMediaMessage,
fileType: string,
): TelegramMessageFile => {
if (fileType === 'photo') return msg.photo?.slice(-1)[0] as TelegramMessageFile;
if (fileType === 'sticker') return msg.sticker as TelegramMessageFile;
return msg[fileType as keyof TelegramMediaMessage] as TelegramMessageFile;
};
export const detectFileType = (msg: TelegramMediaMessage): string => {
if (msg.document) return 'document';
if (msg.photo) return 'photo';
if (msg.video) return 'video';
if (msg.audio) return 'audio';
if (msg.voice) return 'voice';
if (msg.animation) return 'animation';
if (msg.sticker) return 'sticker';
if (msg.video_note) return 'video_note';
return 'document';
};
export const getFileSizeLimit = (fileType: string): number =>
FILE_TYPES[fileType] || FILE_TYPES.document;
export const formatCreatedAt = (createdAt: Date | string | number): string => {
return createdAt instanceof Date ? createdAt.toISOString() : new Date(createdAt).toISOString();
};
export interface UploadResponse {
public_id: string;
file_name: string;
mime_type: string;
size_bytes: number;
file_type: string;
created_at: string;
download_url: string;
}
export const buildUploadResponse = (file: FileMetadata, baseUrl: string): UploadResponse => {
return {
public_id: file.publicId,
file_name: file.fileName,
mime_type: file.mimeType,
size_bytes: file.sizeBytes,
file_type: file.fileType,
created_at: formatCreatedAt(file.createdAt),
download_url: `${baseUrl}/f/${file.publicId}`,
};
};
-16
View File
@@ -1,16 +0,0 @@
import { config } from '../env';
export const extractClientIp = (req: Request): string => {
if (!config.trustProxy) return '127.0.0.1';
const forwardedFor = req.headers.get('x-forwarded-for');
if (forwardedFor) {
const firstIp = forwardedFor.split(',')[0]?.trim();
if (firstIp) return firstIp;
}
const realIp = req.headers.get('x-real-ip')?.trim();
if (realIp) return realIp;
return '127.0.0.1';
};
-25
View File
@@ -1,25 +0,0 @@
import winston from 'winston';
const logger = winston.createLogger({
level: process.env.LOG_LEVEL || 'info',
format: winston.format.combine(
winston.format.timestamp(),
winston.format.errors({ stack: true }),
winston.format.json(),
),
defaultMeta: { service: 'filedrop' },
transports: [
// Write all logs including error logs to file
new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'logs/combined.log' }),
// Console transport for docker logs / CLI visibility
new winston.transports.Console({
format:
process.env.NODE_ENV !== 'production'
? winston.format.combine(winston.format.colorize(), winston.format.simple())
: winston.format.json(),
}),
],
});
export default logger;
-111
View File
@@ -1,111 +0,0 @@
// No imports needed — logger used only by setInterval which moved to index.ts
interface Metric {
name: string;
value: number;
timestamp: number;
tags?: Record<string, string>;
}
interface MetricsSnapshot {
uploadLatency: { p50: number; p95: number; p99: number };
uploadThroughput: number;
queueSize: number;
errorRate: number;
cacheHitRate: number;
botUtilization: number;
timestamp: number;
}
class MetricsCollector {
private metrics: Metric[] = [];
private uploadTimes: number[] = [];
private errorCount = 0;
private totalRequests = 0;
private cacheHits = 0;
private cacheMisses = 0;
private maxMetricsSize = 10000;
recordUploadTime(durationMs: number): void {
this.uploadTimes.push(durationMs);
this.totalRequests++;
// Keep only last 1000 measurements
if (this.uploadTimes.length > 1000) {
this.uploadTimes.shift();
}
}
recordError(): void {
this.errorCount++;
}
recordCacheHit(): void {
this.cacheHits++;
}
recordCacheMiss(): void {
this.cacheMisses++;
}
recordMetric(name: string, value: number, tags?: Record<string, string>): void {
this.metrics.push({
name,
value,
timestamp: Date.now(),
tags,
});
// Keep metrics bounded
if (this.metrics.length > this.maxMetricsSize) {
this.metrics = this.metrics.slice(-this.maxMetricsSize);
}
}
private calculatePercentile(arr: number[], percentile: number): number {
if (arr.length === 0) return 0;
const sorted = [...arr].sort((a, b) => a - b);
const index = Math.ceil((percentile / 100) * sorted.length) - 1;
return sorted[Math.max(0, index)];
}
getSnapshot(): MetricsSnapshot {
const errorRate = this.totalRequests > 0 ? (this.errorCount / this.totalRequests) * 100 : 0;
const cacheHitRate =
this.cacheHits + this.cacheMisses > 0
? (this.cacheHits / (this.cacheHits + this.cacheMisses)) * 100
: 0;
return {
uploadLatency: {
p50: this.calculatePercentile(this.uploadTimes, 50),
p95: this.calculatePercentile(this.uploadTimes, 95),
p99: this.calculatePercentile(this.uploadTimes, 99),
},
uploadThroughput: this.totalRequests > 0 ? this.totalRequests / 60 : 0,
queueSize: 0, // Will be updated by queue
errorRate,
cacheHitRate,
botUtilization: 0, // Will be updated by bot tracker
timestamp: Date.now(),
};
}
reset(): void {
this.uploadTimes = [];
this.errorCount = 0;
this.totalRequests = 0;
this.cacheHits = 0;
this.cacheMisses = 0;
this.metrics = [];
}
getMetrics(name?: string): Metric[] {
if (!name) return this.metrics;
return this.metrics.filter((m) => m.name === name);
}
}
export const metricsCollector = new MetricsCollector();
export { MetricsCollector };
-89
View File
@@ -1,89 +0,0 @@
import { config } from '../env';
import { extractClientIp } from './ip';
import logger from './logger';
interface RateLimitEntry {
count: number;
resetTime: number;
}
const rateLimitStore = new Map<string, RateLimitEntry>();
const MAX_STORE_ENTRIES = 50000;
const evictExpiredEntries = (now = Date.now()): number => {
let cleaned = 0;
for (const [key, entry] of rateLimitStore.entries()) {
if (now > entry.resetTime) {
rateLimitStore.delete(key);
cleaned++;
}
}
return cleaned;
};
const ensureStoreCapacity = (now: number): void => {
if (rateLimitStore.size < MAX_STORE_ENTRIES) return;
evictExpiredEntries(now);
while (rateLimitStore.size >= MAX_STORE_ENTRIES) {
const oldestKey = rateLimitStore.keys().next().value;
if (!oldestKey) break;
rateLimitStore.delete(oldestKey);
}
};
export const checkRateLimit = (key: string): boolean => {
const now = Date.now();
const entry = rateLimitStore.get(key);
if (!entry || now > entry.resetTime) {
ensureStoreCapacity(now);
rateLimitStore.set(key, {
count: 1,
resetTime: now + config.rateLimitWindowMs,
});
return true;
}
if (entry.count >= config.rateLimitMaxRequests) {
logger.warn('Rate limit exceeded', { key, count: entry.count });
return false;
}
entry.count++;
return true;
};
export const withRateLimit = <T extends Request>(
handler: (req: T) => Promise<Response>,
): ((req: T) => Promise<Response>) => {
return async (req: T): Promise<Response> => {
const ip = extractClientIp(req);
if (!checkRateLimit(ip)) {
return Response.json({ error: 'Rate limit exceeded' }, { status: 429 });
}
return handler(req);
};
};
export const cleanupRateLimitCache = (): void => {
const cleaned = evictExpiredEntries();
if (cleaned > 0) {
logger.debug('Rate limit cache cleanup', { cleaned, remaining: rateLimitStore.size });
}
};
export const getRateLimitStats = () => ({
trackedIPs: rateLimitStore.size,
windowSize: config.rateLimitWindowMs,
maxRequests: config.rateLimitMaxRequests,
maxTrackedIPs: MAX_STORE_ENTRIES,
});
export const clearRateLimitCache = (): void => {
rateLimitStore.clear();
};
-91
View File
@@ -1,91 +0,0 @@
import logger from './logger';
interface RetryOptions {
maxRetries?: number;
initialDelayMs?: number;
maxDelayMs?: number;
backoffMultiplier?: number;
shouldRetry?: (error: unknown) => boolean;
}
const DEFAULT_OPTIONS: Required<RetryOptions> = {
maxRetries: 3,
initialDelayMs: 100,
maxDelayMs: 5000,
backoffMultiplier: 2,
shouldRetry: (error: unknown) => {
const errorStr = error instanceof Error ? error.message : String(error);
// Retry on transient errors
return (
errorStr.includes('ECONNREFUSED') ||
errorStr.includes('ETIMEDOUT') ||
errorStr.includes('ENOTFOUND') ||
errorStr.includes('429') ||
errorStr.includes('timeout')
);
},
};
export const withRetry = async <T>(
fn: () => Promise<T>,
options: RetryOptions = {},
): Promise<T> => {
const opts = { ...DEFAULT_OPTIONS, ...options };
let lastError: unknown;
let delay = opts.initialDelayMs;
for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
try {
return await fn();
} catch (error: unknown) {
lastError = error;
const errorStr = error instanceof Error ? error.message : String(error);
if (attempt === opts.maxRetries || !opts.shouldRetry(error)) {
logger.error('Retry exhausted', {
attempt,
maxRetries: opts.maxRetries,
error: errorStr,
});
throw error;
}
logger.warn('Retrying after error', {
attempt,
delay,
error: errorStr,
});
await new Promise((resolve) => setTimeout(resolve, delay));
delay = Math.min(delay * opts.backoffMultiplier, opts.maxDelayMs);
}
}
throw lastError;
};
export const withTimeout = async <T>(
fn: () => Promise<T>,
timeoutMs: number = 30000,
): Promise<T> => {
return Promise.race([
fn(),
new Promise<T>((_, reject) =>
setTimeout(() => reject(new Error(`Operation timeout after ${timeoutMs}ms`)), timeoutMs),
),
]);
};
export const withFallback = async <T>(
primary: () => Promise<T>,
fallback: () => Promise<T>,
): Promise<T> => {
try {
return await primary();
} catch (error: unknown) {
logger.warn('Primary operation failed, using fallback', {
error: error instanceof Error ? error.message : String(error),
});
return fallback();
}
};
-463
View File
@@ -1,463 +0,0 @@
import { timingSafeEqual } from 'node:crypto';
/**
* Timing-safe string comparison that prevents timing attacks.
*
* Uses `crypto.timingSafeEqual` which runs in constant time regardless of
* where the strings differ. Returns false for mismatched-length inputs
* to avoid leaking length information via early return.
*
* @param left - The first string to compare.
* @param right - The second string to compare.
* @returns True if both strings are equal.
*/
const timingSafeCompare = (left: string, right: string): boolean => {
const leftBuffer = Buffer.from(left);
const rightBuffer = Buffer.from(right);
if (leftBuffer.length !== rightBuffer.length) {
return false;
}
return timingSafeEqual(leftBuffer, rightBuffer);
};
export interface SigV4Result {
isValid: boolean;
credential: {
accessKey: string;
date: string;
region: string;
service: string;
} | null;
errorCode?: string;
}
export interface VerifyPresignedUrlInput {
url: string;
method: string;
headers: Record<string, string>;
s3AccessKey: string;
s3SecretKey: string;
region: string;
now?: Date;
}
const SERVICE = 's3';
const TERMINATION = 'aws4_request';
/**
* Maximum acceptable clock skew between client and server for header-based
* SigV4 authentication. AWS allows 15 minutes.
*/
const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const buf = (data: string | ArrayBuffer | Uint8Array): Uint8Array => {
if (data instanceof Uint8Array) return data;
if (data instanceof ArrayBuffer) return new Uint8Array(data);
return new TextEncoder().encode(data);
};
const sha256Hex = async (data: string | Uint8Array | ArrayBuffer): Promise<string> => {
const hashBuffer = await crypto.subtle.digest('SHA-256', buf(data) as never);
const hashArray = Array.from(new Uint8Array(hashBuffer));
return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
};
const hmacSha256 = async (key: Uint8Array, message: string): Promise<Uint8Array> => {
const cryptoKey = await crypto.subtle.importKey(
'raw',
key as never,
{ name: 'HMAC', hash: 'SHA-256' },
false,
['sign'],
);
const result = await crypto.subtle.sign('HMAC', cryptoKey, buf(message) as never);
return new Uint8Array(result);
};
const getSigningKey = async (
secretKey: string,
dateStamp: string,
region: string,
): Promise<Uint8Array> => {
let key = await hmacSha256(buf(`AWS4${secretKey}`), dateStamp);
key = await hmacSha256(key, region);
key = await hmacSha256(key, SERVICE);
return await hmacSha256(key, TERMINATION);
};
const hmacHex = async (key: Uint8Array, message: string): Promise<string> => {
const result = await hmacSha256(key, message);
return Array.from(result)
.map((b) => b.toString(16).padStart(2, '0'))
.join('');
};
const parseAuthorizationHeader = (authHeader: string) => {
const credentialMatch = authHeader.match(/Credential=([^,]+)/);
const signedHeadersMatch = authHeader.match(/SignedHeaders=([^,]+)/);
const signatureMatch = authHeader.match(/Signature=([^,]+)/);
if (!credentialMatch || !signedHeadersMatch || !signatureMatch) return null;
const credentialParts = credentialMatch[1].split('/');
if (credentialParts.length !== 5) return null;
return {
accessKey: credentialParts[0],
date: credentialParts[1],
region: credentialParts[2],
service: credentialParts[3],
termination: credentialParts[4],
signedHeaders: signedHeadersMatch[1],
signature: signatureMatch[1],
};
};
const buildCanonicalRequest = (
method: string,
canonicalUri: string,
canonicalQueryString: string,
signedHeaders: string,
headers: Record<string, string>,
hashedPayload: string,
): string => {
const canonicalHeaders = signedHeaders
.split(';')
.map((h) => {
const value = headers[h.toLowerCase()] || '';
return `${h.toLowerCase()}:${value.trim()}\n`;
})
.join('');
return `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${hashedPayload}`;
};
/**
* Normalizes a URI per AWS SigV4 requirements plus RFC 3986:
*
* 1. Decode percent-encoded characters
* 2. Remove dot-segments (`.` and `..`) per RFC 3986 section 5.2.4
*
* @param uri - The raw URI path to normalize.
* @returns The normalized URI path.
*/
const normalizeUri = (uri: string): string => {
if (!uri || uri === '') return '/';
// AWS SigV4 requires URI-decoded paths in the canonical request
// Only `.` and `..` segments are removed per RFC 3986 section 5.2.4
// Empty segments (from `//` or trailing `/`) are preserved — they are
// part of the URI and the SDK signs them.
const decoded = decodeURIComponent(uri);
const segments = decoded.split('/');
const result: string[] = [];
for (const segment of segments) {
if (segment === '.') continue;
if (segment === '..') {
result.pop();
continue;
}
result.push(segment);
}
// Join preserves empty first segment (from leading /) automatically
return result.join('/') || '/';
};
const awsEncode = (value: string): string =>
encodeURIComponent(value).replace(
/[!'()*]/g,
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
);
export const buildCanonicalQueryString = (
searchParams: URLSearchParams,
excludeKeys: Set<string> = new Set(),
): string => {
const pairs: Array<[string, string]> = [];
for (const [key, value] of searchParams.entries()) {
if (!excludeKeys.has(key)) pairs.push([key, value]);
}
// AWS SigV4 requires UTF-8 byte-order (code point) comparison, NOT localeCompare
pairs.sort(([ak, av], [bk, bv]) => {
const a = `${awsEncode(ak)}=${awsEncode(av)}`;
const b = `${awsEncode(bk)}=${awsEncode(bv)}`;
if (a < b) return -1;
if (a > b) return 1;
return 0;
});
return pairs.map(([key, value]) => `${awsEncode(key)}=${awsEncode(value)}`).join('&');
};
const getHashedPayload = async (body: string | null): Promise<string> => {
if (!body || body.length === 0) return await sha256Hex('');
return await sha256Hex(body);
};
/**
* Parses an AWS SigV4 `x-amz-date` value (e.g. `20260707T120000Z`) into a Date.
*
* @param amzDate - The date string in `YYYYMMDDTHHmmssZ` format.
* @returns The parsed Date, or null if the format is invalid.
*/
const parseAmzDateUtc = (amzDate: string): Date | null => {
const match = amzDate.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/);
if (!match) return null;
const [, year, month, day, hour, minute, second] = match;
return new Date(
Date.UTC(
Number.parseInt(year, 10),
Number.parseInt(month, 10) - 1,
Number.parseInt(day, 10),
Number.parseInt(hour, 10),
Number.parseInt(minute, 10),
Number.parseInt(second, 10),
),
);
};
/**
* Validates that `host` is included in the signed headers list.
*
* AWS SigV4 mandates that `host` is always signed. Reject requests that
* omit it to prevent header injection / replay variants.
*
* @param signedHeaders - The semicolon-separated signed headers string.
* @returns True if `host` is present.
*/
const validateSignedHeaders = (signedHeaders: string): boolean => {
return signedHeaders.split(';').some((h) => h.toLowerCase() === 'host');
};
export const verifySignature = async (
method: string,
url: string,
headers: Record<string, string>,
body: string | null,
s3AccessKey: string,
s3SecretKey: string,
region: string,
): Promise<SigV4Result> => {
const authHeader = headers.authorization;
if (!authHeader?.startsWith('AWS4-HMAC-SHA256')) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const parsed = parseAuthorizationHeader(authHeader);
if (!parsed) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
if (!timingSafeCompare(parsed.accessKey, s3AccessKey)) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
if (!timingSafeCompare(parsed.region, region)) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
// Validate service and termination in credential scope (M2)
if (parsed.service !== SERVICE || parsed.termination !== TERMINATION) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
// Validate host is in signed headers (LOW/host)
if (!validateSignedHeaders(parsed.signedHeaders)) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const parsedUrl = new URL(url, 'http://localhost');
const canonicalUri = normalizeUri(parsedUrl.pathname);
const canonicalQueryString = buildCanonicalQueryString(parsedUrl.searchParams);
const contentSha256 = headers['x-amz-content-sha256'] || null;
if (contentSha256?.startsWith('STREAMING-')) {
return { isValid: false, credential: null, errorCode: 'NotImplemented' };
}
// CRITICAL: Use the x-amz-content-sha256 header value in the canonical
// request because that's what the client signed. The actual body hash is
// verified by verifyBodyHash() after streaming, ensuring integrity without
// breaking SigV4.
const hashedPayload = contentSha256 || (await getHashedPayload(body));
const canonicalRequest = buildCanonicalRequest(
method,
canonicalUri,
canonicalQueryString,
parsed.signedHeaders,
headers,
hashedPayload,
);
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
// M1: Fall back to Date header if x-amz-date is missing
const amzDate = headers['x-amz-date'] || headers.date || '';
// H5: Validate request freshness (clock skew / replay protection)
if (amzDate) {
const requestDate = parseAmzDateUtc(amzDate);
if (requestDate) {
const now = Date.now();
const skew = Math.abs(now - requestDate.getTime());
if (skew > MAX_CLOCK_SKEW_MS) {
return { isValid: false, credential: null, errorCode: 'RequestExpired' };
}
}
}
const dateStamp = parsed.date;
// M3: Ensure date in credential scope matches x-amz-date
if (amzDate) {
const amzDateStamp = amzDate.slice(0, 8); // "YYYYMMDD"
if (amzDateStamp !== dateStamp) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
}
const credentialScope = `${dateStamp}/${region}/${parsed.service}/${parsed.termination}`;
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
const signingKey = await getSigningKey(s3SecretKey, dateStamp, region);
const expectedSignature = await hmacHex(signingKey, stringToSign);
if (!timingSafeCompare(expectedSignature, parsed.signature)) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
return {
isValid: true,
credential: {
accessKey: parsed.accessKey,
date: parsed.date,
region: parsed.region,
service: parsed.service,
},
};
};
export const verifyPresignedUrl = async ({
url,
method,
headers,
s3AccessKey,
s3SecretKey,
region,
now = new Date(),
}: VerifyPresignedUrlInput): Promise<SigV4Result> => {
const parsedUrl = new URL(url);
const searchParams = parsedUrl.searchParams;
const algorithm = searchParams.get('X-Amz-Algorithm');
const credential = searchParams.get('X-Amz-Credential');
const signedHeaders = searchParams.get('X-Amz-SignedHeaders');
const signature = searchParams.get('X-Amz-Signature');
const expiresText = searchParams.get('X-Amz-Expires');
const amzDate = searchParams.get('X-Amz-Date');
if (
algorithm !== 'AWS4-HMAC-SHA256' ||
!credential ||
!signedHeaders ||
!signature ||
!expiresText ||
!amzDate
) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const expires = Number.parseInt(expiresText, 10);
const signedAt = parseAmzDateUtc(amzDate);
if (!Number.isFinite(expires) || expires <= 0 || !signedAt) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
// AWS S3 spec limits presigned URLs to 7 days (604800 seconds)
const MAX_PRESIGNED_EXPIRY_SECONDS = 604800;
if (
now.getTime() > signedAt.getTime() + expires * 1000 ||
expires > MAX_PRESIGNED_EXPIRY_SECONDS
) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const credParts = credential.split('/');
if (credParts.length !== 5) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const [accessKey, dateStamp, credentialRegion, service, termination] = credParts;
if (
!timingSafeCompare(accessKey, s3AccessKey) ||
!timingSafeCompare(credentialRegion, region) ||
service !== SERVICE ||
termination !== TERMINATION
) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
// Validate host is in signed headers for presigned URLs too
if (!validateSignedHeaders(signedHeaders)) {
return { isValid: false, credential: null, errorCode: 'AccessDenied' };
}
const signedHeaderList = signedHeaders.split(';').filter(Boolean);
const canonicalHeaders = signedHeaderList
.map((headerName) => {
const lower = headerName.toLowerCase();
const value = lower === 'host' ? headers.host || parsedUrl.host : headers[lower] || '';
return `${lower}:${value.trim()}\n`;
})
.join('');
const canonicalRequest = `${method}\n${normalizeUri(parsedUrl.pathname)}\n${buildCanonicalQueryString(searchParams, new Set(['X-Amz-Signature']))}\n${canonicalHeaders}\n${signedHeaders}\nUNSIGNED-PAYLOAD`;
const hashedCanonicalRequest = await sha256Hex(canonicalRequest);
const credentialScope = `${dateStamp}/${region}/${SERVICE}/${TERMINATION}`;
const stringToSign = `AWS4-HMAC-SHA256\n${amzDate}\n${credentialScope}\n${hashedCanonicalRequest}`;
const expectedSignature = await hmacHex(
await getSigningKey(s3SecretKey, dateStamp, region),
stringToSign,
);
if (!timingSafeCompare(expectedSignature, signature)) {
return { isValid: false, credential: null, errorCode: 'SignatureDoesNotMatch' };
}
return { isValid: true, credential: { accessKey, date: dateStamp, region, service } };
};
export const isS3Request = (headers: Record<string, string>): boolean => {
const auth = headers.authorization || '';
return auth.startsWith('AWS4-HMAC-SHA256');
};
/**
* Verifies that the actual body SHA-256 matches the `x-amz-content-sha256`
* header from the original request.
*
* This MUST be called AFTER the body has been fully streamed and hashed,
* as a second pass after `verifySignature` (which cannot hash a streaming
* body without consuming it).
*
* @param bodySha256 - The SHA-256 hex digest of the actual body content.
* @param headers - The original request headers.
* @returns An error result on mismatch, or null if the check passes.
*/
export const verifyBodyHash = (
bodySha256: string,
headers: Record<string, string>,
): SigV4Result | null => {
const claimedHash = headers['x-amz-content-sha256'];
// If the client sent UNSIGNED-PAYLOAD, skip verification
if (!claimedHash || claimedHash === 'UNSIGNED-PAYLOAD' || claimedHash.startsWith('STREAMING-')) {
return null;
}
if (claimedHash !== bodySha256) {
return { isValid: false, credential: null, errorCode: 'BadDigest' };
}
return null;
};
-52
View File
@@ -1,52 +0,0 @@
import { nanoid } from 'nanoid';
export const S3_CORS_HEADERS: Record<string, string> = {
'access-control-allow-origin': '*',
'access-control-allow-methods': 'GET, PUT, HEAD, DELETE, POST, OPTIONS',
'access-control-allow-headers': [
'Authorization',
'Content-Type',
'Content-MD5',
'Range',
'If-Match',
'If-None-Match',
'If-Modified-Since',
'If-Unmodified-Since',
'X-Amz-*',
'x-amz-*',
].join(', '),
'access-control-expose-headers': [
'Accept-Ranges',
'Content-Length',
'Content-Range',
'Content-Type',
'ETag',
'Last-Modified',
'x-amz-id-2',
'x-amz-request-id',
].join(', '),
'access-control-max-age': '86400',
};
export const s3Headers = (
requestId: string,
extraHeaders: Record<string, string> = {},
): Record<string, string> => ({
...S3_CORS_HEADERS,
server: 'AmazonS3',
...(requestId
? {
'x-amz-request-id': requestId,
'x-amz-id-2': `${requestId}+${nanoid(16)}`,
}
: {}),
...extraHeaders,
});
export const applyS3Headers = (headers: Headers, requestId: string): Headers => {
const result = new Headers(headers);
for (const [key, value] of Object.entries(s3Headers(requestId))) {
result.set(key, value);
}
return result;
};
-133
View File
@@ -1,133 +0,0 @@
import { gunzipSync } from 'node:zlib';
import { applyS3Headers } from './headers';
import { contentRange, type RangeParseResult } from './range';
export interface ObjectPartSource {
telegramFileId: string;
telegramUrl: string;
sizeBytes: number;
partNumber: number;
storedSizeBytes?: number;
compressionAlgorithm?: 'gzip' | null;
}
export interface ObjectResponseInput {
reqId: string;
contentType: string;
etag: string;
lastModified: Date;
totalSize: number;
parts: ObjectPartSource[];
range: RangeParseResult;
}
interface PlannedPart {
part: ObjectPartSource;
relativeStart: number;
relativeEnd: number;
}
const baseHeaders = (input: ObjectResponseInput, contentLength: number): Headers => {
const headers = new Headers({
'content-type': input.contentType,
'content-length': String(contentLength),
etag: `"${input.etag}"`,
'last-modified': input.lastModified.toUTCString(),
'x-amz-request-id': input.reqId,
'accept-ranges': 'bytes',
'cache-control': 'public, max-age=31536000',
});
return headers;
};
const planParts = (parts: ObjectPartSource[], start: number, end: number): PlannedPart[] => {
const planned: PlannedPart[] = [];
let offset = 0;
for (const part of parts) {
const partStart = offset;
const partEnd = offset + part.sizeBytes - 1;
offset += part.sizeBytes;
if (end < partStart || start > partEnd) continue;
planned.push({
part,
relativeStart: Math.max(start, partStart) - partStart,
relativeEnd: Math.min(end, partEnd) - partStart,
});
}
return planned;
};
const streamFromBytes = (bytes: Uint8Array): ReadableStream<Uint8Array> =>
new Response(bytes).body!;
const TELEGRAM_FETCH_TIMEOUT_MS = 30_000;
const fetchWholePartBytes = async (telegramUrl: string): Promise<Uint8Array> => {
const res = await fetch(telegramUrl, { signal: AbortSignal.timeout(TELEGRAM_FETCH_TIMEOUT_MS) });
if (!res.ok) throw new Error(`Telegram fetch failed: ${res.status}`);
return new Uint8Array(await res.arrayBuffer());
};
const fetchPartBody = async (planned: PlannedPart): Promise<ReadableStream<Uint8Array>> => {
const wantsWholePart =
planned.relativeStart === 0 && planned.relativeEnd === planned.part.sizeBytes - 1;
if (planned.part.compressionAlgorithm === 'gzip') {
const storedBytes = await fetchWholePartBytes(planned.part.telegramUrl);
const bytes = gunzipSync(storedBytes);
return streamFromBytes(bytes.subarray(planned.relativeStart, planned.relativeEnd + 1));
}
const rangeHeader = `bytes=${planned.relativeStart}-${planned.relativeEnd}`;
const fetchOpts: RequestInit = { signal: AbortSignal.timeout(TELEGRAM_FETCH_TIMEOUT_MS) };
if (!wantsWholePart) {
fetchOpts.headers = { range: rangeHeader };
}
const res = await fetch(planned.part.telegramUrl, fetchOpts);
if (!res.ok) throw new Error(`Telegram fetch failed: ${res.status}`);
if (wantsWholePart || res.status === 206) return res.body!;
const bytes = new Uint8Array(await res.arrayBuffer());
return streamFromBytes(bytes.slice(planned.relativeStart, planned.relativeEnd + 1));
};
const concatPartStreams = (plannedParts: PlannedPart[]): ReadableStream<Uint8Array> =>
new ReadableStream<Uint8Array>({
async start(controller) {
try {
for (const planned of plannedParts) {
const stream = await fetchPartBody(planned);
const reader = stream.getReader();
while (true) {
const { value, done } = await reader.read();
if (done) break;
if (value) controller.enqueue(value);
}
}
controller.close();
} catch (error) {
controller.error(error);
}
},
});
export const createGetObjectResponse = async (input: ObjectResponseInput): Promise<Response> => {
if (input.range.type === 'invalid') {
throw new Error('createGetObjectResponse received invalid range');
}
const start = input.range.type === 'valid' ? input.range.start : 0;
const end = input.range.type === 'valid' ? input.range.end : input.totalSize - 1;
const plannedParts = planParts(input.parts, start, end);
const contentLength = end >= start ? end - start + 1 : 0;
const headers = applyS3Headers(baseHeaders(input, contentLength), input.reqId);
if (input.range.type === 'valid') {
headers.set('content-range', contentRange(start, end, input.totalSize));
}
return new Response(concatPartStreams(plannedParts), {
status: input.range.type === 'valid' ? 206 : 200,
headers,
});
};
-46
View File
@@ -1,46 +0,0 @@
export type RangeParseResult =
| { type: 'none' }
| { type: 'valid'; start: number; end: number }
| { type: 'invalid' };
const DECIMAL = /^\d+$/;
export const parseRangeHeader = (rangeHeader: string | null, size: number): RangeParseResult => {
if (!rangeHeader) return { type: 'none' };
if (!Number.isSafeInteger(size) || size < 0) return { type: 'invalid' };
if (!rangeHeader.startsWith('bytes=')) return { type: 'invalid' };
const spec = rangeHeader.slice('bytes='.length).trim();
if (spec.includes(',')) return { type: 'invalid' };
const dash = spec.indexOf('-');
if (dash === -1) return { type: 'invalid' };
const startText = spec.slice(0, dash).trim();
const endText = spec.slice(dash + 1).trim();
if (!startText && !endText) return { type: 'invalid' };
if (size === 0) return { type: 'invalid' };
if (!startText) {
if (!DECIMAL.test(endText)) return { type: 'invalid' };
const suffixLength = Number.parseInt(endText, 10);
if (suffixLength <= 0) return { type: 'invalid' };
return { type: 'valid', start: Math.max(size - suffixLength, 0), end: size - 1 };
}
if (!DECIMAL.test(startText)) return { type: 'invalid' };
const start = Number.parseInt(startText, 10);
if (start >= size) return { type: 'invalid' };
if (!endText) return { type: 'valid', start, end: size - 1 };
if (!DECIMAL.test(endText)) return { type: 'invalid' };
const requestedEnd = Number.parseInt(endText, 10);
if (requestedEnd < start) return { type: 'invalid' };
return { type: 'valid', start, end: Math.min(requestedEnd, size - 1) };
};
export const contentRange = (start: number, end: number, size: number): string =>
`bytes ${start}-${end}/${size}`;
export const unsatisfiedContentRange = (size: number): string => `bytes */${size}`;
-27
View File
@@ -1,27 +0,0 @@
const stripPort = (host: string): string => {
// Handle IPv6: [::1]:8080 -> [::1]
if (host.startsWith('[')) {
const closeBracket = host.indexOf(']');
return host.slice(0, closeBracket + 1).toLowerCase();
}
return host.split(':')[0].toLowerCase().replace(/\.$/, '');
};
const isValidBucketLabel = (bucket: string): boolean =>
/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) &&
!bucket.includes('..') &&
!bucket.includes('.-') &&
!bucket.includes('-.');
export const extractS3BucketFromHost = (host: string, domains: string[]): string | null => {
const normalizedHost = stripPort(host);
for (const domain of domains) {
const normalizedDomain = stripPort(domain);
if (!normalizedDomain || normalizedHost === normalizedDomain) continue;
if (!normalizedHost.endsWith(`.${normalizedDomain}`)) continue;
const bucket = normalizedHost.slice(0, -(normalizedDomain.length + 1));
return isValidBucketLabel(bucket) ? bucket : null;
}
return null;
};
-311
View File
@@ -1,311 +0,0 @@
import { s3Headers } from './headers';
const escapeXml = (str: string): string =>
str
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&apos;');
const isoDate = (d: Date): string => d.toISOString().replace(/\.\d{3}Z$/, 'Z');
const encodeKey = (value: string, encodingType: string | null = null): string =>
encodingType === 'url' ? encodeURIComponent(value) : escapeXml(value);
// ─────── Bucket operations ───────
export const listBucketsXml = (
buckets: { name: string; createdAt: Date }[],
_requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Buckets>
${buckets
.map(
(b) => `<Bucket>
<Name>${escapeXml(b.name)}</Name>
<CreationDate>${isoDate(b.createdAt)}</CreationDate>
</Bucket>`,
)
.join('')}
</Buckets>
</ListAllMyBucketsResult>`;
export const bucketVersioningConfigurationXml =
(): string => `<?xml version="1.0" encoding="UTF-8"?>
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`;
// ─────── Object listing ───────
export const listBucketResultXml = (
bucketName: string,
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
prefixes: string[],
isTruncated: boolean,
marker: string | null,
maxKeys: number,
prefix: string,
delimiter: string | null,
nextMarker: string | null,
_requestId: string,
encodingType: string | null = null,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>${escapeXml(bucketName)}</Name>
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
<Marker>${encodeKey(marker || '', encodingType)}</Marker>
<MaxKeys>${maxKeys}</MaxKeys>
<Delimiter>${encodeKey(delimiter || '', encodingType)}</Delimiter>
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
<IsTruncated>${isTruncated}</IsTruncated>
${objects
.map(
(o) => `<Contents>
<Key>${encodeKey(o.key, encodingType)}</Key>
<LastModified>${isoDate(o.lastModified)}</LastModified>
<ETag>"${o.etag}"</ETag>
<Size>${o.sizeBytes}</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>`,
)
.join('')}
${prefixes
.map(
(p) => `<CommonPrefixes>
<Prefix>${encodeKey(p, encodingType)}</Prefix>
</CommonPrefixes>`,
)
.join('')}
${nextMarker ? `<NextMarker>${encodeKey(nextMarker, encodingType)}</NextMarker>` : ''}
</ListBucketResult>`;
export const listBucketV2ResultXml = (
bucketName: string,
objects: { key: string; sizeBytes: number; etag: string; lastModified: Date; mimeType: string }[],
prefixes: string[],
isTruncated: boolean,
maxKeys: number,
prefix: string,
delimiter: string | null,
continuationToken: string | null,
nextContinuationToken: string | null,
keyCount: number,
_requestId: string,
encodingType: string | null = null,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResultV2 xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>${escapeXml(bucketName)}</Name>
<Prefix>${encodeKey(prefix, encodingType)}</Prefix>
<MaxKeys>${maxKeys}</MaxKeys>
<KeyCount>${keyCount}</KeyCount>
${delimiter ? `<Delimiter>${encodeKey(delimiter, encodingType)}</Delimiter>` : ''}
${encodingType ? `<EncodingType>${escapeXml(encodingType)}</EncodingType>` : ''}
${continuationToken ? `<ContinuationToken>${encodeKey(continuationToken, encodingType)}</ContinuationToken>` : ''}
<IsTruncated>${isTruncated}</IsTruncated>
${objects
.map(
(o) => `<Contents>
<Key>${encodeKey(o.key, encodingType)}</Key>
<LastModified>${isoDate(o.lastModified)}</LastModified>
<ETag>"${o.etag}"</ETag>
<Size>${o.sizeBytes}</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>`,
)
.join('')}
${prefixes
.map(
(p) => `<CommonPrefixes>
<Prefix>${encodeKey(p, encodingType)}</Prefix>
</CommonPrefixes>`,
)
.join('')}
${nextContinuationToken ? `<NextContinuationToken>${encodeKey(nextContinuationToken, encodingType)}</NextContinuationToken>` : ''}
</ListBucketResultV2>`;
// ─────── Multipart ───────
export const initiateMultipartUploadXml = (
bucketName: string,
key: string,
uploadId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>${escapeXml(bucketName)}</Bucket>
<Key>${escapeXml(key)}</Key>
<UploadId>${uploadId}</UploadId>
</InitiateMultipartUploadResult>`;
export const listPartsXml = (
bucketName: string,
key: string,
uploadId: string,
parts: { partNumber: number; etag: string; sizeBytes: number; createdAt: Date }[],
maxParts: number,
isTruncated: boolean,
_requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>${escapeXml(bucketName)}</Bucket>
<Key>${escapeXml(key)}</Key>
<UploadId>${uploadId}</UploadId>
<MaxParts>${maxParts}</MaxParts>
<IsTruncated>${isTruncated}</IsTruncated>
${parts
.map(
(p) => `<Part>
<PartNumber>${p.partNumber}</PartNumber>
<LastModified>${isoDate(p.createdAt)}</LastModified>
<ETag>"${p.etag}"</ETag>
<Size>${p.sizeBytes}</Size>
</Part>`,
)
.join('')}
</ListPartsResult>`;
export const listMultipartUploadsXml = (
bucketName: string,
uploads: { key: string; uploadId: string; initiatedAt: Date; initiatedBy: string }[],
maxUploads: number,
isTruncated: boolean,
nextKeyMarker: string | null,
_requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>${escapeXml(bucketName)}</Bucket>
<KeyMarker></KeyMarker>
<UploadIdMarker></UploadIdMarker>
${nextKeyMarker ? `<NextKeyMarker>${escapeXml(nextKeyMarker)}</NextKeyMarker>` : ''}
<MaxUploads>${maxUploads}</MaxUploads>
<IsTruncated>${isTruncated}</IsTruncated>
${uploads
.map(
(u) => `<Upload>
<Key>${escapeXml(u.key)}</Key>
<UploadId>${u.uploadId}</UploadId>
<Initiator><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Initiator>
<Owner><ID>${escapeXml(u.initiatedBy || 's3')}</ID><DisplayName>${escapeXml(u.initiatedBy || 's3')}</DisplayName></Owner>
<StorageClass>STANDARD</StorageClass>
<Initiated>${isoDate(u.initiatedAt)}</Initiated>
</Upload>`,
)
.join('')}
</ListMultipartUploadsResult>`;
export const completeMultipartUploadXml = (
bucketName: string,
key: string,
etag: string,
location: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Location>${escapeXml(location)}</Location>
<Bucket>${escapeXml(bucketName)}</Bucket>
<Key>${escapeXml(key)}</Key>
<ETag>"${etag}"</ETag>
</CompleteMultipartUploadResult>`;
// ─────── Delete result ───────
export const deleteResultXml = (
deleted: string[],
errors: { key: string; code: string; message: string }[],
): string => `<?xml version="1.0" encoding="UTF-8"?>
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
${deleted
.map(
(key) => `<Deleted>
<Key>${escapeXml(key)}</Key>
</Deleted>`,
)
.join('')}
${errors
.map(
(e) => `<Error>
<Key>${escapeXml(e.key)}</Key>
<Code>${e.code}</Code>
<Message>${escapeXml(e.message)}</Message>
</Error>`,
)
.join('')}
</DeleteResult>`;
// ─────── Copy ───────
export const copyObjectResultXml = (
etag: string,
lastModified: Date,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<ETag>"${etag}"</ETag>
<LastModified>${isoDate(lastModified)}</LastModified>
</CopyObjectResult>`;
// ─────── Error ───────
export const s3ErrorXml = (
code: string,
message: string,
resource: string,
requestId: string,
): string => `<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>${code}</Code>
<Message>${escapeXml(message)}</Message>
<Resource>${escapeXml(resource)}</Resource>
<RequestId>${requestId}</RequestId>
<HostId>${requestId}</HostId>
</Error>`;
export const s3ErrorResponse = (
code: string,
message: string,
resource: string,
status: number,
requestId: string = '',
extraHeaders: Record<string, string> = {},
): Response =>
new Response(s3ErrorXml(code, message, resource, requestId), {
status,
headers: s3Headers(requestId, {
'content-type': 'application/xml',
...extraHeaders,
}),
});
// ─────── DeleteObjects XML parser ───────
export const parseDeleteObjectsBody = (body: string): { keys: string[]; quiet: boolean } => {
// H9: Use non-greedy match to handle keys containing < character
const keys = Array.from(body.matchAll(/<Key>([\s\S]*?)<\/Key>/g), (match) => match[1]);
// Handle whitespace inside <Quiet> element + namespace prefix support
const quiet = /<\w*:?Quiet\w*>\s*true\s*<\/\w*:?Quiet\w*>/i.test(body);
return { keys, quiet };
};
// ─────── CompleteMultipartUpload XML parser ───────
export interface CompletePart {
partNumber: number;
etag: string;
}
export const parseCompleteMultipartBody = (body: string): CompletePart[] => {
const parts: CompletePart[] = [];
const partRegex = /<Part>[\s\S]*?<\/Part>/g;
const partMatch = body.match(partRegex) || [];
for (const partXml of partMatch) {
const numMatch = partXml.match(/<PartNumber>(\d+)<\/PartNumber>/);
const etagMatch = partXml.match(/<ETag>"?([^"<\s]+)"?<\/ETag>/);
if (numMatch && etagMatch) {
parts.push({
partNumber: Number.parseInt(numMatch[1], 10),
etag: etagMatch[1].replace(/^"/, '').replace(/"$/, ''),
});
}
}
return parts;
};
-152
View File
@@ -1,152 +0,0 @@
import { createReadStream } from 'node:fs';
import { nanoid } from 'nanoid';
import { db, files as fileSchema } from '../db';
import type { NewFile } from '../db/schema';
import { config } from '../env';
import { botPool } from '../infrastructure/telegram/bot-pool';
import { cleanupTempFile } from './file';
import { createZip, type ZipEntry } from './zip';
export type PreparedUpload = {
tempPath: string;
fileHash: string;
sizeBytes: number;
signatureBuffer: Buffer;
};
export type UploadedFile = NewFile & {
createdAt: Date;
updatedAt: Date;
};
export type BatchUploadItem = {
prepared: PreparedUpload;
fileName: string;
mimeType: string;
fileType: string;
};
type PendingUpload = BatchUploadItem & {
resolve: (file: UploadedFile) => void;
reject: (error: unknown) => void;
};
const BATCH_WINDOW_MS = 2000;
let pendingUploads: PendingUpload[] = [];
let flushTimer: ReturnType<typeof setTimeout> | null = null;
const buildUploadedFile = (
item: BatchUploadItem,
entry: ZipEntry,
archive: {
telegramFileId: string;
telegramFileUniqueId: string;
storageMessageId: number;
fileName: string;
sizeBytes: number;
},
): UploadedFile => ({
publicId: nanoid(),
telegramFileId: archive.telegramFileId,
telegramFileUniqueId: archive.telegramFileUniqueId,
storageChatId: config.storageChatId,
storageMessageId: archive.storageMessageId,
fileName: item.fileName,
mimeType: item.mimeType || 'application/octet-stream',
sizeBytes: item.prepared.sizeBytes,
fileType: item.fileType,
uploaderId: 0,
fileHash: item.prepared.fileHash,
archiveTelegramFileId: archive.telegramFileId,
archiveStorageMessageId: archive.storageMessageId,
archiveFileName: archive.fileName,
archiveEntryName: entry.entryName,
archiveMimeType: 'application/zip',
archiveSizeBytes: archive.sizeBytes,
createdAt: new Date(),
updatedAt: new Date(),
});
const flushUploads = async (): Promise<void> => {
if (flushTimer) {
clearTimeout(flushTimer);
flushTimer = null;
}
const batch = pendingUploads;
pendingUploads = [];
if (batch.length === 0) return;
let zipTempPath: string | null = null;
try {
const zip = await createZip(
batch.map((item) => ({ tempPath: item.prepared.tempPath, fileName: item.fileName })),
);
zipTempPath = zip.tempPath;
const archiveFileName = `filedrop-${nanoid()}.zip`;
const archiveResult = await botPool.forwardToStorage(
createReadStream(zip.tempPath),
archiveFileName,
'document',
);
const uploadedFiles = batch.map((item, index) =>
buildUploadedFile(item, zip.entries[index], {
telegramFileId: archiveResult.telegramFileId,
telegramFileUniqueId: archiveResult.telegramFileUniqueId,
storageMessageId: archiveResult.storageMessageId,
fileName: archiveFileName,
sizeBytes: zip.sizeBytes,
}),
);
await db.insert(fileSchema).values(uploadedFiles);
for (let i = 0; i < batch.length; i++) {
batch[i].resolve(uploadedFiles[i]);
}
} catch (error) {
for (const item of batch) {
item.reject(error);
}
} finally {
await Promise.all(batch.map((item) => cleanupTempFile(item.prepared.tempPath)));
if (zipTempPath) await cleanupTempFile(zipTempPath);
// Reschedule timer if new items arrived during async processing
if (pendingUploads.length > 0 && !flushTimer) {
flushTimer = setTimeout(() => {
void flushUploads();
}, BATCH_WINDOW_MS);
}
}
};
const getPendingSize = (): number =>
pendingUploads.reduce((total, item) => total + item.prepared.sizeBytes, 0);
export const enqueuePreparedUpload = (item: BatchUploadItem): Promise<UploadedFile> => {
return new Promise((resolve, reject) => {
pendingUploads.push({ ...item, resolve, reject });
if (!flushTimer) {
flushTimer = setTimeout(() => {
void flushUploads();
}, BATCH_WINDOW_MS);
}
if (
pendingUploads.length >= config.batchMaxItems ||
getPendingSize() >= config.batchMaxSizeBytes
) {
void flushUploads();
}
});
};
export const flushPendingUploads = async (): Promise<void> => {
await flushUploads();
};
export const getPendingUploadCount = (): number => pendingUploads.length;
-292
View File
@@ -1,292 +0,0 @@
import { once } from 'node:events';
import { createReadStream, createWriteStream } from 'node:fs';
import { open, stat, unlink } from 'node:fs/promises';
import { basename } from 'node:path';
import { finished } from 'node:stream/promises';
import { nanoid } from 'nanoid';
export type ZipInputFile = {
tempPath: string;
fileName: string;
};
export type ZipEntry = {
fileName: string;
entryName: string;
crc32: number;
compressedSize: number;
uncompressedSize: number;
localHeaderOffset: number;
};
export type CreatedZip = {
tempPath: string;
sizeBytes: number;
fileHash: string;
entries: ZipEntry[];
};
const CRC32_TABLE = new Uint32Array(256).map((_, index) => {
let value = index;
for (let bit = 0; bit < 8; bit++) {
value = value & 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1;
}
return value >>> 0;
});
const updateCrc32 = (crc: number, chunk: Buffer): number => {
let value = crc;
for (const byte of chunk) {
value = CRC32_TABLE[(value ^ byte) & 0xff] ^ (value >>> 8);
}
return value >>> 0;
};
const dosDateTime = (date = new Date()): { date: number; time: number } => {
const year = Math.max(date.getFullYear(), 1980);
return {
time: (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2),
date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
};
};
const writeUInt16 = (value: number): Buffer<ArrayBuffer> => {
const buffer = Buffer.allocUnsafe(2);
buffer.writeUInt16LE(value & 0xffff, 0);
return buffer;
};
const writeUInt32 = (value: number): Buffer<ArrayBuffer> => {
const buffer = Buffer.allocUnsafe(4);
buffer.writeUInt32LE(value >>> 0, 0);
return buffer;
};
const writeChunk = async (
writer: ReturnType<typeof createWriteStream>,
chunk: Buffer,
): Promise<void> => {
if (!writer.write(chunk)) {
await once(writer, 'drain');
}
};
const finishWriter = async (writer: ReturnType<typeof createWriteStream>): Promise<void> => {
writer.end();
await finished(writer);
};
export const sanitizeZipEntryName = (fileName: string, usedNames = new Set<string>()): string => {
const cleaned = basename(fileName)
.replace(/[\\/]+/g, '_')
.replace(/\.\.+/g, '.')
.trim();
const fallback = cleaned && cleaned !== '.' && cleaned !== '..' ? cleaned : 'file';
const dotIndex = fallback.lastIndexOf('.');
const baseName = dotIndex > 0 ? fallback.slice(0, dotIndex) : fallback;
const extension = dotIndex > 0 ? fallback.slice(dotIndex) : '';
let candidate = fallback;
let counter = 1;
while (usedNames.has(candidate)) {
candidate = `${baseName}-${counter}${extension}`;
counter++;
}
usedNames.add(candidate);
return candidate;
};
const calculateFileCrc32 = async (tempPath: string): Promise<number> => {
let crc = 0xffffffff;
const reader = createReadStream(tempPath);
for await (const chunk of reader) {
crc = updateCrc32(crc, chunk as Buffer);
}
return (crc ^ 0xffffffff) >>> 0;
};
export const createZip = async (files: ZipInputFile[]): Promise<CreatedZip> => {
const tempPath = `/tmp/filedrop-${nanoid()}.zip`;
const writer = createWriteStream(tempPath);
const hasher = new Bun.CryptoHasher('sha256');
const entries: ZipEntry[] = [];
const usedNames = new Set<string>();
let offset = 0;
const writeHashed = async (chunk: Buffer): Promise<void> => {
hasher.update(chunk);
await writeChunk(writer, chunk);
offset += chunk.byteLength;
};
try {
for (const file of files) {
const entryName = sanitizeZipEntryName(file.fileName, usedNames);
const nameBuffer = Buffer.from(entryName);
const fileStats = await stat(file.tempPath);
const { date, time } = dosDateTime();
const localHeaderOffset = offset;
const crc32 = await calculateFileCrc32(file.tempPath);
const localHeader = Buffer.concat([
writeUInt32(0x04034b50),
writeUInt16(20),
writeUInt16(0),
writeUInt16(0),
writeUInt16(time),
writeUInt16(date),
writeUInt32(crc32),
writeUInt32(fileStats.size),
writeUInt32(fileStats.size),
writeUInt16(nameBuffer.byteLength),
writeUInt16(0),
nameBuffer,
]);
await writeHashed(localHeader);
const reader = createReadStream(file.tempPath);
for await (const chunk of reader) {
await writeHashed(chunk as Buffer);
}
entries.push({
fileName: file.fileName,
entryName,
crc32,
compressedSize: fileStats.size,
uncompressedSize: fileStats.size,
localHeaderOffset,
});
}
const centralDirectoryOffset = offset;
for (const entry of entries) {
const nameBuffer = Buffer.from(entry.entryName);
const { date, time } = dosDateTime();
await writeHashed(
Buffer.concat([
writeUInt32(0x02014b50),
writeUInt16(20),
writeUInt16(20),
writeUInt16(0),
writeUInt16(0),
writeUInt16(time),
writeUInt16(date),
writeUInt32(entry.crc32),
writeUInt32(entry.compressedSize),
writeUInt32(entry.uncompressedSize),
writeUInt16(nameBuffer.byteLength),
writeUInt16(0),
writeUInt16(0),
writeUInt16(0),
writeUInt16(0),
writeUInt32(0),
writeUInt32(entry.localHeaderOffset),
nameBuffer,
]),
);
}
const centralDirectorySize = offset - centralDirectoryOffset;
await writeHashed(
Buffer.concat([
writeUInt32(0x06054b50),
writeUInt16(0),
writeUInt16(0),
writeUInt16(entries.length),
writeUInt16(entries.length),
writeUInt32(centralDirectorySize),
writeUInt32(centralDirectoryOffset),
writeUInt16(0),
]),
);
await finishWriter(writer);
return {
tempPath,
sizeBytes: offset,
fileHash: hasher.digest('hex'),
entries,
};
} catch (error) {
writer.destroy();
await unlink(tempPath).catch(() => {});
throw error;
}
};
export const extractZipEntry = async (
zipBuffer: Buffer,
entryName: string,
): Promise<Buffer | null> => {
let offset = 0;
while (offset + 30 <= zipBuffer.byteLength) {
const signature = zipBuffer.readUInt32LE(offset);
if (signature !== 0x04034b50) break;
const compressionMethod = zipBuffer.readUInt16LE(offset + 8);
const compressedSize = zipBuffer.readUInt32LE(offset + 18);
const fileNameLength = zipBuffer.readUInt16LE(offset + 26);
const extraLength = zipBuffer.readUInt16LE(offset + 28);
const nameStart = offset + 30;
const nameEnd = nameStart + fileNameLength;
const dataStart = nameEnd + extraLength;
const dataEnd = dataStart + compressedSize;
const currentName = zipBuffer.subarray(nameStart, nameEnd).toString();
if (currentName === entryName) {
if (compressionMethod !== 0) return null;
return zipBuffer.subarray(dataStart, dataEnd);
}
offset = dataEnd;
}
return null;
};
export type LocatedZipEntry = {
start: number;
length: number;
};
export const locateZipEntry = async (
zipPath: string,
entryName: string,
): Promise<LocatedZipEntry | null> => {
const handle = await open(zipPath, 'r');
let offset = 0;
try {
const header = Buffer.alloc(30);
while (true) {
const { bytesRead } = await handle.read(header, 0, header.byteLength, offset);
if (bytesRead < header.byteLength) return null;
const signature = header.readUInt32LE(0);
if (signature !== 0x04034b50) return null;
const compressionMethod = header.readUInt16LE(8);
const compressedSize = header.readUInt32LE(18);
const fileNameLength = header.readUInt16LE(26);
const extraLength = header.readUInt16LE(28);
const nameBuffer = Buffer.alloc(fileNameLength);
const nameOffset = offset + 30;
await handle.read(nameBuffer, 0, fileNameLength, nameOffset);
const dataStart = nameOffset + fileNameLength + extraLength;
if (nameBuffer.toString() === entryName) {
if (compressionMethod !== 0) return null;
return { start: dataStart, length: compressedSize };
}
offset = dataStart + compressedSize;
}
} finally {
await handle.close();
}
};