fix: correct import paths and add missing protocol files for DDD structure

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude committed 2026-07-28 18:19:45 +07:00
1 parent 234ca7b14c
commit 4568644922
34 files changed
+6097 -5

No files matched your search

+48 -1
View File
@@ -1 +1,48 @@
export { fileInfoCache, Cache } from '../../utils/cache';
/**
* Generic in-memory cache with TTL (time-to-live) support.
* Entries expire after a configurable duration and are lazily evicted on access.
*
* @typeParam T - The type of values stored in the cache
*/
export class Cache<T> {
private store = new Map<string, CacheEntry<T>>();
private ttlMs: number;
constructor(ttlSeconds = 3600) {
this.ttlMs = ttlSeconds * 1000;
}
set(key: string, value: T): void {
this.store.set(key, { value, expiresAt: Date.now() + this.ttlMs });
}
get(key: string): T | null {
const entry = this.store.get(key);
if (!entry) return null;
if (Date.now() > entry.expiresAt) {
this.store.delete(key);
return null;
}
return entry.value;
}
has(key: string): boolean { return this.get(key) !== null; }
delete(key: string): void { this.store.delete(key); }
clear(): void { this.store.clear(); }
size(): number { return this.store.size; }
cleanup(): number {
let removed = 0;
const now = Date.now();
for (const [key, entry] of this.store.entries()) {
if (now > entry.expiresAt) { this.store.delete(key); removed++; }
}
return removed;
}
}
interface CacheEntry<T> { value: T; expiresAt: number }
export const fileInfoCache = new Cache<{
file_size: number; mime_type: string; file_path: string; bot_token: string;
}>(3600);
@@ -0,0 +1,16 @@
import { drizzle } from 'drizzle-orm/postgres-js';
import postgres from 'postgres';
import { fileParts, files } from './schema';
const client = postgres(process.env.DATABASE_URL!, {
max: 10,
idle_timeout: 20,
connect_timeout: 10,
});
/** Drizzle ORM database client initialized with the files and fileParts schema. */
export const db = drizzle(client, { schema: { fileParts, files } });
export { fileParts, files };
export default db;
@@ -0,0 +1,56 @@
import postgres from 'postgres';
import { config } from '../../../env';
import { getErrorMessage } from '../../../shared/utils/file';
import logger from '../../../shared/logger/index';
/**
* Run raw SQL migration from schema.sql.
* Safe to call multiple times — all statements use IF NOT EXISTS.
* Searches multiple relative paths to support execution from compiled dist,
* bun --hot, or direct script invocation.
*/
export const runMigration = async (): Promise<void> => {
// In compiled dist: import.meta.dir = .../dist/infrastructure/persistence/drizzle/
// In source via bun --hot: import.meta.dir = .../src/infrastructure/persistence/drizzle/
const dir = import.meta.dir || '';
const candidates = [
`${dir}/../../../../schema.sql`, // from dist/
`${dir}/../../../schema.sql`, // from src/infrastructure/persistence/
`${dir}/../../schema.sql`, // from src/infrastructure/
`${dir}/../schema.sql`, // from src/infrastructure/persistence/drizzle/
`${dir}/schema.sql`, // from next to file (bun run directly)
];
let schemaSql: string | null = null;
for (const p of candidates) {
const file = Bun.file(p);
const exists = await file.exists();
if (exists) {
schemaSql = await file.text();
break;
}
}
if (!schemaSql) {
logger.error(`Migration failed: schema.sql not found (tried ${candidates.join(', ')})`);
process.exitCode = 1;
return;
}
const sql = postgres(config.databaseUrl, { max: 1 });
try {
await sql.unsafe(schemaSql);
logger.info('Database migration completed');
} catch (error: unknown) {
logger.error('Database migration failed', { error: getErrorMessage(error) });
process.exitCode = 1;
} finally {
await sql.end();
}
};
// When run directly: `bun src/infrastructure/persistence/drizzle/migrate.ts`
if (import.meta.path === Bun.main) {
await runMigration();
}
@@ -0,0 +1,77 @@
import type { InferInsertModel, InferSelectModel } from 'drizzle-orm';
import {
bigint,
boolean,
integer,
pgTable,
serial,
text,
timestamp,
uuid,
} from 'drizzle-orm/pg-core';
/**
* Files table definition.
* Stores metadata about uploaded files including Telegram storage references,
* S3 bucket information, multipart upload tracking, and archive entries.
*/
export const files = pgTable('files', {
id: uuid('id').primaryKey().defaultRandom(),
publicId: text('public_id').unique().notNull(),
telegramFileId: text('telegram_file_id').notNull(),
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
fileName: text('file_name').notNull(),
mimeType: text('mime_type').notNull(),
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
fileType: text('file_type').notNull(),
uploaderId: bigint('uploader_id', { mode: 'number' }).notNull(),
fileHash: text('file_hash'),
archiveTelegramFileId: text('archive_telegram_file_id'),
archiveStorageMessageId: bigint('archive_storage_message_id', { mode: 'number' }),
archiveFileName: text('archive_file_name'),
archiveEntryName: text('archive_entry_name'),
archiveMimeType: text('archive_mime_type'),
archiveSizeBytes: bigint('archive_size_bytes', { mode: 'number' }),
bucketId: text('bucket_id'),
s3Key: text('s3_key'),
storageBackend: text('storage_backend').default('telegram'),
isDeleted: boolean('is_deleted').default(false),
multipartUploadId: text('multipart_upload_id'),
partCount: integer('part_count'),
createdAt: timestamp('created_at').defaultNow().notNull(),
updatedAt: timestamp('updated_at').defaultNow().notNull(),
});
/**
* File parts table definition.
* Stores chunks of multipart uploads with per-part Telegram storage references
* and compression metadata.
*/
export const fileParts = pgTable('file_parts', {
id: serial('id').primaryKey(),
fileId: uuid('file_id').notNull(),
partNumber: integer('part_number').notNull(),
telegramFileId: text('telegram_file_id').notNull(),
telegramFileUniqueId: text('telegram_file_unique_id').notNull(),
storageChatId: bigint('storage_chat_id', { mode: 'number' }).notNull(),
storageMessageId: bigint('storage_message_id', { mode: 'number' }).notNull(),
sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
storedSizeBytes: bigint('stored_size_bytes', { mode: 'number' }).notNull(),
compressionAlgorithm: text('compression_algorithm'),
etag: text('etag').notNull(),
createdAt: timestamp('created_at').defaultNow().notNull(),
});
/** Type representing a file row selected from the database. */
export type File = InferSelectModel<typeof files>;
/** Type representing a file row being inserted into the database. */
export type NewFile = InferInsertModel<typeof files>;
/** Type representing a file part row selected from the database. */
export type FilePart = InferSelectModel<typeof fileParts>;
/** Type representing a file part row being inserted into the database. */
export type NewFilePart = InferInsertModel<typeof fileParts>;
@@ -0,0 +1,100 @@
import { sql } from 'drizzle-orm';
import { db } from '../drizzle/index';
import type { Bucket } from '../../../domain/entities/bucket';
import type { IBucketRepository } from '../../../domain/ports/bucket-repository';
/** Raw result row from `db.execute()`. */
type QueryRow = Record<string, unknown>;
/** Array of raw result rows. */
type QueryResult = QueryRow[];
/**
* Maps a raw database row to a {@link Bucket} domain entity.
*/
const mapRowToBucket = (row: Record<string, unknown>): Bucket => ({
id: row.id as string,
name: row.name as string,
createdAt: new Date(row.created_at as string),
updatedAt: new Date(row.updated_at as string),
});
/**
* Drizzle-backed implementation of {@link IBucketRepository}.
*
* Delegates to the same SQL queries as the original `src/db/buckets.ts`
* module, using raw SQL for drizzle tables that are not part of the
* typed schema.
*/
export class DrizzleBucketRepository implements IBucketRepository {
/**
* {@inheritDoc IBucketRepository.create}
*/
async create(name: string): Promise<Bucket> {
const result = (await db.execute(
sql`INSERT INTO buckets (name) VALUES (${name}) RETURNING id, name, created_at, updated_at`,
)) as unknown as QueryResult;
return mapRowToBucket(result[0]!);
}
/**
* {@inheritDoc IBucketRepository.findByName}
*/
async findByName(name: string): Promise<Bucket | null> {
const result = (await db.execute(
sql`SELECT id, name, created_at, updated_at FROM buckets WHERE name = ${name}`,
)) as unknown as QueryResult;
if (result.length === 0) return null;
return mapRowToBucket(result[0]!);
}
/**
* {@inheritDoc IBucketRepository.list}
*/
async list(): Promise<Bucket[]> {
const result = (await db.execute(
sql`SELECT id, name, created_at, updated_at FROM buckets ORDER BY name`,
)) as unknown as QueryResult;
return result.map(mapRowToBucket);
}
/**
* {@inheritDoc IBucketRepository.delete}
*
* Cascade-deletes multipart and file rows that hold foreign-key
* references to the bucket before deleting the bucket itself.
* Failures during cascade are silently caught to match the original
* defensive-cleanup behaviour.
*/
async delete(name: string): Promise<boolean> {
// Cascade-delete rows that hold FK references to the bucket
await db
.execute(
sql`DELETE FROM multipart_parts WHERE upload_id IN (SELECT upload_id FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name}))`,
)
.catch(() => {});
await db
.execute(
sql`DELETE FROM multipart_uploads WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
)
.catch(() => {});
await db
.execute(
sql`DELETE FROM files WHERE bucket_id IN (SELECT id FROM buckets WHERE name = ${name})`,
)
.catch(() => {});
const result = (await db.execute(
sql`DELETE FROM buckets WHERE name = ${name}`,
)) as unknown as QueryResult;
return result.length > 0;
}
/**
* {@inheritDoc IBucketRepository.exists}
*/
async exists(name: string): Promise<boolean> {
const result = (await db.execute(
sql`SELECT 1 FROM buckets WHERE name = ${name}`,
)) as unknown as QueryResult;
return result.length > 0;
}
}
@@ -0,0 +1,106 @@
import { sql } from 'drizzle-orm';
import { db } from '../drizzle/index';
import type { FilePart, NewFilePart } from '../../../domain/entities/file-part';
import type { IFilePartRepository } from '../../../domain/ports/file-part-repository';
/** Compression algorithm type matching the domain entity. */
type CompressionAlgorithm = 'gzip' | null;
/** Safely converts a raw value to a number, defaulting to 0. */
const toNumber = (value: unknown): number => Number(value ?? 0);
/**
* Maps a raw database row (snake_case keys) to a {@link FilePart}
* domain entity.
*/
const mapRowToFilePart = (row: Record<string, unknown>): FilePart => ({
id: toNumber(row.id),
fileId: row.file_id as string,
partNumber: toNumber(row.part_number),
telegramFileId: row.telegram_file_id as string,
telegramFileUniqueId: row.telegram_file_unique_id as string,
storageChatId: toNumber(row.storage_chat_id),
storageMessageId: toNumber(row.storage_message_id),
sizeBytes: toNumber(row.size_bytes),
storedSizeBytes: toNumber(row.stored_size_bytes),
compressionAlgorithm:
(row.compression_algorithm as CompressionAlgorithm) || null,
etag: row.etag as string,
createdAt: new Date(row.created_at as string),
});
/**
* Drizzle-backed implementation of {@link IFilePartRepository}.
*
* Delegates to the same SQL queries as the original `src/db/file-parts.ts`
* module, using raw SQL for all operations.
*/
export class DrizzleFilePartRepository implements IFilePartRepository {
/**
* {@inheritDoc IFilePartRepository.insert}
*/
async insert(parts: NewFilePart[]): Promise<void> {
for (const part of parts) {
await db.execute(
sql`INSERT INTO file_parts (
file_id,
part_number,
telegram_file_id,
telegram_file_unique_id,
storage_chat_id,
storage_message_id,
size_bytes,
stored_size_bytes,
compression_algorithm,
etag
) VALUES (
${part.fileId}::uuid,
${part.partNumber},
${part.telegramFileId},
${part.telegramFileUniqueId},
${part.storageChatId},
${part.storageMessageId},
${part.sizeBytes},
${part.storedSizeBytes},
${part.compressionAlgorithm},
${part.etag}
)`,
);
}
}
/**
* {@inheritDoc IFilePartRepository.listByFileId}
*/
async listByFileId(fileId: string): Promise<FilePart[]> {
const result = (await db.execute(
sql`SELECT id,
file_id,
part_number,
telegram_file_id,
telegram_file_unique_id,
storage_chat_id,
storage_message_id,
size_bytes,
stored_size_bytes,
compression_algorithm,
etag,
created_at
FROM file_parts
WHERE file_id = ${fileId}::uuid
ORDER BY part_number`,
)) as unknown as Record<string, unknown>[];
return result.map(mapRowToFilePart);
}
/**
* {@inheritDoc IFilePartRepository.countByFileId}
*/
async countByFileId(fileId: string): Promise<number> {
const result = (await db.execute(
sql`SELECT COUNT(*) AS count FROM file_parts WHERE file_id = ${fileId}::uuid`,
)) as unknown as Record<string, unknown>[];
return toNumber(result[0]?.count);
}
}
@@ -0,0 +1,241 @@
import { and, eq, sql } from 'drizzle-orm';
import { db, files as fileSchema } from '../drizzle/index';
import type { File, NewFile } from '../../../domain/entities/file';
import type {
IFileRepository,
S3FileRecord,
} from '../../../domain/ports/file-repository';
/** Safely converts a raw value to a number, defaulting to 0. */
const toNumber = (value: unknown): number => Number(value ?? 0);
/**
* Escape special LIKE wildcard characters (`%`, `_`, `\`) so that
* a user-supplied prefix can be safely used in a LIKE expression.
*/
const escapeLike = (s: string): string => s.replace(/[%_\\]/g, '\\$&');
/**
* Maps a raw database row (snake_case keys) to an {@link S3FileRecord}
* domain entity. Used only when raw SQL via `db.execute()` returns
* un-typed result sets.
*/
const mapDbRowToS3Record = (row: Record<string, unknown>): S3FileRecord => ({
id: row.id as string,
publicId: row.public_id as string,
telegramFileId: row.telegram_file_id as string,
telegramFileUniqueId: row.telegram_file_unique_id as string,
storageChatId: toNumber(row.storage_chat_id),
storageMessageId: toNumber(row.storage_message_id),
fileName: row.file_name as string,
mimeType: row.mime_type as string,
sizeBytes: toNumber(row.size_bytes),
fileType: row.file_type as string,
uploaderId: toNumber(row.uploader_id),
fileHash: row.file_hash as string | null,
archiveTelegramFileId: row.archive_telegram_file_id as string | null,
archiveStorageMessageId:
row.archive_storage_message_id === null
? null
: toNumber(row.archive_storage_message_id),
archiveFileName: row.archive_file_name as string | null,
archiveEntryName: row.archive_entry_name as string | null,
archiveMimeType: row.archive_mime_type as string | null,
archiveSizeBytes:
row.archive_size_bytes === null
? null
: toNumber(row.archive_size_bytes),
bucketId: row.bucket_id as string,
s3Key: row.s3_key as string,
storageBackend: (row.storage_backend as string) || 'telegram',
isDeleted: row.is_deleted as boolean,
multipartUploadId: row.multipart_upload_id as string | null,
partCount:
row.part_count === null || row.part_count === undefined
? null
: toNumber(row.part_count),
createdAt: new Date(row.created_at as string),
updatedAt: new Date(row.updated_at as string),
});
/**
* Drizzle-backed implementation of {@link IFileRepository}.
*
* Delegates to the same SQL queries as the original `src/db/files.ts` and
* `src/db/files-ext.ts` modules while presenting a clean domain interface.
*/
export class DrizzleFileRepository implements IFileRepository {
/**
* {@inheritDoc IFileRepository.findByHash}
*/
async findByHash(hash: string): Promise<File | null> {
const result = await db
.select()
.from(fileSchema)
.where(eq(fileSchema.fileHash, hash))
.limit(1);
return result[0] || null;
}
/**
* {@inheritDoc IFileRepository.findByPublicId}
*/
async findByPublicId(publicId: string): Promise<File | null> {
const result = await db
.select()
.from(fileSchema)
.where(eq(fileSchema.publicId, publicId))
.limit(1);
return result[0] || null;
}
/**
* {@inheritDoc IFileRepository.findByUniqueId}
*/
async findByUniqueId(telegramFileUniqueId: string): Promise<File | null> {
const result = await db
.select()
.from(fileSchema)
.where(eq(fileSchema.telegramFileUniqueId, telegramFileUniqueId))
.limit(1);
return result[0] || null;
}
/**
* {@inheritDoc IFileRepository.findByBucketAndKey}
*/
async findByBucketAndKey(
bucketId: string,
s3Key: string,
): Promise<File | null> {
const result = await db
.select()
.from(fileSchema)
.where(
and(
eq(fileSchema.bucketId, bucketId),
eq(fileSchema.s3Key, s3Key),
eq(fileSchema.isDeleted, false),
),
)
.limit(1);
return result[0] || null;
}
/**
* {@inheritDoc IFileRepository.create}
*/
async create(file: NewFile): Promise<File> {
const result = await db
.insert(fileSchema)
.values(file)
.returning();
return result[0]!;
}
/**
* {@inheritDoc IFileRepository.listByPrefix}
*/
async listByPrefix(
bucketId: string,
prefix: string,
delimiter: string | null,
maxKeys: number,
startAfter: string | null,
): Promise<{ objects: S3FileRecord[]; prefixes: string[] }> {
let query = prefix
? sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false AND s3_key LIKE ${`${escapeLike(prefix)}%`}`
: sql`SELECT * FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`;
if (startAfter) {
query = sql`${query} AND s3_key > ${startAfter}`;
}
query = sql`${query} ORDER BY s3_key LIMIT ${maxKeys + 1}`;
const rawResult = (await db.execute(
query,
)) as unknown as Record<string, unknown>[];
if (delimiter === '/') {
const prefixSet = new Set<string>();
const objects: S3FileRecord[] = [];
for (const row of rawResult) {
const s3Key = row.s3_key as string;
const relativeKey = s3Key.substring(prefix.length);
const slashIndex = relativeKey.indexOf('/');
if (slashIndex >= 0) {
const folderPrefix =
prefix + relativeKey.substring(0, slashIndex + 1);
if (folderPrefix !== prefix) {
prefixSet.add(folderPrefix);
}
} else {
objects.push(mapDbRowToS3Record(row));
}
}
return {
objects: objects.slice(0, maxKeys),
prefixes: Array.from(prefixSet).sort(),
};
}
return {
objects: rawResult.slice(0, maxKeys).map(mapDbRowToS3Record),
prefixes: [],
};
}
/**
* {@inheritDoc IFileRepository.softDelete}
*/
async softDelete(bucketId: string, s3Key: string): Promise<boolean> {
const result = (await db.execute(
sql`UPDATE files SET is_deleted = true WHERE bucket_id = ${bucketId}::uuid AND s3_key = ${s3Key} RETURNING id`,
)) as unknown as Record<string, unknown>[];
return result.length > 0;
}
/**
* {@inheritDoc IFileRepository.softDeleteBatch}
*/
async softDeleteBatch(
bucketId: string,
keys: string[],
): Promise<number> {
let deleted = 0;
for (const key of keys) {
const ok = await this.softDelete(bucketId, key);
if (ok) deleted++;
}
return deleted;
}
/**
* {@inheritDoc IFileRepository.countByBucket}
*/
async countByBucket(bucketId: string): Promise<number> {
const result = (await db.execute(
sql`SELECT count(*) as count FROM files WHERE bucket_id = ${bucketId}::uuid AND is_deleted = false`,
)) as unknown as Record<string, unknown>[];
return Number(result[0]?.count || 0);
}
/**
* {@inheritDoc IFileRepository.findOrphansByBucket}
*/
async findOrphansByBucket(bucketId: string): Promise<File[]> {
return await db
.select()
.from(fileSchema)
.where(
and(
eq(fileSchema.bucketId, bucketId),
eq(fileSchema.isDeleted, true),
),
)
.limit(100);
}
}
@@ -0,0 +1,149 @@
import { sql } from 'drizzle-orm';
import { nanoid } from 'nanoid';
import { db } from '../drizzle/index';
import type { MultipartUpload, MultipartPart } from '../../../domain/entities/multipart';
import type { IMultipartRepository } from '../../../domain/ports/multipart-repository';
/**
* Maps a raw database row to a {@link MultipartUpload} domain entity.
*/
const mapRowToMultipartUpload = (
r: Record<string, unknown>,
): MultipartUpload => ({
uploadId: r.upload_id as string,
bucketId: r.bucket_id as string,
s3Key: r.s3_key as string,
initiatedAt: new Date(r.initiated_at as string),
status: r.status as string,
initiatedBy: (r.initiated_by as string | null) || '',
});
/**
* Drizzle-backed implementation of {@link IMultipartRepository}.
*
* Delegates to the same SQL queries as the original `src/db/multipart.ts`
* module, using raw SQL for all operations on the un-typed
* `multipart_uploads` and `multipart_parts` tables.
*/
export class DrizzleMultipartRepository implements IMultipartRepository {
/**
* {@inheritDoc IMultipartRepository.create}
*/
async create(
bucketId: string,
s3Key: string,
initiatedBy: string,
): Promise<string> {
const uploadId = nanoid(32);
await db.execute(
sql`INSERT INTO multipart_uploads (upload_id, bucket_id, s3_key, initiated_by) VALUES (${uploadId}, ${bucketId}, ${s3Key}, ${initiatedBy})`,
);
return uploadId;
}
/**
* {@inheritDoc IMultipartRepository.findById}
*/
async findById(uploadId: string): Promise<MultipartUpload | null> {
const result = (await db.execute(
sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status FROM multipart_uploads WHERE upload_id = ${uploadId} AND status = 'in_progress'`,
)) as unknown as Record<string, unknown>[];
if (result.length === 0) return null;
const r = result[0]!;
return {
uploadId: r.upload_id as string,
bucketId: r.bucket_id as string,
s3Key: r.s3_key as string,
initiatedAt: new Date(r.initiated_at as string),
status: r.status as string,
initiatedBy: '',
};
}
/**
* {@inheritDoc IMultipartRepository.complete}
*/
async complete(uploadId: string): Promise<void> {
await db.execute(
sql`UPDATE multipart_uploads SET status = 'completed' WHERE upload_id = ${uploadId}`,
);
}
/**
* {@inheritDoc IMultipartRepository.abort}
*/
async abort(uploadId: string): Promise<void> {
await db.execute(
sql`UPDATE multipart_uploads SET status = 'aborted' WHERE upload_id = ${uploadId}`,
);
}
/**
* {@inheritDoc IMultipartRepository.insertPart}
*/
async insertPart(
part: Omit<MultipartPart, 'id' | 'createdAt'>,
): Promise<void> {
await db.execute(
sql`INSERT INTO multipart_parts (upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag)
VALUES (${part.uploadId}, ${part.partNumber}, ${part.telegramFileId}, ${part.telegramFileUniqueId}, ${part.storageMessageId}, ${part.sizeBytes}, ${part.etag})`,
);
}
/**
* {@inheritDoc IMultipartRepository.listParts}
*/
async listParts(uploadId: string): Promise<MultipartPart[]> {
const result = (await db.execute(
sql`SELECT id, upload_id, part_number, telegram_file_id, telegram_file_unique_id, storage_message_id, size_bytes, etag, created_at
FROM multipart_parts WHERE upload_id = ${uploadId} ORDER BY part_number`,
)) as unknown as Record<string, unknown>[];
return result.map((r) => ({
id: r.id as number,
uploadId: r.upload_id as string,
partNumber: r.part_number as number,
telegramFileId: r.telegram_file_id as string,
telegramFileUniqueId: r.telegram_file_unique_id as string,
storageMessageId: r.storage_message_id as number,
sizeBytes: Number(r.size_bytes),
etag: r.etag as string,
createdAt: new Date(r.created_at as string),
}));
}
/**
* {@inheritDoc IMultipartRepository.listByBucket}
*/
async listByBucket(
bucketId: string,
maxUploads: number,
keyMarker: string | null,
): Promise<{
uploads: MultipartUpload[];
isTruncated: boolean;
nextKeyMarker: string | null;
}> {
const limit = Math.min(Math.max(maxUploads || 1000, 1), 1000);
const result = (await db.execute(
keyMarker
? sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
FROM multipart_uploads
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress' AND s3_key > ${keyMarker}
ORDER BY s3_key, initiated_at
LIMIT ${limit + 1}`
: sql`SELECT upload_id, bucket_id, s3_key, initiated_at, status, initiated_by
FROM multipart_uploads
WHERE bucket_id = ${bucketId}::uuid AND status = 'in_progress'
ORDER BY s3_key, initiated_at
LIMIT ${limit + 1}`,
)) as unknown as Record<string, unknown>[];
const uploads = result.slice(0, limit).map(mapRowToMultipartUpload);
return {
uploads,
isTruncated: result.length > limit,
nextKeyMarker:
result.length > limit ? uploads.at(-1)?.s3Key || null : null,
};
}
}
+213
View File
@@ -0,0 +1,213 @@
import { Telegraf } from 'telegraf';
import { config } from '../../env';
import logger from '../../shared/logger/index';
import type { ITelegramService, ForwardResult, TelegramFileInfo } from '../../domain/ports/telegram-service';
import { enqueueUpload } from './upload-queue';
import {
sendMethodMap,
extractUploadedFile,
buildSendPayload,
type TelegramMessageResult,
type SendMethod,
} from './types';
/**
* Sleep for a given number of seconds.
*
* Used as a backoff mechanism when all bots in the pool are rate-limited.
*
* @param seconds - Number of seconds to sleep.
* @returns A promise that resolves after the specified delay.
*/
const sleep = (seconds: number): Promise<void> => {
return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
};
/**
* Manages a pool of Telegram bots with automatic rotation and rate-limit handling.
*
* Distributes uploads across multiple bot tokens to maximise throughput.
* When a bot receives a 429 (rate-limit) error, the pool instantly rotates
* to the next available bot. If all bots are rate-limited, a coordinated
* sleep is performed before retrying.
*
* Implements the {@link ITelegramService} contract.
*/
export class BotPool implements ITelegramService {
private readonly bots: Telegraf[];
private readonly botTokens: string[];
private nextBotIndex = 0;
/** Create a new BotPool from the application configuration. */
constructor() {
this.botTokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens]));
this.bots = this.botTokens.map((token) => new Telegraf(token));
}
/**
* Claim the next bot index using round-robin rotation.
*
* @returns The index of the selected bot.
*/
private claimBotIndex(): number {
const botIndex = this.nextBotIndex;
this.nextBotIndex = (this.nextBotIndex + 1) % this.bots.length;
return botIndex;
}
/**
* Execute a Telegram API action with automatic retry and bot rotation.
*
* On 429 errors the pool either:
* 1. Rotates to the next bot immediately (if another bot is available), or
* 2. Sleeps for the required duration after all bots are exhausted, then retries.
*
* @param action - The action to execute on a bot instance.
* @param retries - Number of full-pool retry cycles remaining.
* @param attemptedBots - Number of bots attempted in the current cycle.
* @returns The result of the action.
*/
private async executeWithBotRetry<T>(
action: (botInstance: Telegraf, botToken: string) => Promise<T>,
retries = 5,
attemptedBots = 0,
): Promise<T> {
const botIndex = this.claimBotIndex();
const currentBot = this.bots[botIndex];
const currentToken = this.botTokens[botIndex];
try {
return await action(currentBot, currentToken);
} catch (error: unknown) {
const errorStr = error instanceof Error ? error.message : String(error);
const match = errorStr.match(/retry after (\d+)/i);
if (match) {
const nextIndex = this.nextBotIndex;
const nextAttemptedBots = attemptedBots + 1;
if (nextAttemptedBots < this.bots.length) {
logger.info(
`Bot Index ${botIndex} hit 429. Instantly rotating to Bot Index ${nextIndex}...`,
);
return this.executeWithBotRetry(action, retries, nextAttemptedBots);
}
if (retries > 0) {
const seconds = parseInt(match[1], 10);
logger.warn(
`All bots in the pool are rate-limited. Sleeping for ${seconds} seconds...`,
{ error: errorStr },
);
await sleep(seconds);
return this.executeWithBotRetry(action, retries - 1, 0);
}
}
throw error;
}
}
/**
* Forward a file chunk to the configured Telegram storage chat.
*
* The upload is queued (via {@link enqueueUpload}) and executed with
* automatic bot rotation on rate-limit errors.
*
* @param fileChunk - The file data (ReadStream, Buffer, or file path).
* @param fileName - The original file name.
* @param fileType - The file type classification (e.g. "document", "photo").
* @returns The Telegram identifiers of the stored file.
*/
async forwardToStorage(
fileChunk: unknown,
fileName: string,
fileType: string,
): Promise<ForwardResult> {
try {
const result = await this.enqueueUpload<TelegramMessageResult>(async () => {
const filePayload = { source: fileChunk, filename: fileName };
const sendMethodName = sendMethodMap[fileType] || 'sendDocument';
const payload = buildSendPayload(fileType, fileName);
return this.executeWithBotRetry<TelegramMessageResult>((activeBot) => {
const telegram = activeBot.telegram as unknown as Record<string, SendMethod>;
return telegram[sendMethodName](config.storageChatId, filePayload, payload);
});
});
const uploadedFile = extractUploadedFile(result, fileType);
logger.info('File forwarded to storage', { fileName, message: result.message_id });
return {
telegramFileId: uploadedFile?.file_id || '',
telegramFileUniqueId: uploadedFile?.file_unique_id || '',
storageMessageId: result.message_id,
};
} catch (error: unknown) {
logger.error('Failed to forward file to storage', {
fileName,
error: error instanceof Error ? error.message : String(error),
});
throw error;
}
}
/**
* Retrieve file metadata from Telegram by file ID.
*
* Tries all configured bots sequentially; returns info from the first
* bot that can retrieve the file. Errors indicating the file belongs
* to a different bot are silently skipped.
*
* @param telegramFileId - The Telegram file_id to look up.
* @returns Metadata including size, MIME type, download path, and bot token.
*/
async getFileInfo(telegramFileId: string): Promise<TelegramFileInfo> {
let lastError: unknown;
for (const activeBot of this.bots) {
try {
const result = await activeBot.telegram.getFile(telegramFileId);
const fileData = result as unknown as Omit<TelegramFileInfo, 'bot_token'>;
return {
file_size: fileData.file_size || 0,
mime_type: fileData.mime_type || 'application/octet-stream',
file_path: fileData.file_path || '',
bot_token: activeBot.telegram.token,
};
} catch (error: unknown) {
lastError = error;
const errorStr = error instanceof Error ? error.message : String(error);
if (
errorStr.includes('wrong file_id') ||
errorStr.includes('file is temporarily unavailable') ||
errorStr.includes('retry after')
) {
continue;
}
throw error;
}
}
logger.error('Failed to get file info from any bot', {
error: lastError instanceof Error ? lastError.message : String(lastError),
});
throw lastError;
}
/**
* Enqueue a task for sequential upload execution.
*
* Delegates to the shared upload queue to ensure only a limited number
* of Telegram uploads run concurrently.
*
* @param task - An async function performing the upload.
* @returns The result of the task.
*/
enqueueUpload<T>(task: () => Promise<T>): Promise<T> {
return enqueueUpload(task);
}
}
/**
* Singleton BotPool instance initialised from application configuration.
*/
export const botPool = new BotPool();
@@ -0,0 +1,336 @@
import { createReadStream } from 'node:fs';
import { gzipSync } from 'node:zlib';
import { nanoid } from 'nanoid';
import { config } from '../../env';
import { computeHash } from '../../utils/file';
import { createGetObjectResponse, type ObjectPartSource } from '../../utils/s3/object-stream';
import type { RangeParseResult } from '../../utils/s3/range';
import type { IFileRepository } from '../../domain/ports/file-repository';
import type { IFilePartRepository } from '../../domain/ports/file-part-repository';
import type { ITelegramService } from '../../domain/ports/telegram-service';
import type { File as FileEntity } from '../../domain/entities/file';
import type { NewFilePart, CompressionAlgorithm } from '../../domain/entities/file-part';
/**
* Chunk compression algorithm identifier.
* `"gzip"` if gzip compression was applied, `null` for uncompressed.
*/
export type ChunkCompressionAlgorithm = CompressionAlgorithm;
/**
* Metadata about a single uploaded chunk (part) stored in Telegram.
*/
export interface ChunkedUploadPart {
/** 1-based part number within the file */
partNumber: number;
/** Telegram file_id for retrieving this part */
telegramFileId: string;
/** Telegram unique file_id (stable across bot tokens) */
telegramFileUniqueId: string;
/** Message ID within the storage chat */
storageMessageId: number;
/** Original (pre-compression) size in bytes */
sizeBytes: number;
/** Stored (post-compression) size in bytes */
storedSizeBytes: number;
/** Compression algorithm applied, or null */
compressionAlgorithm: ChunkCompressionAlgorithm;
/** ETag (SHA-256 hash) of the original chunk */
etag: string;
}
/**
* Result of uploading a file in Telegram chunks.
*/
export interface ChunkedUploadResult {
/** Ordered list of uploaded parts */
parts: ChunkedUploadPart[];
/** SHA-256 hash of the complete file content */
fileHash: string;
/** Total file size in bytes */
totalSizeBytes: number;
}
/**
* Input parameters for storing a file via chunked Telegram uploads.
*/
export interface ChunkedFileInput {
/** Path to the temporary file on disk */
tempPath: string;
/** Prefix for generated part file names */
partFileNamePrefix: string;
/** Original file name */
fileName: string;
/** MIME type of the file */
mimeType: string;
/** File size in bytes */
sizeBytes: number;
/** File type classification (e.g. "document", "video") */
fileType: string;
/** Telegram user ID of the uploader */
uploaderId: number;
/** S3 bucket ID if the file is also tracked in S3, or null */
bucketId?: string | null;
/** S3 object key if the file is also tracked in S3, or null */
s3Key?: string | null;
}
/**
* Validate and sanitise the Telegram chunk size.
*
* @param chunkSizeBytes - The desired chunk size in bytes.
* @returns The validated chunk size.
* @throws {Error} If the chunk size is not a safe positive integer.
*/
const asSafeChunkSize = (chunkSizeBytes: number): number => {
if (!Number.isSafeInteger(chunkSizeBytes) || chunkSizeBytes <= 0) {
throw new Error('Invalid Telegram chunk size');
}
return chunkSizeBytes;
};
/**
* Optionally compress a chunk with gzip.
*
* Compression is skipped if:
* - The `compress` flag is false.
* - The chunk is smaller than `compressionMinSizeBytes`.
* - The compressed result is larger than the original.
*
* @param chunk - The raw chunk buffer.
* @param compress - Whether compression is enabled.
* @param compressionMinSizeBytes - Minimum chunk size to attempt compression.
* @returns The (possibly compressed) bytes and the algorithm used.
*/
const maybeCompressChunk = (
chunk: Buffer,
compress: boolean,
compressionMinSizeBytes: number,
): { bytes: Buffer; compressionAlgorithm: ChunkCompressionAlgorithm } => {
if (!compress || chunk.byteLength < compressionMinSizeBytes) {
return { bytes: chunk, compressionAlgorithm: null };
}
const gzipped = gzipSync(chunk);
if (gzipped.byteLength >= chunk.byteLength) {
return { bytes: chunk, compressionAlgorithm: null };
}
return { bytes: gzipped, compressionAlgorithm: 'gzip' };
};
/**
* Manages chunked storage of large files in Telegram.
*
* Large files are split into smaller chunks, each uploaded as a separate
* Telegram document. File and part metadata is persisted through the
* provided repository interfaces.
*
* Injects dependencies via constructor — can be used with any
* {@link IFileRepository}, {@link IFilePartRepository}, and
* {@link ITelegramService} implementation.
*/
export class ChunkedStorage {
/**
* @param fileRepository - Repository for File entity persistence.
* @param filePartRepository - Repository for FilePart entity persistence.
* @param telegramService - Service for Telegram API interactions.
*/
constructor(
private readonly fileRepository: IFileRepository,
private readonly filePartRepository: IFilePartRepository,
private readonly telegramService: ITelegramService,
) {}
/**
* Upload a file to Telegram in chunks and return chunk metadata.
*
* Reads the file from disk in fixed-size chunks, compresses each chunk
* if beneficial, and forwards each chunk to Telegram storage.
*
* @param input - Upload parameters including temp path, chunk size, and compression settings.
* @returns Metadata about all uploaded chunks and the file hash.
*/
async uploadFileInTelegramChunks(input: {
tempPath: string;
partFileNamePrefix: string;
chunkSizeBytes: number;
compress: boolean;
compressionMinSizeBytes: number;
}): Promise<ChunkedUploadResult> {
const chunkSizeBytes = asSafeChunkSize(input.chunkSizeBytes);
const hasher = new Bun.CryptoHasher('sha256');
const parts: ChunkedUploadPart[] = [];
let totalSizeBytes = 0;
let partNumber = 0;
const stream = createReadStream(input.tempPath, { highWaterMark: chunkSizeBytes });
for await (const data of stream) {
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data as Uint8Array);
if (chunk.byteLength === 0) continue;
partNumber += 1;
totalSizeBytes += chunk.byteLength;
hasher.update(chunk);
const { bytes, compressionAlgorithm } = maybeCompressChunk(
chunk,
input.compress,
input.compressionMinSizeBytes,
);
const forwardResult = await this.telegramService.forwardToStorage(
bytes,
`${input.partFileNamePrefix}.part-${partNumber}`,
'document',
);
parts.push({
partNumber,
telegramFileId: forwardResult.telegramFileId,
telegramFileUniqueId: forwardResult.telegramFileUniqueId,
storageMessageId: forwardResult.storageMessageId,
sizeBytes: chunk.byteLength,
storedSizeBytes: bytes.byteLength,
compressionAlgorithm,
etag: computeHash(chunk),
});
}
return {
parts,
fileHash: hasher.digest('hex'),
totalSizeBytes,
};
}
/**
* Upload a file to Telegram in chunks and persist file + part records.
*
* Combines chunk upload ({@link uploadFileInTelegramChunks}) with
* repository persistence for both the File and FilePart entities.
*
* @param input - The file metadata and upload parameters.
* @returns The persisted File entity.
*/
async storeFileInTelegramChunks(input: ChunkedFileInput): Promise<FileEntity> {
const upload = await this.uploadFileInTelegramChunks({
tempPath: input.tempPath,
partFileNamePrefix: input.partFileNamePrefix,
chunkSizeBytes: config.telegramChunkSizeBytes,
compress: config.compressChunkedUploads,
compressionMinSizeBytes: config.chunkCompressionMinSizeBytes,
});
const firstPart = upload.parts[0];
if (!firstPart) {
throw new Error('Chunked upload produced no parts');
}
const publicId = nanoid();
const file = await this.fileRepository.create({
publicId,
telegramFileId: firstPart.telegramFileId,
telegramFileUniqueId: firstPart.telegramFileUniqueId,
storageChatId: config.storageChatId,
storageMessageId: firstPart.storageMessageId,
fileName: input.fileName,
mimeType: input.mimeType,
sizeBytes: upload.totalSizeBytes,
fileType: input.fileType,
uploaderId: input.uploaderId,
fileHash: upload.fileHash,
archiveTelegramFileId: null,
archiveStorageMessageId: null,
archiveFileName: null,
archiveEntryName: null,
archiveMimeType: null,
archiveSizeBytes: null,
bucketId: input.bucketId ?? null,
s3Key: input.s3Key ?? null,
storageBackend: 'chunked',
isDeleted: false,
multipartUploadId: null,
partCount: upload.parts.length,
});
const fileParts: NewFilePart[] = upload.parts.map((part) => ({
fileId: file.id,
partNumber: part.partNumber,
telegramFileId: part.telegramFileId,
telegramFileUniqueId: part.telegramFileUniqueId,
storageChatId: config.storageChatId,
storageMessageId: part.storageMessageId,
sizeBytes: part.sizeBytes,
storedSizeBytes: part.storedSizeBytes,
compressionAlgorithm: part.compressionAlgorithm,
etag: part.etag,
}));
await this.filePartRepository.insert(fileParts);
return file;
}
/**
* Build a list of object-part sources for reconstructing a chunked file.
*
* Queries the file-part repository and enriches each part with
* the Telegram download URL by calling {@link ITelegramService.getFileInfo}.
*
* @param file - The File entity whose parts should be resolved.
* @returns An ordered list of object part sources ready for streaming.
*/
async buildChunkedObjectSources(file: FileEntity): Promise<ObjectPartSource[]> {
const parts = await this.filePartRepository.listByFileId(file.id);
const sources: ObjectPartSource[] = [];
for (const part of parts) {
const fileInfo = await this.telegramService.getFileInfo(part.telegramFileId);
sources.push({
telegramFileId: part.telegramFileId,
telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
sizeBytes: part.sizeBytes,
storedSizeBytes: part.storedSizeBytes,
compressionAlgorithm: part.compressionAlgorithm,
partNumber: part.partNumber,
});
}
return sources;
}
/**
* Create an HTTP Response that streams a chunked file's content.
*
* Supports HTTP range requests for partial content delivery.
* The response is constructed by reassembling parts in order and
* optionally decompressing gzip-compressed parts.
*
* @param input - Parameters including the file entity, range, and request ID.
* @returns A Response object streaming the requested byte range.
*/
async createChunkedObjectResponse(input: {
file: FileEntity;
range: RangeParseResult;
reqId: string;
}): Promise<Response> {
const parts = await this.buildChunkedObjectSources(input.file);
if (parts.length === 0) {
throw new Error('Chunked object has no parts');
}
return createGetObjectResponse({
reqId: input.reqId,
contentType: input.file.mimeType,
etag: input.file.fileHash || parts.map((p) => p.telegramFileId).join('-'),
lastModified:
input.file.createdAt instanceof Date
? input.file.createdAt
: new Date(input.file.createdAt),
totalSize: Number(input.file.sizeBytes),
parts,
range: input.range,
});
}
}
+130
View File
@@ -0,0 +1,130 @@
/**
* Telegram-specific types used internally by the infrastructure layer.
*
* These types represent the raw Telegram Bot API response shapes and
* the internal abstractions built on top of them. The higher-level domain
* types (ForwardResult, TelegramFileInfo) are defined in
* src/domain/ports/telegram-service.ts.
*/
/**
* File reference within a Telegram message result.
* Contains identifiers returned by the Telegram API for uploaded media.
*/
export interface UploadedTelegramFile {
/** Telegram file_id for retrieving the file */
file_id?: string;
/** Telegram unique file_id (stable across bot tokens) */
file_unique_id?: string;
}
/**
* Result structure returned by Telegram send* API methods.
* Covers all media types a Telegram message can carry.
*/
export interface TelegramMessageResult {
/** Unique message identifier inside the chat */
message_id: number;
/** Sent document, if applicable */
document?: UploadedTelegramFile;
/** Sent photo (array of sizes, last element is largest), if applicable */
photo?: UploadedTelegramFile[];
/** Sent video, if applicable */
video?: UploadedTelegramFile;
/** Sent audio, if applicable */
audio?: UploadedTelegramFile;
/** Sent voice message, if applicable */
voice?: UploadedTelegramFile;
/** Sent animation (GIF), if applicable */
animation?: UploadedTelegramFile;
/** Sent sticker, if applicable */
sticker?: UploadedTelegramFile;
/** Sent video note, if applicable */
video_note?: UploadedTelegramFile;
/** Catch-all for any additional Telegram response fields */
[key: string]: unknown;
}
/**
* Payload structure for sending a file via the Telegram Bot API.
*
* @internal
*/
export type FilePayload = { source: unknown; filename: string };
/**
* Additional optional payload for Telegram send method calls.
*
* @internal
*/
export type SendPayload = { caption?: string };
/**
* Function signature for Telegram send* method calls on a bot instance.
*
* @internal
*/
export type SendMethod = (
chatId: number,
filePayload: FilePayload,
payload?: SendPayload,
) => Promise<TelegramMessageResult>;
/**
* Mapping from file type identifier to Telegram Bot API method name.
*
* Each key corresponds to a Telegram media type; the value is the
* method name to call on `bot.telegram`.
*/
export const sendMethodMap: Record<string, string> = {
photo: 'sendPhoto',
audio: 'sendAudio',
video: 'sendVideo',
voice: 'sendVoice',
animation: 'sendAnimation',
sticker: 'sendSticker',
document: 'sendDocument',
video_note: 'sendDocument',
};
/**
* Extract the uploaded file reference from a Telegram message result
* based on the media type present in the result.
*
* Falls back to looking up the file type key directly on the result object.
*
* @param result - The message result from a Telegram send* call.
* @param fileType - The file type classification (e.g. "document", "photo").
* @returns The uploaded file reference, or `undefined` if none was found.
*/
export const extractUploadedFile = (
result: TelegramMessageResult,
fileType: string,
): UploadedTelegramFile | undefined => {
if (result.document) return result.document;
if (result.photo) return result.photo?.slice(-1)[0];
if (result.video) return result.video;
if (result.audio) return result.audio;
if (result.voice) return result.voice;
if (result.animation) return result.animation;
if (result.sticker) return result.sticker;
if (result.video_note) return result.video_note;
return result[fileType] as UploadedTelegramFile | undefined;
};
/**
* Build the send payload (caption, etc.) for a Telegram send* method call.
*
* Stickers do not support captions. Documents get a labelled caption
* with the file name. All other types use the plain file name as caption.
*
* @param fileType - The file type (e.g. "document", "photo", "sticker").
* @param fileName - The file name to use in the caption.
* @returns The payload object with caption (or empty for sticker).
*/
export const buildSendPayload = (fileType: string, fileName: string): SendPayload => {
const basePayload: SendPayload = { caption: fileName };
if (fileType === 'sticker') return {};
if (fileType === 'document') return { caption: `📁 ${fileName}` };
return basePayload;
};
@@ -0,0 +1,238 @@
import { createReadStream } from 'node:fs';
import { nanoid } from 'nanoid';
import { config } from '../../env';
import { cleanupTempFile } from '../../shared/utils/file';
import { createZip, type ZipEntry } from '../../shared/utils/zip';
import type { IFileRepository } from '../../domain/ports/file-repository';
import type { ITelegramService } from '../../domain/ports/telegram-service';
import type { File as FileEntity, NewFile } from '../../domain/entities/file';
/**
* Metadata about a prepared upload before it is submitted to the batcher.
*/
export type PreparedUpload = {
/** Temporary file path on disk */
tempPath: string;
/** SHA-256 hash of the file contents */
fileHash: string;
/** File size in bytes */
sizeBytes: number;
/** First bytes of the file for MIME detection */
signatureBuffer: Buffer;
};
/**
* A fully materialised file record returned from the batcher.
*/
export type UploadedFile = FileEntity;
/**
* An item ready for batched upload to Telegram storage.
*/
export type BatchUploadItem = {
/** Prepared upload metadata */
prepared: PreparedUpload;
/** Original file name */
fileName: string;
/** MIME type of the file */
mimeType: string;
/** File type classification (e.g. "document", "photo") */
fileType: string;
};
/**
* Internal pending upload tracking type, extending BatchUploadItem
* with resolve/reject callbacks.
*/
type PendingUpload = BatchUploadItem & {
resolve: (file: FileEntity) => void;
reject: (error: unknown) => void;
};
/** Time window in milliseconds during which uploads are batched together. */
const BATCH_WINDOW_MS = 2000;
/**
* Batches multiple file uploads into a single ZIP archive before forwarding
* them to Telegram storage. This reduces the number of Telegram API calls
* and improves throughput for small-file workloads.
*
* Injects dependencies via constructor — can be used with any
* {@link IFileRepository} and {@link ITelegramService} implementation.
*/
export class UploadBatcher {
private readonly pendingUploads: PendingUpload[] = [];
private flushTimer: ReturnType<typeof setTimeout> | null = null;
/**
* @param fileRepository - Repository for persisting file records.
* @param telegramService - Service for forwarding files to Telegram storage.
*/
constructor(
private readonly fileRepository: IFileRepository,
private readonly telegramService: ITelegramService,
) {}
/**
* Build a NewFile record from a batch item and its archive metadata.
*
* @param item - The batched upload item.
* @param entry - ZIP entry metadata for the individual file.
* @param archive - Archive-level Telegram storage metadata.
* @returns A NewFile record ready for repository insertion.
*/
private buildUploadedFile(
item: BatchUploadItem,
entry: ZipEntry,
archive: {
telegramFileId: string;
telegramFileUniqueId: string;
storageMessageId: number;
fileName: string;
sizeBytes: number;
},
): NewFile {
return {
publicId: nanoid(),
telegramFileId: archive.telegramFileId,
telegramFileUniqueId: archive.telegramFileUniqueId,
storageChatId: config.storageChatId,
storageMessageId: archive.storageMessageId,
fileName: item.fileName,
mimeType: item.mimeType || 'application/octet-stream',
sizeBytes: item.prepared.sizeBytes,
fileType: item.fileType,
uploaderId: 0,
fileHash: item.prepared.fileHash,
archiveTelegramFileId: archive.telegramFileId,
archiveStorageMessageId: archive.storageMessageId,
archiveFileName: archive.fileName,
archiveEntryName: entry.entryName,
archiveMimeType: 'application/zip',
archiveSizeBytes: archive.sizeBytes,
bucketId: null,
s3Key: null,
storageBackend: null,
isDeleted: null,
multipartUploadId: null,
partCount: null,
};
}
/**
* Flush all pending uploads by zipping them together and sending
* the archive to Telegram storage.
*/
private async flushUploads(): Promise<void> {
if (this.flushTimer) {
clearTimeout(this.flushTimer);
this.flushTimer = null;
}
const batch = this.pendingUploads.splice(0);
if (batch.length === 0) return;
let zipTempPath: string | null = null;
try {
const zip = await createZip(
batch.map((item) => ({ tempPath: item.prepared.tempPath, fileName: item.fileName })),
);
zipTempPath = zip.tempPath;
const archiveFileName = `filedrop-${nanoid()}.zip`;
const archiveResult = await this.telegramService.forwardToStorage(
createReadStream(zip.tempPath),
archiveFileName,
'document',
);
const newFileInputs = batch.map((item, index) =>
this.buildUploadedFile(item, zip.entries[index], {
telegramFileId: archiveResult.telegramFileId,
telegramFileUniqueId: archiveResult.telegramFileUniqueId,
storageMessageId: archiveResult.storageMessageId,
fileName: archiveFileName,
sizeBytes: zip.sizeBytes,
}),
);
// Persist each file record through the repository
const createdFiles = await Promise.all(
newFileInputs.map((input) => this.fileRepository.create(input)),
);
for (let i = 0; i < batch.length; i++) {
batch[i].resolve(createdFiles[i]);
}
} catch (error) {
for (const item of batch) {
item.reject(error);
}
} finally {
await Promise.all(batch.map((item) => cleanupTempFile(item.prepared.tempPath)));
if (zipTempPath) await cleanupTempFile(zipTempPath);
// Reschedule timer if new items arrived during async processing
if (this.pendingUploads.length > 0 && !this.flushTimer) {
this.flushTimer = setTimeout(() => {
void this.flushUploads();
}, BATCH_WINDOW_MS);
}
}
}
/**
* Calculate total size of all pending uploads in bytes.
*
* @returns The sum of all pending file sizes.
*/
private getPendingSize(): number {
return this.pendingUploads.reduce((total, item) => total + item.prepared.sizeBytes, 0);
}
/**
* Enqueue a prepared upload for batched processing.
*
* The upload is held for up to {@link BATCH_WINDOW_MS} milliseconds
* (or until the batch size/byte thresholds in config are exceeded)
* before being flushed to Telegram storage.
*
* @param item - The prepared upload item to enqueue.
* @returns A promise that resolves with the fully created File record.
*/
enqueuePreparedUpload(item: BatchUploadItem): Promise<FileEntity> {
return new Promise<FileEntity>((resolve, reject) => {
this.pendingUploads.push({ ...item, resolve, reject });
if (!this.flushTimer) {
this.flushTimer = setTimeout(() => {
void this.flushUploads();
}, BATCH_WINDOW_MS);
}
if (
this.pendingUploads.length >= config.batchMaxItems ||
this.getPendingSize() >= config.batchMaxSizeBytes
) {
void this.flushUploads();
}
});
}
/**
* Immediately flush all pending uploads, regardless of batch size.
*
* @returns A promise that resolves when the flush is complete.
*/
async flushPendingUploads(): Promise<void> {
await this.flushUploads();
}
/**
* Get the number of uploads currently waiting in the batch queue.
*
* @returns The pending upload count.
*/
getPendingUploadCount(): number {
return this.pendingUploads.length;
}
}
@@ -0,0 +1,81 @@
import PQueue from 'p-queue';
import { config } from '../../env';
import logger from '../../shared/logger/index';
/**
* P-queue instance for serialising Telegram upload tasks.
*
* Concurrency is governed by {@link config.uploadConcurrency}.
* Built-in logging emits warnings when the queue grows beyond 5 pending items.
*/
const uploadQueue = new PQueue({
concurrency: config.uploadConcurrency,
});
/* Monitor queue growth and emit warnings for large backlogs */
uploadQueue.on('add', () => {
const stats = getQueueStats();
if (stats.size > 5) {
logger.warn('Upload queue building up', { pending: stats.pending, size: stats.size });
}
});
uploadQueue.on('next', () => {
const stats = getQueueStats();
logger.debug('Processing next upload', { pending: stats.pending, size: stats.size });
});
/**
* Enqueue an upload task to be executed by the queue.
*
* Tasks are executed in FIFO order, subject to the concurrency limit.
*
* @param task - An async function representing the upload operation.
* @returns A promise that resolves with the task's result.
*/
export const enqueueUpload = <T>(task: () => Promise<T>): Promise<T> => {
return uploadQueue.add(task);
};
/**
* Get current queue statistics.
*
* @returns An object with `pending` (actively executing) and `size` (waiting) counts.
*/
export const getQueueStats = (): { pending: number; size: number } => ({
pending: uploadQueue.pending,
size: uploadQueue.size,
});
/**
* Get the number of items waiting in the queue (not yet started).
*
* @returns The number of queued items.
*/
export const getQueueSize = (): number => uploadQueue.size;
/**
* Get the number of items currently being processed.
*
* @returns The number of pending (in-flight) items.
*/
export const getPendingCount = (): number => uploadQueue.pending;
/**
* Clear all pending items and wait for in-flight ones to finish.
*
* @returns A promise that resolves when the queue is idle after clearing.
*/
export const clearQueue = async (): Promise<void> => {
uploadQueue.clear();
await uploadQueue.onIdle();
};
/**
* Wait for the queue to become idle (all tasks finished).
*
* @returns A promise that resolves when no tasks are pending or in-flight.
*/
export const waitForQueue = async (): Promise<void> => {
await uploadQueue.onIdle();
};