feat: implement chunked storage for Telegram file uploads and retrieval

2026-07-07 08:08:22 +07:00
parent fbfff3d9ec
commit 144ebe6dd3
18 changed files with 750 additions and 11 deletions
@@ -2,6 +2,7 @@ import { createReadStream } from 'node:fs';
 import { nanoid } from 'nanoid';
 import { findFileByPublicId } from '../db/files';
 import { fileInfoCache } from '../utils/cache';
+import { createChunkedObjectResponse } from '../utils/chunked-storage';
 import { cleanupTempFile, formatCreatedAt, getErrorMessage } from '../utils/file';
 import logger from '../utils/logger';
 import { metricsCollector } from '../utils/metrics';
@@ -52,6 +53,14 @@ export const handleFileRedirect = async (req: RequestWithParams): Promise<Respon
      return fail(404, 'File not found');
    }

+    if (file.storageBackend === 'chunked') {
+      if (file.archiveEntryName) {
+        return fail(501, 'Archive entry extraction is not supported for chunked files');
+      }
+      const range = { type: 'none' as const };
+      return createChunkedObjectResponse({ file, range, reqId: '' });
+    }
+
    const archiveEntryName = file.archiveEntryName;
    if (archiveEntryName) {
      const archiveFileId = file.archiveTelegramFileId || file.telegramFileId;
@@ -18,6 +18,7 @@ import {
 } from '../db/multipart';
 import type { File } from '../db/schema';
 import { config } from '../env';
+import { createChunkedObjectResponse, storeFileInTelegramChunks } from '../utils/chunked-storage';
 import { cleanupTempFile, computeHash, ensureExtension, getErrorMessage } from '../utils/file';
 import logger from '../utils/logger';
 import { verifyPresignedUrl, verifySignature } from '../utils/s3/auth';
@@ -319,6 +320,35 @@ const handleGetObject = async (
      reqId,
    );

+  if (file.storageBackend === 'chunked') {
+    const totalSize = Number(file.sizeBytes);
+    const range = parseRangeHeader(headers.range || null, totalSize);
+    if (range.type === 'invalid') {
+      return s3ErrorResponse(
+        'InvalidRange',
+        'The requested range is not satisfiable.',
+        `/${bucket}/${key}`,
+        416,
+        reqId,
+        {
+          'content-range': unsatisfiedContentRange(totalSize),
+        },
+      );
+    }
+    try {
+      return await createChunkedObjectResponse({ file, range, reqId });
+    } catch (error) {
+      logger.warn('Chunked object content fetch failed', { key, error: getErrorMessage(error) });
+      return s3ErrorResponse(
+        'InternalError',
+        'Failed to fetch object content from storage',
+        `/${bucket}/${key}`,
+        502,
+        reqId,
+      );
+    }
+  }
+
  if (file.multipartUploadId) {
    return handleGetMultipartObject(file, bucket, key, headers, reqId);
  }
@@ -545,9 +575,28 @@ const storeFileToTelegram = async (
    contentType,
  );

+  const bucketId = bucketRecord.id;
+  const partFileNamePrefix = `s3-${bucketRecord.name}-${key.replace(/\//g, '_')}`;
+
+  if (buffer.byteLength > config.telegramChunkSizeBytes) {
+    const file = await storeFileInTelegramChunks({
+      tempPath,
+      partFileNamePrefix,
+      fileName: finalFileName,
+      mimeType,
+      sizeBytes: buffer.byteLength,
+      fileType: 'document',
+      uploaderId: 0,
+      bucketId,
+      s3Key: key,
+    });
+    await cleanupTempFile(tempPath);
+    return s3Response(null, 200, reqId, { etag: `"${file.fileHash}"` });
+  }
+
  const forwardResult = await forwardToStorage(
    createReadStream(tempPath),
-    `s3-${bucketRecord.name}-${key.replace(/\//g, '_')}`,
+    partFileNamePrefix,
    'document',
  );

@@ -566,7 +615,7 @@ const storeFileToTelegram = async (
    fileType: 'document',
    uploaderId: 0,
    fileHash: hash,
-    bucketId: bucketRecord.id,
+    bucketId,
    s3Key: key,
    storageBackend: 'telegram',
    isDeleted: false,
@@ -613,6 +662,17 @@ const handleCopyObject = async (
      reqId,
    );

+  if (sourceFile.storageBackend === 'chunked') {
+    // Copying chunked objects is not yet supported.
+    return s3ErrorResponse(
+      'NotImplemented',
+      'Copying chunked objects is not yet implemented.',
+      copySource,
+      501,
+      reqId,
+    );
+  }
+
  // Conditional copy: if-match / if-none-match checks
  const ifMatch = headers['x-amz-copy-source-if-match'];
  const ifNoneMatch = headers['x-amz-copy-source-if-none-match'];
@@ -883,6 +943,16 @@ const handleUploadPart = async (
  const body = await req.arrayBuffer();
  const buffer = Buffer.from(body);

+  if (buffer.byteLength > config.telegramChunkSizeBytes) {
+    return s3ErrorResponse(
+      'EntityTooLarge',
+      `Your proposed upload part size (${buffer.byteLength} bytes) exceeds the maximum allowed part size (${config.telegramChunkSizeBytes} bytes) for this storage backend. Use smaller part sizes.`,
+      `/${bucket}/${key}`,
+      400,
+      reqId,
+    );
+  }
+
  const tempPath = `/tmp/teleuploader-mp-${nanoid()}`;
  await Bun.write(tempPath, buffer);

@@ -2,6 +2,7 @@ import { createWriteStream } from 'node:fs';
 import { nanoid } from 'nanoid';
 import { findFileByHash } from '../db/files';
 import { config } from '../env';
+import { storeFileInTelegramChunks } from '../utils/chunked-storage';
 import {
  buildUploadResponse,
  checkFileSize,
@@ -200,6 +201,20 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
      return Response.json({ error: `File size exceeds ${fileType} limit` }, { status: 400 });
    }

+    if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
+      const file = await storeFileInTelegramChunks({
+        tempPath: prepared.tempPath,
+        partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'upload'}`,
+        fileName: finalFileName,
+        mimeType,
+        sizeBytes: prepared.sizeBytes,
+        fileType,
+        uploaderId: 0,
+      });
+      await cleanupTempFile(prepared.tempPath);
+      return Response.json(buildUploadResponse(file, config.baseUrl), { status: 200 });
+    }
+
    const uploaded = await enqueuePreparedUpload({
      prepared,
      fileName: finalFileName,
@@ -257,6 +272,21 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
    }

    const prepared = await writeBufferToTemp(fileBytes, hash);
+
+    if (prepared.sizeBytes > config.telegramChunkSizeBytes) {
+      const file = await storeFileInTelegramChunks({
+        tempPath: prepared.tempPath,
+        partFileNamePrefix: `direct-${prepared.fileHash?.slice(0, 16) || 'json'}`,
+        fileName: finalFileName,
+        mimeType,
+        sizeBytes: prepared.sizeBytes,
+        fileType,
+        uploaderId: 0,
+      });
+      await cleanupTempFile(prepared.tempPath);
+      return Response.json(buildUploadResponse(file, config.baseUrl), { status: 200 });
+    }
+
    const uploaded = await enqueuePreparedUpload({
      prepared,
      fileName: finalFileName,
@@ -8,6 +8,7 @@ import {
  softDeleteFile,
 } from '../db/files-ext';
 import { config } from '../env';
+import { createChunkedObjectResponse, storeFileInTelegramChunks } from '../utils/chunked-storage';
 import { cleanupTempFile, computeHash, ensureExtension, getErrorMessage } from '../utils/file';
 import logger from '../utils/logger';
 import { forwardToStorage, getFileInfo } from '../utils/telegram';
@@ -126,9 +127,35 @@ export const handleUploadObjectV1 = async (
    file.type || 'application/octet-stream',
  );

+  const partFileNamePrefix = `s3-${bucket.name}-${key.replace(/\//g, '_')}`;
+
+  if (buffer.byteLength > config.telegramChunkSizeBytes) {
+    const file = await storeFileInTelegramChunks({
+      tempPath,
+      partFileNamePrefix,
+      fileName: finalFileName,
+      mimeType,
+      sizeBytes: buffer.byteLength,
+      fileType: 'document',
+      uploaderId: 0,
+      bucketId: bucket.id,
+      s3Key: key,
+    });
+    await cleanupTempFile(tempPath);
+    return json(
+      {
+        key,
+        size: buffer.byteLength,
+        etag: hash,
+        downloadUrl: `${config.baseUrl}/f/${file.publicId}`,
+      },
+      201,
+    );
+  }
+
  const forwardResult = await forwardToStorage(
    createReadStream(tempPath),
-    `s3-${bucket.name}-${key.replace(/\//g, '_')}`,
+    partFileNamePrefix,
    'document',
  );

@@ -183,6 +210,11 @@ export const handleDownloadObjectV1 = async (
  const file = await findFileByBucketAndKey(bucket.id, params.key!);
  if (!file) return jsonError('Object not found', 404);

+  if (file.storageBackend === 'chunked') {
+    const range = { type: 'none' as const };
+    return createChunkedObjectResponse({ file, range, reqId: '' });
+  }
+
  const fileInfo = await getFileInfo(file.telegramFileId);
  const redirectUrl = `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`;

@@ -209,6 +241,10 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro
  const sourceFile = await findFileByBucketAndKey(sourceBucket.id, body.sourceKey);
  if (!sourceFile) return jsonError('Source object not found', 404);

+  if (sourceFile.storageBackend === 'chunked') {
+    return json({ error: 'Copying chunked objects is not implemented' }, 501);
+  }
+
  const publicId = nanoid();
  const { db, files: fileSchema } = await import('../db/index');