feat: implement deduplication in HTTP upload router

- Compute SHA-256 hash using computeHash for multipart and JSON uploads
- Query database for existing file records with the same hash
- Return existing metadata and download URL immediately on duplicate match without reforwarding to Telegram
- Include fileHash in newly inserted file database records

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-05-18 19:39:55 +07:00
co-authored by Claude Opus 4.7
parent 4e805689c1
commit 04a2e52250
2 changed files with 194 additions and 15 deletions
+64 -1
View File
@@ -1,7 +1,8 @@
import { eq } from 'drizzle-orm';
import { nanoid } from 'nanoid';
import { db, files as fileSchema } from '../db';
import { config } from '../env';
import { checkFileSize, ensureExtension, extractMimeType, getFileType } from '../utils/file';
import { checkFileSize, computeHash, ensureExtension, extractMimeType, getFileType } from '../utils/file';
import logger from '../utils/logger';
import { forwardToStorage, getBot } from '../utils/telegram';
@@ -38,6 +39,36 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
const fileBytes = await file.arrayBuffer();
const fileBuffer = Buffer.from(fileBytes);
const hash = computeHash(fileBuffer);
// Check for duplicate in DB
const existing = await db
.select()
.from(fileSchema)
.where(eq(fileSchema.fileHash, hash))
.limit(1);
if (existing.length > 0) {
const existingFile = existing[0];
const responsePayload = {
public_id: existingFile.publicId,
telegram_file_id: existingFile.telegramFileId,
telegram_file_unique_id: existingFile.telegramFileUniqueId,
storage_chat_id: existingFile.storageChatId,
storage_message_id: existingFile.storageMessageId,
file_name: existingFile.fileName,
mime_type: existingFile.mimeType,
size_bytes: existingFile.sizeBytes,
file_type: existingFile.fileType,
uploader_id: existingFile.uploaderId,
created_at: existingFile.createdAt instanceof Date
? existingFile.createdAt.toISOString()
: new Date(existingFile.createdAt).toISOString(),
download_url: `${config.baseUrl}/f/${existingFile.publicId}`,
};
return Response.json(responsePayload, { status: 200 });
}
const rawMimeType = file.type || extractMimeType({}, req) || 'application/octet-stream';
const { fileName: finalFileName, mimeType } = ensureExtension(
fileName,
@@ -69,6 +100,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
sizeBytes: fileInfo.file_size || fileBuffer.byteLength,
fileType: fileType,
uploaderId: 0,
fileHash: hash,
createdAt: new Date(),
updatedAt: new Date(),
};
@@ -119,6 +151,36 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
}
const fileBytes = Buffer.from(base64Data, 'base64');
const hash = computeHash(fileBytes);
// Check for duplicate in DB
const existing = await db
.select()
.from(fileSchema)
.where(eq(fileSchema.fileHash, hash))
.limit(1);
if (existing.length > 0) {
const existingFile = existing[0];
const responsePayload = {
public_id: existingFile.publicId,
telegram_file_id: existingFile.telegramFileId,
telegram_file_unique_id: existingFile.telegramFileUniqueId,
storage_chat_id: existingFile.storageChatId,
storage_message_id: existingFile.storageMessageId,
file_name: existingFile.fileName,
mime_type: existingFile.mimeType,
size_bytes: existingFile.sizeBytes,
file_type: existingFile.fileType,
uploader_id: existingFile.uploaderId,
created_at: existingFile.createdAt instanceof Date
? existingFile.createdAt.toISOString()
: new Date(existingFile.createdAt).toISOString(),
download_url: `${config.baseUrl}/f/${existingFile.publicId}`,
};
return Response.json(responsePayload, { status: 200 });
}
const { fileName: finalFileName, mimeType } = ensureExtension(fileName, fileBytes, rawMimeType);
const fileType =
getFileType(mimeType, finalFileName) === 'application'
@@ -148,6 +210,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
sizeBytes: fileInfo.file_size || fileBytes.byteLength,
fileType: fileType,
uploaderId: 0,
fileHash: hash,
createdAt: new Date(),
updatedAt: new Date(),
};