feat: implement deduplication in HTTP upload router
- Compute SHA-256 hash using computeHash for multipart and JSON uploads - Query database for existing file records with the same hash - Return existing metadata and download URL immediately on duplicate match without reforwarding to Telegram - Include fileHash in newly inserted file database records Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
4e805689c1
commit
04a2e52250
+64
-1
@@ -1,7 +1,8 @@
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { db, files as fileSchema } from '../db';
|
||||
import { config } from '../env';
|
||||
import { checkFileSize, ensureExtension, extractMimeType, getFileType } from '../utils/file';
|
||||
import { checkFileSize, computeHash, ensureExtension, extractMimeType, getFileType } from '../utils/file';
|
||||
import logger from '../utils/logger';
|
||||
import { forwardToStorage, getBot } from '../utils/telegram';
|
||||
|
||||
@@ -38,6 +39,36 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
|
||||
|
||||
const fileBytes = await file.arrayBuffer();
|
||||
const fileBuffer = Buffer.from(fileBytes);
|
||||
const hash = computeHash(fileBuffer);
|
||||
|
||||
// Check for duplicate in DB
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(fileSchema)
|
||||
.where(eq(fileSchema.fileHash, hash))
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) {
|
||||
const existingFile = existing[0];
|
||||
const responsePayload = {
|
||||
public_id: existingFile.publicId,
|
||||
telegram_file_id: existingFile.telegramFileId,
|
||||
telegram_file_unique_id: existingFile.telegramFileUniqueId,
|
||||
storage_chat_id: existingFile.storageChatId,
|
||||
storage_message_id: existingFile.storageMessageId,
|
||||
file_name: existingFile.fileName,
|
||||
mime_type: existingFile.mimeType,
|
||||
size_bytes: existingFile.sizeBytes,
|
||||
file_type: existingFile.fileType,
|
||||
uploader_id: existingFile.uploaderId,
|
||||
created_at: existingFile.createdAt instanceof Date
|
||||
? existingFile.createdAt.toISOString()
|
||||
: new Date(existingFile.createdAt).toISOString(),
|
||||
download_url: `${config.baseUrl}/f/${existingFile.publicId}`,
|
||||
};
|
||||
return Response.json(responsePayload, { status: 200 });
|
||||
}
|
||||
|
||||
const rawMimeType = file.type || extractMimeType({}, req) || 'application/octet-stream';
|
||||
const { fileName: finalFileName, mimeType } = ensureExtension(
|
||||
fileName,
|
||||
@@ -69,6 +100,7 @@ const handleMultipartUpload = async (req: Request): Promise<Response> => {
|
||||
sizeBytes: fileInfo.file_size || fileBuffer.byteLength,
|
||||
fileType: fileType,
|
||||
uploaderId: 0,
|
||||
fileHash: hash,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
@@ -119,6 +151,36 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
|
||||
}
|
||||
|
||||
const fileBytes = Buffer.from(base64Data, 'base64');
|
||||
const hash = computeHash(fileBytes);
|
||||
|
||||
// Check for duplicate in DB
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(fileSchema)
|
||||
.where(eq(fileSchema.fileHash, hash))
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) {
|
||||
const existingFile = existing[0];
|
||||
const responsePayload = {
|
||||
public_id: existingFile.publicId,
|
||||
telegram_file_id: existingFile.telegramFileId,
|
||||
telegram_file_unique_id: existingFile.telegramFileUniqueId,
|
||||
storage_chat_id: existingFile.storageChatId,
|
||||
storage_message_id: existingFile.storageMessageId,
|
||||
file_name: existingFile.fileName,
|
||||
mime_type: existingFile.mimeType,
|
||||
size_bytes: existingFile.sizeBytes,
|
||||
file_type: existingFile.fileType,
|
||||
uploader_id: existingFile.uploaderId,
|
||||
created_at: existingFile.createdAt instanceof Date
|
||||
? existingFile.createdAt.toISOString()
|
||||
: new Date(existingFile.createdAt).toISOString(),
|
||||
download_url: `${config.baseUrl}/f/${existingFile.publicId}`,
|
||||
};
|
||||
return Response.json(responsePayload, { status: 200 });
|
||||
}
|
||||
|
||||
const { fileName: finalFileName, mimeType } = ensureExtension(fileName, fileBytes, rawMimeType);
|
||||
const fileType =
|
||||
getFileType(mimeType, finalFileName) === 'application'
|
||||
@@ -148,6 +210,7 @@ const handleJSONUpload = async (req: Request): Promise<Response> => {
|
||||
sizeBytes: fileInfo.file_size || fileBytes.byteLength,
|
||||
fileType: fileType,
|
||||
uploaderId: 0,
|
||||
fileHash: hash,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
+130
-14
@@ -2,6 +2,19 @@
|
||||
import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';
|
||||
|
||||
// Mock db
|
||||
let mockSelectResult: any[] = [];
|
||||
|
||||
const mockLimit = mock(() => Promise.resolve(mockSelectResult));
|
||||
const mockWhere = mock(() => ({
|
||||
limit: mockLimit,
|
||||
}));
|
||||
const mockFrom = mock(() => ({
|
||||
where: mockWhere,
|
||||
}));
|
||||
const mockSelect = mock(() => ({
|
||||
from: mockFrom,
|
||||
}));
|
||||
|
||||
const mockInsert = mock(() => ({
|
||||
values: mock(() => Promise.resolve()),
|
||||
}));
|
||||
@@ -9,6 +22,7 @@ const mockInsert = mock(() => ({
|
||||
mock.module('../src/db/index', () => ({
|
||||
db: {
|
||||
insert: mockInsert,
|
||||
select: mockSelect,
|
||||
},
|
||||
files: {},
|
||||
}));
|
||||
@@ -19,23 +33,27 @@ mock.module('nanoid', () => ({
|
||||
}));
|
||||
|
||||
// Mock telegram utils
|
||||
const mockForwardToStorage = mock(() =>
|
||||
Promise.resolve({
|
||||
telegramFileId: 'tg-file-id-123',
|
||||
telegramFileUniqueId: 'tg-unique-id-abc',
|
||||
storageMessageId: 98765,
|
||||
}),
|
||||
);
|
||||
|
||||
const mockGetFile = mock(() =>
|
||||
Promise.resolve({
|
||||
file_id: 'tg-file-id-123',
|
||||
file_size: 1000,
|
||||
mime_type: 'image/jpeg',
|
||||
}),
|
||||
);
|
||||
|
||||
mock.module('../src/utils/telegram', () => ({
|
||||
forwardToStorage: mock(() =>
|
||||
Promise.resolve({
|
||||
telegramFileId: 'tg-file-id-123',
|
||||
telegramFileUniqueId: 'tg-unique-id-abc',
|
||||
storageMessageId: 98765,
|
||||
}),
|
||||
),
|
||||
forwardToStorage: mockForwardToStorage,
|
||||
getBot: () => ({
|
||||
telegram: {
|
||||
getFile: mock(() =>
|
||||
Promise.resolve({
|
||||
file_id: 'tg-file-id-123',
|
||||
file_size: 1000,
|
||||
mime_type: 'image/jpeg',
|
||||
}),
|
||||
),
|
||||
getFile: mockGetFile,
|
||||
},
|
||||
}),
|
||||
}));
|
||||
@@ -45,6 +63,13 @@ describe('Upload Route Handler', () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
mockInsert.mockClear();
|
||||
mockSelect.mockClear();
|
||||
mockFrom.mockClear();
|
||||
mockWhere.mockClear();
|
||||
mockLimit.mockClear();
|
||||
mockForwardToStorage.mockClear();
|
||||
mockGetFile.mockClear();
|
||||
mockSelectResult = [];
|
||||
const uploadRoute = await import('../src/routes/upload');
|
||||
handleUpload = uploadRoute.handleUpload;
|
||||
});
|
||||
@@ -121,6 +146,97 @@ describe('Upload Route Handler', () => {
|
||||
expect(body.file_name).toBe('test_multi.txt');
|
||||
});
|
||||
|
||||
it('should deduplicate multipart upload if hash exists', async () => {
|
||||
mockSelectResult = [
|
||||
{
|
||||
publicId: 'existing-id-123',
|
||||
telegramFileId: 'existing-tg-id',
|
||||
telegramFileUniqueId: 'existing-tg-unique',
|
||||
storageChatId: 12345,
|
||||
storageMessageId: 67890,
|
||||
fileName: 'existing_name.txt',
|
||||
mimeType: 'text/plain',
|
||||
sizeBytes: 100,
|
||||
fileType: 'document',
|
||||
uploaderId: 0,
|
||||
createdAt: new Date('2026-05-18T00:00:00.000Z'),
|
||||
updatedAt: new Date('2026-05-18T00:00:00.000Z'),
|
||||
},
|
||||
];
|
||||
|
||||
const formData = new FormData();
|
||||
const fileBlob = new Blob([Buffer.from('multipart hello')], { type: 'text/plain' });
|
||||
formData.append('file', fileBlob, 'test_multi.txt');
|
||||
|
||||
const req = new Request('http://localhost:3000/api/upload', {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
|
||||
const res = await handleUpload(req);
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.json();
|
||||
|
||||
expect(body.public_id).toBe('existing-id-123');
|
||||
expect(body.telegram_file_id).toBe('existing-tg-id');
|
||||
expect(body.telegram_file_unique_id).toBe('existing-tg-unique');
|
||||
expect(body.file_name).toBe('existing_name.txt');
|
||||
expect(body.download_url).toContain('/f/existing-id-123');
|
||||
|
||||
// DB query happened
|
||||
expect(mockSelect).toHaveBeenCalled();
|
||||
// No telegram upload happened
|
||||
expect(mockForwardToStorage).not.toHaveBeenCalled();
|
||||
// No db insertion happened
|
||||
expect(mockInsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should deduplicate JSON upload if hash exists', async () => {
|
||||
mockSelectResult = [
|
||||
{
|
||||
publicId: 'existing-json-id',
|
||||
telegramFileId: 'existing-tg-json-id',
|
||||
telegramFileUniqueId: 'existing-tg-json-unique',
|
||||
storageChatId: 12345,
|
||||
storageMessageId: 67890,
|
||||
fileName: 'existing_json.txt',
|
||||
mimeType: 'text/plain',
|
||||
sizeBytes: 200,
|
||||
fileType: 'document',
|
||||
uploaderId: 0,
|
||||
createdAt: new Date('2026-05-18T00:00:00.000Z'),
|
||||
updatedAt: new Date('2026-05-18T00:00:00.000Z'),
|
||||
},
|
||||
];
|
||||
|
||||
const req = new Request('http://localhost:3000/api/upload', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
file: Buffer.from('hello world').toString('base64'),
|
||||
fileName: 'test.txt',
|
||||
}),
|
||||
});
|
||||
|
||||
const res = await handleUpload(req);
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.json();
|
||||
|
||||
expect(body.public_id).toBe('existing-json-id');
|
||||
expect(body.telegram_file_id).toBe('existing-tg-json-id');
|
||||
expect(body.file_name).toBe('existing_json.txt');
|
||||
expect(body.download_url).toContain('/f/existing-json-id');
|
||||
|
||||
// DB query happened
|
||||
expect(mockSelect).toHaveBeenCalled();
|
||||
// No telegram upload happened
|
||||
expect(mockForwardToStorage).not.toHaveBeenCalled();
|
||||
// No db insertion happened
|
||||
expect(mockInsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
mock.restore();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user