perf: parallelize S3 object part fetch + cache Telegram file info
Optimize the S3 GET path for chunked/multipart objects and reduce Telegram API round-trips: - object-stream: fetch object parts concurrently (bounded, in-order fan-in) instead of serializing N sequential Telegram CDN fetches. Response latency is now ~the slowest part fetch, not the sum of all part fetches. - bot-pool.getFileInfo: cache file_id -> file_path in the existing in-memory cache so repeated S3 GET/HEAD of the same object skip the Telegram API call (file-controller had its own cache wrapper; the S3 path did not). - chunked-storage + s3-controller: resolve multipart/chunked part CDN URLs concurrently via Promise.all instead of sequentially. - s3-controller multipart: await writer.end() before re-reading the temp part file to avoid a flush race. Adds object-stream-parallel.test.ts covering in-order fan-in, byte ranges, and single-part responses even when the slowest part resolves out of order.
This commit is contained in:
@@ -229,21 +229,27 @@ export class ChunkedStorage {
|
||||
*/
|
||||
async buildChunkedObjectSources(file: FileEntity): Promise<ObjectPartSource[]> {
|
||||
const parts = await this.filePartRepository.listByFileId(file.id);
|
||||
const sources: ObjectPartSource[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
const fileInfo = await this.telegramService.getFileInfo(part.telegramFileId);
|
||||
sources.push({
|
||||
telegramFileId: part.telegramFileId,
|
||||
telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
|
||||
sizeBytes: part.sizeBytes,
|
||||
storedSizeBytes: part.storedSizeBytes,
|
||||
compressionAlgorithm: part.compressionAlgorithm,
|
||||
partNumber: part.partNumber,
|
||||
});
|
||||
}
|
||||
// Resolve every part's Telegram CDN URL concurrently (each is an independent
|
||||
// getFile call) so total resolution time is ~1 round-trip, not N.
|
||||
const partInfos = await Promise.all(
|
||||
parts.map(async (part) => {
|
||||
const fileInfo = await this.telegramService.getFileInfo(part.telegramFileId);
|
||||
return {
|
||||
part,
|
||||
url: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
return sources;
|
||||
return partInfos.map(({ part, url }) => ({
|
||||
telegramFileId: part.telegramFileId,
|
||||
telegramUrl: url,
|
||||
sizeBytes: part.sizeBytes,
|
||||
storedSizeBytes: part.storedSizeBytes,
|
||||
compressionAlgorithm: part.compressionAlgorithm,
|
||||
partNumber: part.partNumber,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user