fix: make S3 resilient for Docker registry — no rate limit, retry on transient Telegram errors
Deploy FileDrop / deploy (push) Failing after 15s
Deploy FileDrop / deploy (push) Failing after 15s
- Remove rate limiting from all S3 endpoints (used by Docker registry for concurrent blob pushes — 429 would abort the entire push). - Add retry with exponential backoff in botPool.forwardToStorage for transient Telegram errors (network timeouts, 5xx, socket issues). - Add retry with exponential backoff in botPool.getFileInfo per bot. - Introduce isTransientError() pattern matcher covering ~20 transient error signatures. - Fix temp file leak in handlePutObject when storeFileFromTemp throws. - Fix pre-existing missing botPool namespace on getFileInfo call in handleGetMultipartObject. - Fix route handler return type in PUT handler. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -43,7 +43,7 @@ import {
|
||||
parseDeleteObjectsBody,
|
||||
s3ErrorResponse,
|
||||
} from '../../../utils/s3/xml';
|
||||
import { forwardToStorage, getFileInfo } from '../../../utils/telegram';
|
||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||
|
||||
/**
|
||||
* The default S3 region returned when no region is explicitly configured.
|
||||
@@ -487,7 +487,7 @@ const handleGetObject = async (
|
||||
}
|
||||
|
||||
// Regular Telegram object
|
||||
const fileInfo = await getFileInfo(file.telegramFileId);
|
||||
const fileInfo = await botPool.getFileInfo(file.telegramFileId);
|
||||
const redirectUrl = `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`;
|
||||
|
||||
const totalSize = file.sizeBytes;
|
||||
@@ -592,7 +592,7 @@ const handleGetMultipartObject = async (
|
||||
|
||||
const sources: ObjectPartSource[] = [];
|
||||
for (const part of parts) {
|
||||
const fileInfo = await getFileInfo(part.telegramFileId);
|
||||
const fileInfo = await botPool.getFileInfo(part.telegramFileId);
|
||||
sources.push({
|
||||
telegramFileId: part.telegramFileId,
|
||||
telegramUrl: `https://api.telegram.org/file/bot${fileInfo.bot_token}/${fileInfo.file_path}`,
|
||||
@@ -797,7 +797,12 @@ const handlePutObject = async (
|
||||
return s3Response(null, 200, reqId, { etag: `"${streamed.fileHash}"` });
|
||||
}
|
||||
|
||||
return await storeFileFromTemp(streamed, key, bucketRecord, contentType, reqId);
|
||||
try {
|
||||
return await storeFileFromTemp(streamed, key, bucketRecord, contentType, reqId);
|
||||
} catch (uploadError) {
|
||||
await cleanupTempFile(streamed.tempPath);
|
||||
throw uploadError;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -848,7 +853,7 @@ const storeFileFromTemp = async (
|
||||
return s3Response(null, 200, reqId, { etag: `"${file.fileHash}"` });
|
||||
}
|
||||
|
||||
const forwardResult = await forwardToStorage(
|
||||
const forwardResult = await botPool.forwardToStorage(
|
||||
createReadStream(streamed.tempPath),
|
||||
partFileNamePrefix,
|
||||
'document',
|
||||
@@ -1310,7 +1315,7 @@ const handleUploadPart = async (
|
||||
);
|
||||
}
|
||||
|
||||
const forwardResult = await forwardToStorage(
|
||||
const forwardResult = await botPool.forwardToStorage(
|
||||
createReadStream(tempPath),
|
||||
`mp-${uploadId}-part-${partNumber}`,
|
||||
'document',
|
||||
|
||||
@@ -59,18 +59,18 @@ const _handleMaybeS3Root = (req: Request): Response | Promise<Response> => {
|
||||
};
|
||||
|
||||
/**
|
||||
* Wraps `handleS3Request` with rate limiting.
|
||||
* Dispatches an S3 request directly, bypassing rate limiting.
|
||||
*
|
||||
* S3 API calls (used by Docker registry) are rate-limited per client IP to
|
||||
* prevent resource exhaustion. The default limit (150 req/60s window) allows
|
||||
* concurrent layer pushes while still providing protection.
|
||||
* S3 API calls (used by Docker registry for blob pushes) must not be
|
||||
* rate-limited — large concurrent layer uploads would hit the limit and
|
||||
* fail. The Docker registry client retries on 5xx, not 4xx, so a 429
|
||||
* would abort the entire push.
|
||||
*
|
||||
* @param req - The incoming S3 request.
|
||||
* @returns The S3 response or a 429 Too Many Requests error.
|
||||
* @returns The S3 response.
|
||||
*/
|
||||
const handleS3WithRateLimit = (req: Request): Promise<Response> => {
|
||||
const handler = () => handleS3Request(req, getS3RouteBucket(req));
|
||||
return withRateLimit(handler as (req: Request) => Promise<Response>)(req);
|
||||
const handleS3Direct = (req: Request): Promise<Response> => {
|
||||
return handleS3Request(req, getS3RouteBucket(req));
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -108,7 +108,7 @@ export const routes = {
|
||||
GET: (req: Request): Promise<Response> => {
|
||||
const headers = Object.fromEntries(req.headers);
|
||||
if (shouldHandleS3(req, headers)) {
|
||||
return handleS3WithRateLimit(req);
|
||||
return handleS3Direct(req);
|
||||
}
|
||||
return handleHome();
|
||||
},
|
||||
@@ -118,14 +118,14 @@ export const routes = {
|
||||
}
|
||||
const headers = Object.fromEntries(req.headers);
|
||||
if (shouldHandleS3(req, headers)) {
|
||||
return handleS3WithRateLimit(req);
|
||||
return handleS3Direct(req);
|
||||
}
|
||||
return new Response('Not Allowed', { status: 405 });
|
||||
return Promise.resolve(new Response('Not Allowed', { status: 405 }));
|
||||
},
|
||||
HEAD: handleS3WithRateLimit,
|
||||
DELETE: handleS3WithRateLimit,
|
||||
POST: handleS3WithRateLimit,
|
||||
OPTIONS: handleS3WithRateLimit,
|
||||
HEAD: handleS3Direct,
|
||||
DELETE: handleS3Direct,
|
||||
POST: handleS3Direct,
|
||||
OPTIONS: handleS3Direct,
|
||||
},
|
||||
'/api/v1/auth/login': {
|
||||
POST: withRateLimit(handleLogin),
|
||||
|
||||
Reference in New Issue
Block a user