refactor: merge BOT_TOKEN + ADDITIONAL_BOT_TOKENS into single BOT_TOKENS env + speed audit
Deploy FileDrop / deploy (push) Failing after 19s
Deploy FileDrop / deploy (push) Failing after 19s
BOT_TOKENS env:
- Single BOT_TOKENS env var (comma-separated) replaces BOT_TOKEN + ADDITIONAL_BOT_TOKENS
- Backward compat: falls back to BOT_TOKEN + ADDITIONAL_BOT_TOKENS if BOT_TOKENS unset
- Config exposes botTokens: string[] instead of botToken + additionalBotTokens
- Updated env.ts, bot-pool.ts, docker-compose.yml, .env.example, CLAUDE.md, all tests
Speed audit (S3 -> Telegram upload flow):
- Hoisted 5 dynamic await import('../../../db/index') to top-level static imports
in s3-controller.ts (3x) and web-api-controller.ts (2x)
-> saves module resolution + async overhead on every upload
- Removed stale UPLOAD_CONCURRENCY env from docker-compose.yml
(already removed from env.ts in prior refactor)
Upload flow is already concurrent:
- streamBodyToTemp() uses Bun.file(path).writer() — O(1) memory, safe for multi-GB blobs
- utils/chunked-storage.ts reads chunks serially but uploads concurrently with
inFlight backpressure at effectiveConcurrency * 2 (= 16 with 8 bots)
- bot-pool.ts: per-bot PQueue(concurrency=1), 8 bots = 8 concurrent uploads per file,
selectBot() picks least-loaded, 429 detection + inner+outer retry loops
- TELEGRAM_API_TIMEOUT_MS=120s — ample for 48MB chunks
- Infrastructure chunked-storage.ts (DI-based, dead code) has serial upload trap —
noted for future cleanup
This commit is contained in:
+18
-7
@@ -1,8 +1,8 @@
|
||||
import logger from './utils/logger';
|
||||
|
||||
interface AppConfig {
|
||||
botToken: string;
|
||||
additionalBotTokens: string[];
|
||||
/** All bot tokens merged from BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS fallback) */
|
||||
botTokens: string[];
|
||||
storageChatId: number;
|
||||
baseUrl: string;
|
||||
databaseUrl: string;
|
||||
@@ -28,8 +28,21 @@ interface AppConfig {
|
||||
s3VhostDomains: string[];
|
||||
}
|
||||
|
||||
// Validate bot tokens: BOT_TOKENS (new) or fallback to BOT_TOKEN + ADDITIONAL_BOT_TOKENS
|
||||
const botTokensRaw =
|
||||
process.env.BOT_TOKENS ||
|
||||
[process.env.BOT_TOKEN, process.env.ADDITIONAL_BOT_TOKENS].filter(Boolean).join(',');
|
||||
|
||||
if (!botTokensRaw) {
|
||||
logger.error(
|
||||
'Missing required environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)',
|
||||
);
|
||||
throw new Error(
|
||||
'Missing environment variables: BOT_TOKENS (or BOT_TOKEN + ADDITIONAL_BOT_TOKENS)',
|
||||
);
|
||||
}
|
||||
|
||||
const requiredEnv = {
|
||||
BOT_TOKEN: process.env.BOT_TOKEN,
|
||||
STORAGE_CHANNEL_ID: process.env.STORAGE_CHANNEL_ID,
|
||||
BASE_URL: process.env.BASE_URL,
|
||||
DATABASE_URL: process.env.DATABASE_URL,
|
||||
@@ -93,8 +106,7 @@ const maskDatabaseUrl = (value: string): string =>
|
||||
value.replace(/:\/\/([^:]+):([^@]+)@/, '://$1:***@');
|
||||
|
||||
export const config: AppConfig = {
|
||||
botToken: process.env.BOT_TOKEN!,
|
||||
additionalBotTokens: parseTokens(process.env.ADDITIONAL_BOT_TOKENS),
|
||||
botTokens: parseTokens(botTokensRaw),
|
||||
storageChatId: parseInt(process.env.STORAGE_CHANNEL_ID!, 10),
|
||||
baseUrl: process.env.BASE_URL!,
|
||||
databaseUrl: process.env.DATABASE_URL!,
|
||||
@@ -126,8 +138,7 @@ export const config: AppConfig = {
|
||||
logger.info('Environment variables loaded', {
|
||||
config: {
|
||||
...config,
|
||||
botToken: maskSecret(config.botToken),
|
||||
additionalBotTokens: config.additionalBotTokens.map(maskSecret),
|
||||
botTokens: config.botTokens.map(maskSecret),
|
||||
databaseUrl: maskDatabaseUrl(config.databaseUrl),
|
||||
adminApiToken: maskSecret(config.adminApiToken),
|
||||
adminApiTokenEnabled: config.adminApiToken.length > 0,
|
||||
|
||||
@@ -64,7 +64,7 @@ export class BotPool implements ITelegramService {
|
||||
private readonly bots: BotEntry[] = [];
|
||||
|
||||
constructor() {
|
||||
const tokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens]));
|
||||
const tokens = Array.from(new Set(config.botTokens));
|
||||
this.bots = tokens.map((token, index) => ({
|
||||
index,
|
||||
token,
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
listObjectsByPrefix,
|
||||
softDeleteFile,
|
||||
} from '../../../db/files-ext';
|
||||
import { db, files as fileSchema } from '../../../db/index';
|
||||
import {
|
||||
abortMultipartUpload,
|
||||
completeMultipartUpload,
|
||||
@@ -1082,7 +1083,6 @@ const storeFileFromTemp = async (
|
||||
fileStream.destroy();
|
||||
|
||||
const publicId = nanoid();
|
||||
const { db, files: fileSchema } = await import('../../../db/index');
|
||||
|
||||
await db.insert(fileSchema).values({
|
||||
publicId,
|
||||
@@ -1194,7 +1194,6 @@ const handleCopyObject = async (
|
||||
}
|
||||
|
||||
const publicId = nanoid();
|
||||
const { db, files: fileSchema } = await import('../../../db/index');
|
||||
|
||||
await db.insert(fileSchema).values({
|
||||
publicId,
|
||||
@@ -1679,7 +1678,6 @@ const handleCompleteMultipartUpload = async (
|
||||
const combinedEtag = storedParts.map((p) => p.etag).join('-');
|
||||
|
||||
const publicId = nanoid();
|
||||
const { db, files: fileSchema } = await import('../../../db/index');
|
||||
|
||||
// M7: Use stored content-type from the multipart record if available
|
||||
const mimeType = multipart.contentType || 'application/octet-stream';
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
listObjectsByPrefix,
|
||||
softDeleteFile,
|
||||
} from '../../../db/files-ext';
|
||||
import { db, files as fileSchema } from '../../../db/index';
|
||||
import { botPool } from '../../../infrastructure/telegram/bot-pool';
|
||||
import logger from '../../../shared/logger/index';
|
||||
import { cleanupTempFile, ensureExtension, getErrorMessage } from '../../../shared/utils/file';
|
||||
@@ -246,7 +247,6 @@ export const handleUploadObjectV1 = async (
|
||||
);
|
||||
|
||||
const publicId = nanoid();
|
||||
const { db, files: fileSchema } = await import('../../../db/index');
|
||||
|
||||
await db.insert(fileSchema).values({
|
||||
publicId,
|
||||
@@ -361,7 +361,6 @@ export const handleCopyObjectV1 = async (req: Request, params: RouteParams): Pro
|
||||
}
|
||||
|
||||
const publicId = nanoid();
|
||||
const { db, files: fileSchema } = await import('../../../db/index');
|
||||
|
||||
await db.insert(fileSchema).values({
|
||||
publicId,
|
||||
|
||||
Reference in New Issue
Block a user