MythEclipse/TeleUploader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: enhance file handling and metrics tracking; remove unused bot health tracker

Browse Source
This commit is contained in:
asepharyana
2026-07-06 01:25:18 +07:00
parent bbf420759b
commit 52bd704d21
17 changed files with 90 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/botHealth.ts
-129
View File
@@ -1,129 +0,0 @@
import logger from './logger';
interface BotHealth {
index: number;
isHealthy: boolean;
rateLimitedUntil: number;
failureCount: number;
successCount: number;
lastUsed: number;
}
class BotHealthTracker {
private botHealth: Map<number, BotHealth> = new Map();
private totalBots: number;
constructor(totalBots: number) {
this.totalBots = totalBots;
for (let i = 0; i < totalBots; i++) {
this.botHealth.set(i, {
index: i,
isHealthy: true,
rateLimitedUntil: 0,
failureCount: 0,
successCount: 0,
lastUsed: 0,
});
}
}
recordSuccess(botIndex: number): void {
const health = this.botHealth.get(botIndex);
if (health) {
health.successCount++;
health.failureCount = 0;
health.isHealthy = true;
health.lastUsed = Date.now();
}
}
recordFailure(botIndex: number, retryAfterSeconds?: number): void {
const health = this.botHealth.get(botIndex);
if (health) {
health.failureCount++;
health.lastUsed = Date.now();
if (retryAfterSeconds) {
health.rateLimitedUntil = Date.now() + retryAfterSeconds * 1000;
health.isHealthy = false;
logger.warn('Bot rate limited', {
botIndex,
retryAfter: retryAfterSeconds,
});
} else if (health.failureCount >= 3) {
health.isHealthy = false;
logger.warn('Bot marked unhealthy', { botIndex, failures: health.failureCount });
}
}
}
getHealthiestBot(): number {
const now = Date.now();
let bestBot = 0;
let bestScore = -Infinity;
for (let i = 0; i < this.totalBots; i++) {
const health = this.botHealth.get(i)!;
// Skip rate-limited bots
if (health.rateLimitedUntil > now) {
continue;
}
// Calculate score: prefer healthy bots with fewer failures and more successes
const score =
(health.isHealthy ? 100 : 0) +
health.successCount -
health.failureCount * 10 -
(now - health.lastUsed) / 1000;
if (score > bestScore) {
bestScore = score;
bestBot = i;
}
}
return bestBot;
}
getStats() {
const stats = {
healthy: 0,
rateLimited: 0,
unhealthy: 0,
bots: [] as any[],
};
const now = Date.now();
for (const health of this.botHealth.values()) {
if (health.rateLimitedUntil > now) {
stats.rateLimited++;
} else if (health.isHealthy) {
stats.healthy++;
} else {
stats.unhealthy++;
}
stats.bots.push({
index: health.index,
healthy: health.isHealthy,
rateLimitedUntil: health.rateLimitedUntil > now ? health.rateLimitedUntil - now : 0,
failures: health.failureCount,
successes: health.successCount,
});
}
return stats;
}
reset(): void {
for (const health of this.botHealth.values()) {
health.isHealthy = true;
health.rateLimitedUntil = 0;
health.failureCount = 0;
health.successCount = 0;
}
}
}
export { BotHealthTracker };
src/utils/cache.ts
-11
View File
@@ -71,15 +71,4 @@ export const fileInfoCache = new Cache<{
bot_token: string;
}>(3600);
// Cleanup expired cache entries every 5 minutes
setInterval(
() => {
const removed = fileInfoCache.cleanup();
if (removed > 0) {
console.log(`Cleaned up ${removed} expired cache entries`);
}
},
5 * 60 * 1000,
);
export { Cache };
src/utils/file.ts
+11
View File
@@ -1,7 +1,18 @@
import { unlink } from 'node:fs/promises';
import logger from './logger';
export const getErrorMessage = (error: unknown): string => {
return error instanceof Error ? error.message : String(error);
};
export const cleanupTempFile = async (tempPath: string): Promise<void> => {
try {
await unlink(tempPath);
} catch (err) {
logger.warn('Failed to cleanup temp file', { tempPath, error: getErrorMessage(err) });
}
};
interface FileMetadata {
publicId: string;
telegramFileId: string;
src/utils/logger.ts
+7 -9
View File
@@ -12,16 +12,14 @@ const logger = winston.createLogger({
// Write all logs including error logs to file
new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
new winston.transports.File({ filename: 'logs/combined.log' }),
// Console transport for docker logs / CLI visibility
new winston.transports.Console({
format:
process.env.NODE_ENV !== 'production'
? winston.format.combine(winston.format.colorize(), winston.format.simple())
: winston.format.json(),
}),
],
});
// If not production, also log to console
if (process.env.NODE_ENV !== 'production') {
logger.add(
new winston.transports.Console({
format: winston.format.combine(winston.format.colorize(), winston.format.simple()),
}),
);
}
export default logger;
src/utils/metrics.ts
+1 -15
View File
@@ -1,4 +1,4 @@
import logger from './logger';
// No imports needed — logger used only by setInterval which moved to index.ts
interface Metric {
name: string;
@@ -108,18 +108,4 @@ class MetricsCollector {
export const metricsCollector = new MetricsCollector();
// Log metrics every 5 minutes
setInterval(
() => {
const snapshot = metricsCollector.getSnapshot();
logger.info('Metrics snapshot', {
uploadLatency: snapshot.uploadLatency,
uploadThroughput: snapshot.uploadThroughput.toFixed(2),
errorRate: snapshot.errorRate.toFixed(2),
cacheHitRate: snapshot.cacheHitRate.toFixed(2),
});
},
5 * 60 * 1000,
);
export { MetricsCollector };
src/utils/telegram.ts
-72
View File
@@ -129,27 +129,6 @@ const buildSendPayload = (fileType: string, fileName: string): SendPayload => {
return basePayload;
};
const getMediaGroupType = (fileType: string): string => {
if (fileType === 'photo') return 'photo';
if (fileType === 'video') return 'video';
if (fileType === 'audio') return 'audio';
return 'document';
};
interface MediaGroupPayloadItem {
type: string;
media: string;
caption: string;
}
const buildMediaGroup = (items: MediaGroupItem[]): MediaGroupPayloadItem[] => {
return items.map((item) => ({
type: getMediaGroupType(item.fileType),
media: item.fileId,
caption: item.fileName,
}));
};
export const forwardToStorage = async (
fileChunk: unknown,
fileName: string,
@@ -184,57 +163,6 @@ export const forwardToStorage = async (
}
};
export interface MediaGroupItem {
fileId: string;
fileName: string;
fileType: string;
}
export const forwardMediaGroupToStorage = async (
items: MediaGroupItem[],
): Promise<{
storageMessageId: number;
telegramFileIds: string[];
telegramFileUniqueIds: string[];
}> => {
try {
const result = await enqueueUpload(async (): Promise<TelegramMessageResult[]> => {
const mediaGroup = buildMediaGroup(items);
return executeWithBotRetry((activeBot) => {
const sendMediaGroup = activeBot.telegram.sendMediaGroup as unknown as (
chatId: number,
media: MediaGroupPayloadItem[],
) => Promise<TelegramMessageResult[]>;
return sendMediaGroup(config.storageChatId, mediaGroup);
});
});
const messages = Array.isArray(result) ? result : [result];
const storageMessageId = messages[0]?.message_id || 0;
const telegramFileIds: string[] = [];
const telegramFileUniqueIds: string[] = [];
for (let i = 0; i < messages.length; i++) {
const uploadedFile = extractUploadedFile(messages[i], items[i]?.fileType || 'document');
telegramFileIds.push(uploadedFile?.file_id || '');
telegramFileUniqueIds.push(uploadedFile?.file_unique_id || '');
}
return {
storageMessageId,
telegramFileIds,
telegramFileUniqueIds,
};
} catch (error: unknown) {
logger.error('Failed to forward media group to storage', {
error: error instanceof Error ? error.message : String(error),
});
throw error;
}
};
export const getFileInfo = async (telegramFileId: string): Promise<TelegramFileInfo> => {
let lastError: unknown;
for (const activeBot of bots) {
src/utils/uploadBatcher.ts
+7 -11
View File
@@ -1,11 +1,9 @@
import { createReadStream } from 'node:fs';
import { unlink } from 'node:fs/promises';
import { nanoid } from 'nanoid';
import { db, files as fileSchema } from '../db';
import type { NewFile } from '../db/schema';
import { config } from '../env';
import { getErrorMessage } from './file';
import logger from './logger';
import { cleanupTempFile } from './file';
import { forwardToStorage } from './telegram';
import { createZip, type ZipEntry } from './zip';
@@ -38,14 +36,6 @@ const BATCH_WINDOW_MS = 2000;
let pendingUploads: PendingUpload[] = [];
let flushTimer: ReturnType<typeof setTimeout> | null = null;
const cleanupTempFile = async (tempPath: string): Promise<void> => {
try {
await unlink(tempPath);
} catch (error) {
logger.warn('Failed to cleanup temp file', { tempPath, error: getErrorMessage(error) });
}
};
const buildUploadedFile = (
item: BatchUploadItem,
entry: ZipEntry,
@@ -124,6 +114,12 @@ const flushUploads = async (): Promise<void> => {
} finally {
await Promise.all(batch.map((item) => cleanupTempFile(item.prepared.tempPath)));
if (zipTempPath) await cleanupTempFile(zipTempPath);
// Reschedule timer if new items arrived during async processing
if (pendingUploads.length > 0 && !flushTimer) {
flushTimer = setTimeout(() => {
void flushUploads();
}, BATCH_WINDOW_MS);
}
}
};