MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(logging): add error serialization and log metadata formatting

Browse Source 
- Introduced `loggerSerialization.ts` to handle error serialization and log metadata formatting.
- Added `serializeError` function to convert Error objects into a structured format.
- Implemented `serializeLogValue` to handle various data types including Errors, Dates, RegExps, and plain objects.
- Created `formatLogMetadata` to format log metadata using the serialization functions.

feat(pagination): implement cursor encoding and decoding

- Added `pagination.ts` to manage cursor-based pagination.
- Implemented `encodeCursor` to convert cursor data into a base64 string.
- Developed `decodeCursor` to parse base64 strings back into cursor data, with error handling for invalid inputs.
This commit is contained in:
MythEclipse
2026-05-21 00:34:30 +07:00
parent 31e8de3185
commit 480bcff5e2
9 changed files with 234 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/logger.ts
+1 -90
View File
@@ -1,6 +1,7 @@
import fs from "node:fs";
import path from "node:path";
import winston from "winston";
import { formatLogMetadata, serializeLogValue } from "./loggerSerialization";
const isDev = process.env.NODE_ENV !== "production";
const logLevel = process.env.LOG_LEVEL || (isDev ? "debug" : "info");
@@ -8,96 +9,6 @@ const logsDir = path.resolve(process.cwd(), "logs");
fs.mkdirSync(logsDir, { recursive: true });
type LogMetadata = Record<string, unknown>;
type SerializedError = {
name: string;
message: string;
stack?: string;
code?: unknown;
statusCode?: unknown;
} & Record<string, unknown>;
const serializeError = (error: Error): SerializedError => {
const serialized: SerializedError = {
name: error.name,
message: error.message,
};
if (error.stack) {
serialized.stack = error.stack;
}
const errorWithFields = error as Error & {
code?: unknown;
statusCode?: unknown;
[key: string]: unknown;
};
if (errorWithFields.code !== undefined) {
serialized.code = errorWithFields.code;
}
if (errorWithFields.statusCode !== undefined) {
serialized.statusCode = errorWithFields.statusCode;
}
for (const [key, value] of Object.entries(errorWithFields)) {
if (serialized[key] === undefined) {
serialized[key] = value;
}
}
return serialized;
};
const isPlainObject = (value: unknown): value is Record<string, unknown> => {
if (!value || typeof value !== "object") {
return false;
}
const prototype = Object.getPrototypeOf(value);
return prototype === Object.prototype || prototype === null;
};
const serializeLogValue = (value: unknown): unknown => {
if (value instanceof Error) {
return serializeError(value);
}
if (value instanceof Date) {
return value.toISOString();
}
if (value instanceof RegExp) {
return value.toString();
}
if (Array.isArray(value)) {
return value.map(serializeLogValue);
}
if (isPlainObject(value)) {
return Object.fromEntries(
Object.entries(value).map(([key, nestedValue]) => [
key,
serializeLogValue(nestedValue),
]),
);
}
return value;
};
const formatLogMetadata = (metadata: LogMetadata): LogMetadata => {
return Object.fromEntries(
Object.entries(metadata).map(([key, value]) => [
key,
serializeLogValue(value),
]),
);
};
const metadataFormat = winston.format((info) => {
const {
level: _level,
src/loggerSerialization.ts
+89
View File
@@ -0,0 +1,89 @@
export type LogMetadata = Record<string, unknown>;
type SerializedError = {
name: string;
message: string;
stack?: string;
code?: unknown;
statusCode?: unknown;
} & Record<string, unknown>;
const serializeError = (error: Error): SerializedError => {
const serialized: SerializedError = {
name: error.name,
message: error.message,
};
if (error.stack) {
serialized.stack = error.stack;
}
const errorWithFields = error as Error & {
code?: unknown;
statusCode?: unknown;
[key: string]: unknown;
};
if (errorWithFields.code !== undefined) {
serialized.code = errorWithFields.code;
}
if (errorWithFields.statusCode !== undefined) {
serialized.statusCode = errorWithFields.statusCode;
}
for (const [key, value] of Object.entries(errorWithFields)) {
if (serialized[key] === undefined) {
serialized[key] = value;
}
}
return serialized;
};
const isPlainObject = (value: unknown): value is Record<string, unknown> => {
if (!value || typeof value !== "object") {
return false;
}
const prototype = Object.getPrototypeOf(value);
return prototype === Object.prototype || prototype === null;
};
export const serializeLogValue = (value: unknown): unknown => {
if (value instanceof Error) {
return serializeError(value);
}
if (value instanceof Date) {
return value.toISOString();
}
if (value instanceof RegExp) {
return value.toString();
}
if (Array.isArray(value)) {
return value.map(serializeLogValue);
}
if (isPlainObject(value)) {
return Object.fromEntries(
Object.entries(value).map(([key, nestedValue]) => [
key,
serializeLogValue(nestedValue),
]),
);
}
return value;
};
export const formatLogMetadata = (metadata: LogMetadata): LogMetadata => {
return Object.fromEntries(
Object.entries(metadata).map(([key, value]) => [
key,
serializeLogValue(value),
]),
);
};
src/moderation/attachmentUploader.ts
+25 -23
View File
@@ -1,6 +1,5 @@
import { config } from "../config";
import { createChildLogger } from "../logger";
import type { SqliteDatabase } from "../muxer-queue";
import { retryWithBackoff } from "../retry";
import {
updateAttachmentAsFailedUpload,
@@ -9,6 +8,17 @@ import {
const logger = createChildLogger("attachment-uploader");
const ATTACHMENT_UPLOAD_RETRY_OPTIONS = {
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
minTimeout: 1000,
maxTimeout: 5000,
logger,
} as const;
function toErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
export interface PicserUploadResponse {
success: boolean;
filename: string;
@@ -60,34 +70,26 @@ export async function uploadAttachmentToPicser(
formData.append("file", blob, filename);
try {
const response = await retryWithBackoff(
async () => {
const res = await fetch(config.PICSER_UPLOAD_URL, {
method: "POST",
body: formData,
signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS),
});
const response = await retryWithBackoff(async () => {
const res = await fetch(config.PICSER_UPLOAD_URL, {
method: "POST",
body: formData,
signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS),
});
if (!res.ok) {
throw new Error(`Upload failed with status ${res.status}`);
}
if (!res.ok) {
throw new Error(`Upload failed with status ${res.status}`);
}
return res.json() as Promise<PicserUploadResponse>;
},
{
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
minTimeout: 1000,
maxTimeout: 5000,
logger,
},
);
return res.json() as Promise<PicserUploadResponse>;
}, ATTACHMENT_UPLOAD_RETRY_OPTIONS);
return parseUploadResponse(response);
} catch (error) {
logger.error(
{
filename,
error: error instanceof Error ? error.message : String(error),
error: toErrorMessage(error),
},
"Failed to upload attachment",
);
@@ -109,7 +111,7 @@ export async function downloadDiscordAttachment(url: string): Promise<Buffer> {
return Buffer.from(buffer);
} catch (error) {
logger.error(
{ url, error: error instanceof Error ? error.message : String(error) },
{ url, error: toErrorMessage(error) },
"Failed to download Discord attachment",
);
throw error;
@@ -135,7 +137,7 @@ export async function processAttachmentUpload(
await updateAttachmentAsUploaded(attachmentId, result.url, Date.now());
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
const errorMsg = toErrorMessage(error);
await updateAttachmentAsFailedUpload(attachmentId, errorMsg);
logger.error({ attachmentId, error: errorMsg }, "Attachment upload failed");
}
src/moderation/messageCapture.ts
+49 -32
View File
@@ -1,6 +1,7 @@
import type { Client, Message } from "discord.js-selfbot-v13";
import { config } from "../config";
import { createChildLogger } from "../logger";
import { getModerationBroadcaster } from "../ws/broadcastGlobals";
import { queueMessageAnalysis } from "./aiAnalyzer";
import { processAttachmentUpload } from "./attachmentUploader";
import {
@@ -15,22 +16,10 @@ import {
updateMessageAsEdited,
upsertMessageForCapture,
} from "./messageStore";
import type {
AttachmentRecord,
MessageRecord,
ModerationBroadcaster,
} from "./types";
import type { AttachmentRecord, MessageRecord } from "./types";
const logger = createChildLogger("message-capture");
type ModerationGlobal = typeof globalThis & {
moderationBroadcaster?: ModerationBroadcaster;
};
function getModerationBroadcaster(): ModerationBroadcaster | undefined {
return (globalThis as ModerationGlobal).moderationBroadcaster;
}
export interface TextCaptureTarget {
guildId?: string;
channelId?: string;
@@ -57,15 +46,14 @@ function getTextCaptureTarget(): TextCaptureTarget {
};
}
export async function captureMessage(
function buildMessageRecord(
message: Message,
type: "text" | "edited" | "deleted",
options: { source?: "live" | "backlog" } = {},
): Promise<void> {
): MessageRecord {
const location = getMessageLocation(message);
const metadata = getMessageMetadata(message);
const messageRecord: MessageRecord = {
return {
id: message.id,
guild_id: message.guildId!,
channel_id: location.channelId,
@@ -81,6 +69,45 @@ export async function captureMessage(
type,
metadata: JSON.stringify(metadata),
};
}
function buildAttachmentRecord(
message: Message,
location: ReturnType<typeof getMessageLocation>,
attachment: {
id: string;
name: string | null;
size: number;
contentType: string | null;
url: string;
},
): AttachmentRecord {
return {
id: attachment.id,
message_id: message.id,
guild_id: message.guildId!,
channel_id: location.channelId,
thread_id: location.threadId,
user_id: message.author?.id,
filename: attachment.name || "unknown",
size: attachment.size,
type: attachment.contentType || "application/octet-stream",
discord_url: attachment.url,
uploaded_url: null,
upload_status: "pending",
upload_error: null,
created_at: Date.now(),
uploaded_at: null,
};
}
export async function captureMessage(
message: Message,
type: "text" | "edited" | "deleted",
options: { source?: "live" | "backlog" } = {},
): Promise<void> {
const location = getMessageLocation(message);
const messageRecord = buildMessageRecord(message, type);
const inserted = await upsertMessageForCapture(messageRecord);
if (!inserted) {
@@ -97,23 +124,13 @@ export async function captureMessage(
// Insert attachments before queuing analysis to avoid race condition
if (message.attachments.size > 0) {
for (const [, attachment] of message.attachments) {
const attachmentRecord: AttachmentRecord = {
const attachmentRecord = buildAttachmentRecord(message, location, {
id: attachment.id,
message_id: message.id,
guild_id: message.guildId!,
channel_id: location.channelId,
thread_id: location.threadId,
user_id: message.author?.id,
filename: attachment.name || "unknown",
name: attachment.name,
size: attachment.size,
type: attachment.contentType || "application/octet-stream",
discord_url: attachment.url,
uploaded_url: null,
upload_status: "pending",
upload_error: null,
created_at: Date.now(),
uploaded_at: null,
};
contentType: attachment.contentType,
url: attachment.url,
});
await insertAttachment(attachmentRecord);
src/moderation/messageStore.ts
+2 -22
View File
@@ -12,6 +12,7 @@ import {
import { getDatabase } from "../database/drizzle.ts";
import { attachmentsTable, messagesTable } from "../database/schema.ts";
import { createChildLogger } from "../logger.ts";
import { decodeCursor, encodeCursor } from "./pagination";
import type {
AttachmentRecord,
MessageQuery,
@@ -44,28 +45,7 @@ function db(): MessageDatabase {
return getDatabase() as unknown as MessageDatabase;
}
// Cursor helpers for pagination
interface CursorData {
created_at: number;
id: string;
}
export function encodeCursor(data: CursorData): string {
return Buffer.from(JSON.stringify(data)).toString("base64");
}
export function decodeCursor(cursor?: string): CursorData | null {
if (!cursor) return null;
try {
const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8"));
if (typeof data.created_at === "number" && typeof data.id === "string") {
return data;
}
return null;
} catch {
return null;
}
}
export { decodeCursor, encodeCursor } from "./pagination";
export async function insertMessage(message: MessageRecord): Promise<void> {
try {
src/moderation/pagination.ts
+21
View File
@@ -0,0 +1,21 @@
export interface CursorData {
created_at: number;
id: string;
}
export function encodeCursor(data: CursorData): string {
return Buffer.from(JSON.stringify(data)).toString("base64");
}
export function decodeCursor(cursor?: string): CursorData | null {
if (!cursor) return null;
try {
const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8"));
if (typeof data.created_at === "number" && typeof data.id === "string") {
return data;
}
return null;
} catch {
return null;
}
}
src/ws/broadcastGlobals.ts
+4
View File
@@ -22,6 +22,10 @@ type VoiceGlobals = typeof globalThis & {
) => void;
};
export function getModerationBroadcaster(): ModerationBroadcaster | undefined {
return (globalThis as VoiceGlobals).moderationBroadcaster;
}
export function exposeModerationGlobals(
broadcaster: ModerationBroadcaster,
adminPassword: string,