MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(logging): add error serialization and log metadata formatting

Browse Source 
- Introduced `loggerSerialization.ts` to handle error serialization and log metadata formatting.
- Added `serializeError` function to convert Error objects into a structured format.
- Implemented `serializeLogValue` to handle various data types including Errors, Dates, RegExps, and plain objects.
- Created `formatLogMetadata` to format log metadata using the serialization functions.

feat(pagination): implement cursor encoding and decoding

- Added `pagination.ts` to manage cursor-based pagination.
- Implemented `encodeCursor` to convert cursor data into a base64 string.
- Developed `decodeCursor` to parse base64 strings back into cursor data, with error handling for invalid inputs.
This commit is contained in:
MythEclipse
2026-05-21 00:34:30 +07:00
parent 31e8de3185
commit 480bcff5e2
9 changed files with 234 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -7,3 +7,4 @@ public/app/
.claude/
.env.test
logs/
.codegraph/
pnpm-lock.yaml
Generated
+41 -53
View File
@@ -206,8 +206,8 @@ importers:
specifier: ^2.0.0
version: 2.1.1
otplib:
specifier: ^13.0.0
version: 13.4.0
specifier: ^12.0.1
version: 12.0.1
prism-media:
specifier: ^2.0.0-alpha.0
version: 2.0.0-alpha.0
@@ -1104,10 +1104,6 @@ packages:
resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}
engines: {node: ^14.21.3 || >=16}
'@noble/hashes@2.2.0':
resolution: {integrity: sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==}
engines: {node: '>= 20.19.0'}
'@nodelib/fs.scandir@2.1.5':
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
engines: {node: '>= 8'}
@@ -1180,23 +1176,23 @@ packages:
resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
engines: {node: '>=8.0.0'}
'@otplib/core@13.4.0':
resolution: {integrity: sha512-JqOGcvZQi2wIkEQo8f3/iAjstavpXy6gouIDMHygjNuH6Q0FjbHOiXMdcE94RwfgDNMABhzwUmvaPsxvgm9NYw==}
'@otplib/core@12.0.1':
resolution: {integrity: sha512-4sGntwbA/AC+SbPhbsziRiD+jNDdIzsZ3JUyfZwjtKyc/wufl1pnSIaG4Uqx8ymPagujub0o92kgBnB89cuAMA==}
'@otplib/hotp@13.4.0':
resolution: {integrity: sha512-MJjE0x06mn2ptymz5qZmQveb+vWFuaIftqE0b5/TZZqUOK7l97cV8lRTmid5BpAQMwJDNLW6RnYxGeCRiNdekw==}
'@otplib/plugin-crypto@12.0.1':
resolution: {integrity: sha512-qPuhN3QrT7ZZLcLCyKOSNhuijUi9G5guMRVrxq63r9YNOxxQjPm59gVxLM+7xGnHnM6cimY57tuKsjK7y9LM1g==}
deprecated: Please upgrade to v13 of otplib. Refer to otplib docs for migration paths
'@otplib/plugin-base32-scure@13.4.0':
resolution: {integrity: sha512-/t9YWJmMbB8bF5z8mXrBZc2FXBe8B/3hG5FhWr9K8cFwFhyxScbPysmZe8s1UTzSA6N+s8Uv8aIfCtVXPNjJWw==}
'@otplib/plugin-thirty-two@12.0.1':
resolution: {integrity: sha512-MtT+uqRso909UkbrrYpJ6XFjj9D+x2Py7KjTO9JDPhL0bJUYVu5kFP4TFZW4NFAywrAtFRxOVY261u0qwb93gA==}
deprecated: Please upgrade to v13 of otplib. Refer to otplib docs for migration paths
'@otplib/plugin-crypto-noble@13.4.0':
resolution: {integrity: sha512-KrvE4m7Zv+TT1944HzgqFJWJpKb6AyoxDbvhPStmBqdMlv5Gekb80d66cuFRL08kkPgJ5gXUSb5SFpYeB+bACg==}
'@otplib/preset-default@12.0.1':
resolution: {integrity: sha512-xf1v9oOJRyXfluBhMdpOkr+bsE+Irt+0D5uHtvg6x1eosfmHCsCC6ej/m7FXiWqdo0+ZUI6xSKDhJwc8yfiOPQ==}
deprecated: Please upgrade to v13 of otplib. Refer to otplib docs for migration paths
'@otplib/totp@13.4.0':
resolution: {integrity: sha512-dK+vl0f0ekzf6mCENRI9AKS2NJUC7OjI3+X8e7QSnhQ2WM7I+i4PGpb3QxKi5hxjTtwVuoZwXR2CFtXdcRtNdQ==}
'@otplib/uri@13.4.0':
resolution: {integrity: sha512-x1ozBa5bPbdZCrrTL/HK21qchiK7jYElTu+0ft22abeEhiLYgH1+SIULvOcVk3CK8YwF4kdcidvkq4ciejucJA==}
'@otplib/preset-v11@12.0.1':
resolution: {integrity: sha512-9hSetMI7ECqbFiKICrNa4w70deTUfArtwXykPUvSHWOdzOlfa9ajglu7mNCntlvxycTiOAXkQGwjQCzzDEMRMg==}
'@oxc-project/types@0.130.0':
resolution: {integrity: sha512-ibD2usx9JRu7f5pu2tMKMI4cpA4NgXJQoYRP4pQ7Pxmn1l6k/53qWtQWZayhYy3X4QZkt90Ot+mJEaeXouio6Q==}
@@ -1518,9 +1514,6 @@ packages:
resolution: {integrity: sha512-d9dUmWVA7MMiKobL3VpLF8P2aeanRTu6ypG2OIaEv/ZHH/SUQ2iHOVyi5wAPjQ+HmnMuL0whK9ez8I/raWbtIg==}
engines: {node: '>=v16'}
'@scure/base@2.2.0':
resolution: {integrity: sha512-b8XEupJibegiXV+tDUseI8oLQc8ei3d/4Jkb2RpbHh3MfE054ov3uIz2dhFkB3FI8iwYkEh0gGCApkrYggkPNg==}
'@sec-ant/readable-stream@0.4.1':
resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==}
@@ -3345,8 +3338,8 @@ packages:
resolution: {integrity: sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==}
engines: {node: '>=0.10.0'}
otplib@13.4.0:
resolution: {integrity: sha512-RUcYcRMCgRWhUE/XabRppXpUwCwaWBNHe5iPXhdvP8wwDGpGpsIf/kxX/ec3zFsOaM1Oq8lEhUqDwk6W7DHkwg==}
otplib@12.0.1:
resolution: {integrity: sha512-xDGvUOQjop7RDgxTQ+o4pOol0/3xSZzawTiPKRrHnQWAy0WjhNs/5HdIDJCrqC4MBynmjXgULc6YfioaxZeFgg==}
p-cancelable@2.1.1:
resolution: {integrity: sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==}
@@ -4036,6 +4029,10 @@ packages:
text-table@0.2.0:
resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==}
thirty-two@1.0.2:
resolution: {integrity: sha512-OEI0IWCe+Dw46019YLl6V10Us5bi574EvlJEOcAkB29IzQ/mYD1A6RyNHLjZPiHCmuodxvgF6U+vZO1L15lxVA==}
engines: {node: '>=0.2.6'}
through@2.3.8:
resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}
@@ -5056,8 +5053,6 @@ snapshots:
'@noble/hashes@1.8.0': {}
'@noble/hashes@2.2.0': {}
'@nodelib/fs.scandir@2.1.5':
dependencies:
'@nodelib/fs.stat': 2.0.5
@@ -5144,32 +5139,28 @@ snapshots:
'@opentelemetry/api@1.9.1': {}
'@otplib/core@13.4.0': {}
'@otplib/core@12.0.1': {}
'@otplib/hotp@13.4.0':
'@otplib/plugin-crypto@12.0.1':
dependencies:
'@otplib/core': 13.4.0
'@otplib/uri': 13.4.0
'@otplib/core': 12.0.1
'@otplib/plugin-base32-scure@13.4.0':
'@otplib/plugin-thirty-two@12.0.1':
dependencies:
'@otplib/core': 13.4.0
'@scure/base': 2.2.0
'@otplib/core': 12.0.1
thirty-two: 1.0.2
'@otplib/plugin-crypto-noble@13.4.0':
'@otplib/preset-default@12.0.1':
dependencies:
'@noble/hashes': 2.2.0
'@otplib/core': 13.4.0
'@otplib/core': 12.0.1
'@otplib/plugin-crypto': 12.0.1
'@otplib/plugin-thirty-two': 12.0.1
'@otplib/totp@13.4.0':
'@otplib/preset-v11@12.0.1':
dependencies:
'@otplib/core': 13.4.0
'@otplib/hotp': 13.4.0
'@otplib/uri': 13.4.0
'@otplib/uri@13.4.0':
dependencies:
'@otplib/core': 13.4.0
'@otplib/core': 12.0.1
'@otplib/plugin-crypto': 12.0.1
'@otplib/plugin-thirty-two': 12.0.1
'@oxc-project/types@0.130.0': {}
@@ -5476,8 +5467,6 @@ snapshots:
fast-deep-equal: 3.1.3
lodash: 4.18.1
'@scure/base@2.2.0': {}
'@sec-ant/readable-stream@0.4.1': {}
'@seydx/node-av-darwin-arm64@5.2.4':
@@ -7270,14 +7259,11 @@ snapshots:
os-tmpdir@1.0.2: {}
otplib@13.4.0:
otplib@12.0.1:
dependencies:
'@otplib/core': 13.4.0
'@otplib/hotp': 13.4.0
'@otplib/plugin-base32-scure': 13.4.0
'@otplib/plugin-crypto-noble': 13.4.0
'@otplib/totp': 13.4.0
'@otplib/uri': 13.4.0
'@otplib/core': 12.0.1
'@otplib/preset-default': 12.0.1
'@otplib/preset-v11': 12.0.1
p-cancelable@2.1.1: {}
@@ -8004,6 +7990,8 @@ snapshots:
text-table@0.2.0: {}
thirty-two@1.0.2: {}
through@2.3.8: {}
thunky@1.1.0: {}
src/logger.ts
+1 -90
View File
@@ -1,6 +1,7 @@
import fs from "node:fs";
import path from "node:path";
import winston from "winston";
import { formatLogMetadata, serializeLogValue } from "./loggerSerialization";
const isDev = process.env.NODE_ENV !== "production";
const logLevel = process.env.LOG_LEVEL || (isDev ? "debug" : "info");
@@ -8,96 +9,6 @@ const logsDir = path.resolve(process.cwd(), "logs");
fs.mkdirSync(logsDir, { recursive: true });
type LogMetadata = Record<string, unknown>;
type SerializedError = {
name: string;
message: string;
stack?: string;
code?: unknown;
statusCode?: unknown;
} & Record<string, unknown>;
const serializeError = (error: Error): SerializedError => {
const serialized: SerializedError = {
name: error.name,
message: error.message,
};
if (error.stack) {
serialized.stack = error.stack;
}
const errorWithFields = error as Error & {
code?: unknown;
statusCode?: unknown;
[key: string]: unknown;
};
if (errorWithFields.code !== undefined) {
serialized.code = errorWithFields.code;
}
if (errorWithFields.statusCode !== undefined) {
serialized.statusCode = errorWithFields.statusCode;
}
for (const [key, value] of Object.entries(errorWithFields)) {
if (serialized[key] === undefined) {
serialized[key] = value;
}
}
return serialized;
};
const isPlainObject = (value: unknown): value is Record<string, unknown> => {
if (!value || typeof value !== "object") {
return false;
}
const prototype = Object.getPrototypeOf(value);
return prototype === Object.prototype || prototype === null;
};
const serializeLogValue = (value: unknown): unknown => {
if (value instanceof Error) {
return serializeError(value);
}
if (value instanceof Date) {
return value.toISOString();
}
if (value instanceof RegExp) {
return value.toString();
}
if (Array.isArray(value)) {
return value.map(serializeLogValue);
}
if (isPlainObject(value)) {
return Object.fromEntries(
Object.entries(value).map(([key, nestedValue]) => [
key,
serializeLogValue(nestedValue),
]),
);
}
return value;
};
const formatLogMetadata = (metadata: LogMetadata): LogMetadata => {
return Object.fromEntries(
Object.entries(metadata).map(([key, value]) => [
key,
serializeLogValue(value),
]),
);
};
const metadataFormat = winston.format((info) => {
const {
level: _level,
src/loggerSerialization.ts
+89
View File
@@ -0,0 +1,89 @@
export type LogMetadata = Record<string, unknown>;
type SerializedError = {
name: string;
message: string;
stack?: string;
code?: unknown;
statusCode?: unknown;
} & Record<string, unknown>;
const serializeError = (error: Error): SerializedError => {
const serialized: SerializedError = {
name: error.name,
message: error.message,
};
if (error.stack) {
serialized.stack = error.stack;
}
const errorWithFields = error as Error & {
code?: unknown;
statusCode?: unknown;
[key: string]: unknown;
};
if (errorWithFields.code !== undefined) {
serialized.code = errorWithFields.code;
}
if (errorWithFields.statusCode !== undefined) {
serialized.statusCode = errorWithFields.statusCode;
}
for (const [key, value] of Object.entries(errorWithFields)) {
if (serialized[key] === undefined) {
serialized[key] = value;
}
}
return serialized;
};
const isPlainObject = (value: unknown): value is Record<string, unknown> => {
if (!value || typeof value !== "object") {
return false;
}
const prototype = Object.getPrototypeOf(value);
return prototype === Object.prototype || prototype === null;
};
export const serializeLogValue = (value: unknown): unknown => {
if (value instanceof Error) {
return serializeError(value);
}
if (value instanceof Date) {
return value.toISOString();
}
if (value instanceof RegExp) {
return value.toString();
}
if (Array.isArray(value)) {
return value.map(serializeLogValue);
}
if (isPlainObject(value)) {
return Object.fromEntries(
Object.entries(value).map(([key, nestedValue]) => [
key,
serializeLogValue(nestedValue),
]),
);
}
return value;
};
export const formatLogMetadata = (metadata: LogMetadata): LogMetadata => {
return Object.fromEntries(
Object.entries(metadata).map(([key, value]) => [
key,
serializeLogValue(value),
]),
);
};
src/moderation/attachmentUploader.ts
+25 -23
View File
@@ -1,6 +1,5 @@
import { config } from "../config";
import { createChildLogger } from "../logger";
import type { SqliteDatabase } from "../muxer-queue";
import { retryWithBackoff } from "../retry";
import {
updateAttachmentAsFailedUpload,
@@ -9,6 +8,17 @@ import {
const logger = createChildLogger("attachment-uploader");
const ATTACHMENT_UPLOAD_RETRY_OPTIONS = {
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
minTimeout: 1000,
maxTimeout: 5000,
logger,
} as const;
function toErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
export interface PicserUploadResponse {
success: boolean;
filename: string;
@@ -60,34 +70,26 @@ export async function uploadAttachmentToPicser(
formData.append("file", blob, filename);
try {
const response = await retryWithBackoff(
async () => {
const res = await fetch(config.PICSER_UPLOAD_URL, {
method: "POST",
body: formData,
signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS),
});
const response = await retryWithBackoff(async () => {
const res = await fetch(config.PICSER_UPLOAD_URL, {
method: "POST",
body: formData,
signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS),
});
if (!res.ok) {
throw new Error(`Upload failed with status ${res.status}`);
}
if (!res.ok) {
throw new Error(`Upload failed with status ${res.status}`);
}
return res.json() as Promise<PicserUploadResponse>;
},
{
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
minTimeout: 1000,
maxTimeout: 5000,
logger,
},
);
return res.json() as Promise<PicserUploadResponse>;
}, ATTACHMENT_UPLOAD_RETRY_OPTIONS);
return parseUploadResponse(response);
} catch (error) {
logger.error(
{
filename,
error: error instanceof Error ? error.message : String(error),
error: toErrorMessage(error),
},
"Failed to upload attachment",
);
@@ -109,7 +111,7 @@ export async function downloadDiscordAttachment(url: string): Promise<Buffer> {
return Buffer.from(buffer);
} catch (error) {
logger.error(
{ url, error: error instanceof Error ? error.message : String(error) },
{ url, error: toErrorMessage(error) },
"Failed to download Discord attachment",
);
throw error;
@@ -135,7 +137,7 @@ export async function processAttachmentUpload(
await updateAttachmentAsUploaded(attachmentId, result.url, Date.now());
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
const errorMsg = toErrorMessage(error);
await updateAttachmentAsFailedUpload(attachmentId, errorMsg);
logger.error({ attachmentId, error: errorMsg }, "Attachment upload failed");
}
src/moderation/messageCapture.ts
+49 -32
View File
@@ -1,6 +1,7 @@
import type { Client, Message } from "discord.js-selfbot-v13";
import { config } from "../config";
import { createChildLogger } from "../logger";
import { getModerationBroadcaster } from "../ws/broadcastGlobals";
import { queueMessageAnalysis } from "./aiAnalyzer";
import { processAttachmentUpload } from "./attachmentUploader";
import {
@@ -15,22 +16,10 @@ import {
updateMessageAsEdited,
upsertMessageForCapture,
} from "./messageStore";
import type {
AttachmentRecord,
MessageRecord,
ModerationBroadcaster,
} from "./types";
import type { AttachmentRecord, MessageRecord } from "./types";
const logger = createChildLogger("message-capture");
type ModerationGlobal = typeof globalThis & {
moderationBroadcaster?: ModerationBroadcaster;
};
function getModerationBroadcaster(): ModerationBroadcaster | undefined {
return (globalThis as ModerationGlobal).moderationBroadcaster;
}
export interface TextCaptureTarget {
guildId?: string;
channelId?: string;
@@ -57,15 +46,14 @@ function getTextCaptureTarget(): TextCaptureTarget {
};
}
export async function captureMessage(
function buildMessageRecord(
message: Message,
type: "text" | "edited" | "deleted",
options: { source?: "live" | "backlog" } = {},
): Promise<void> {
): MessageRecord {
const location = getMessageLocation(message);
const metadata = getMessageMetadata(message);
const messageRecord: MessageRecord = {
return {
id: message.id,
guild_id: message.guildId!,
channel_id: location.channelId,
@@ -81,6 +69,45 @@ export async function captureMessage(
type,
metadata: JSON.stringify(metadata),
};
}
function buildAttachmentRecord(
message: Message,
location: ReturnType<typeof getMessageLocation>,
attachment: {
id: string;
name: string | null;
size: number;
contentType: string | null;
url: string;
},
): AttachmentRecord {
return {
id: attachment.id,
message_id: message.id,
guild_id: message.guildId!,
channel_id: location.channelId,
thread_id: location.threadId,
user_id: message.author?.id,
filename: attachment.name || "unknown",
size: attachment.size,
type: attachment.contentType || "application/octet-stream",
discord_url: attachment.url,
uploaded_url: null,
upload_status: "pending",
upload_error: null,
created_at: Date.now(),
uploaded_at: null,
};
}
export async function captureMessage(
message: Message,
type: "text" | "edited" | "deleted",
options: { source?: "live" | "backlog" } = {},
): Promise<void> {
const location = getMessageLocation(message);
const messageRecord = buildMessageRecord(message, type);
const inserted = await upsertMessageForCapture(messageRecord);
if (!inserted) {
@@ -97,23 +124,13 @@ export async function captureMessage(
// Insert attachments before queuing analysis to avoid race condition
if (message.attachments.size > 0) {
for (const [, attachment] of message.attachments) {
const attachmentRecord: AttachmentRecord = {
const attachmentRecord = buildAttachmentRecord(message, location, {
id: attachment.id,
message_id: message.id,
guild_id: message.guildId!,
channel_id: location.channelId,
thread_id: location.threadId,
user_id: message.author?.id,
filename: attachment.name || "unknown",
name: attachment.name,
size: attachment.size,
type: attachment.contentType || "application/octet-stream",
discord_url: attachment.url,
uploaded_url: null,
upload_status: "pending",
upload_error: null,
created_at: Date.now(),
uploaded_at: null,
};
contentType: attachment.contentType,
url: attachment.url,
});
await insertAttachment(attachmentRecord);
src/moderation/messageStore.ts
+2 -22
View File
@@ -12,6 +12,7 @@ import {
import { getDatabase } from "../database/drizzle.ts";
import { attachmentsTable, messagesTable } from "../database/schema.ts";
import { createChildLogger } from "../logger.ts";
import { decodeCursor, encodeCursor } from "./pagination";
import type {
AttachmentRecord,
MessageQuery,
@@ -44,28 +45,7 @@ function db(): MessageDatabase {
return getDatabase() as unknown as MessageDatabase;
}
// Cursor helpers for pagination
interface CursorData {
created_at: number;
id: string;
}
export function encodeCursor(data: CursorData): string {
return Buffer.from(JSON.stringify(data)).toString("base64");
}
export function decodeCursor(cursor?: string): CursorData | null {
if (!cursor) return null;
try {
const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8"));
if (typeof data.created_at === "number" && typeof data.id === "string") {
return data;
}
return null;
} catch {
return null;
}
}
export { decodeCursor, encodeCursor } from "./pagination";
export async function insertMessage(message: MessageRecord): Promise<void> {
try {
src/moderation/pagination.ts
+21
View File
@@ -0,0 +1,21 @@
export interface CursorData {
created_at: number;
id: string;
}
export function encodeCursor(data: CursorData): string {
return Buffer.from(JSON.stringify(data)).toString("base64");
}
export function decodeCursor(cursor?: string): CursorData | null {
if (!cursor) return null;
try {
const data = JSON.parse(Buffer.from(cursor, "base64").toString("utf-8"));
if (typeof data.created_at === "number" && typeof data.id === "string") {
return data;
}
return null;
} catch {
return null;
}
}
src/ws/broadcastGlobals.ts
+4
View File
@@ -22,6 +22,10 @@ type VoiceGlobals = typeof globalThis & {
) => void;
};
export function getModerationBroadcaster(): ModerationBroadcaster | undefined {
return (globalThis as VoiceGlobals).moderationBroadcaster;
}
export function exposeModerationGlobals(
broadcaster: ModerationBroadcaster,
adminPassword: string,