exceed/dc-recorder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: migrate muxer-queue to drizzle-orm

Browse Source
This commit is contained in:
MythEclipse
2026-05-14 15:35:55 +07:00
parent 7e528a473b
commit dfe3444018
1 changed files with 145 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

358
src/muxer-queue.ts
View File

@@ -1,13 +1,12 @@
import { import { getDatabase as getDrizzleDatabase, initializeDatabase } from "./database/drizzle";
DatabaseAdapter, import { muxerJobsTable, uiStateTable } from "./database/schema";
getDatabase as getDatabaseAdapter, import { eq, asc, lt, and, sql } from "drizzle-orm";
} from "./database/adapter";
import { createChildLogger } from "./logger"; import { createChildLogger } from "./logger";
const logger = createChildLogger("muxer-queue"); const logger = createChildLogger("muxer-queue");
// Export DatabaseAdapter as SqliteDatabase for backward compatibility // Type alias for backward compatibility
export type SqliteDatabase = DatabaseAdapter; export type SqliteDatabase = any;
export interface MuxerJobData { export interface MuxerJobData {
userId: string; userId: string;
@@ -27,130 +26,28 @@ interface StoredJob {
error?: string; error?: string;
} }
let dbAdapter: DatabaseAdapter | null = null; // Export getDatabase for backward compatibility with webserver.ts
export function getDatabase(): SqliteDatabase {
async function initializeDatabase(): Promise<DatabaseAdapter> { return getDrizzleDatabase() as any;
const adapter = await getDatabaseAdapter();
adapter.exec(`
PRAGMA journal_mode = WAL;
CREATE TABLE IF NOT EXISTS muxer_jobs (
id TEXT PRIMARY KEY,
data TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
maxAttempts INTEGER NOT NULL DEFAULT 3,
createdAt INTEGER NOT NULL,
updatedAt INTEGER NOT NULL,
error TEXT
);
CREATE INDEX IF NOT EXISTS idx_status ON muxer_jobs(status);
CREATE INDEX IF NOT EXISTS idx_createdAt ON muxer_jobs(createdAt);
CREATE TABLE IF NOT EXISTS messages (
id TEXT PRIMARY KEY,
guild_id TEXT NOT NULL,
channel_id TEXT NOT NULL,
thread_id TEXT,
user_id TEXT NOT NULL,
username TEXT NOT NULL,
avatar_url TEXT,
content TEXT NOT NULL,
edited_content TEXT,
created_at INTEGER NOT NULL,
edited_at INTEGER,
deleted_at INTEGER,
type TEXT NOT NULL DEFAULT 'text',
metadata TEXT,
ai_status TEXT NOT NULL DEFAULT 'pending',
ai_moderation_flags TEXT,
ai_moderation_score REAL,
ai_moderation_raw TEXT,
ai_analysis TEXT,
ai_analyzed_at INTEGER,
ai_error TEXT
);
CREATE INDEX IF NOT EXISTS idx_messages_channel ON messages(channel_id);
CREATE INDEX IF NOT EXISTS idx_messages_user ON messages(user_id);
CREATE INDEX IF NOT EXISTS idx_messages_created ON messages(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_messages_thread ON messages(thread_id);
CREATE TABLE IF NOT EXISTS attachments (
id TEXT PRIMARY KEY,
message_id TEXT NOT NULL,
guild_id TEXT NOT NULL,
channel_id TEXT NOT NULL,
thread_id TEXT,
user_id TEXT NOT NULL,
filename TEXT NOT NULL,
size INTEGER NOT NULL,
type TEXT NOT NULL,
discord_url TEXT NOT NULL,
uploaded_url TEXT,
upload_status TEXT NOT NULL DEFAULT 'pending',
upload_error TEXT,
created_at INTEGER NOT NULL,
uploaded_at INTEGER,
FOREIGN KEY (message_id) REFERENCES messages(id)
);
CREATE INDEX IF NOT EXISTS idx_attachments_channel ON attachments(channel_id);
CREATE INDEX IF NOT EXISTS idx_attachments_message ON attachments(message_id);
CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(upload_status);
CREATE TABLE IF NOT EXISTS ui_state (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
`);
const migrations = [
"ALTER TABLE attachments ADD COLUMN thread_id TEXT",
"ALTER TABLE messages ADD COLUMN ai_status TEXT NOT NULL DEFAULT 'pending'",
"ALTER TABLE messages ADD COLUMN ai_moderation_flags TEXT",
"ALTER TABLE messages ADD COLUMN ai_moderation_score REAL",
"ALTER TABLE messages ADD COLUMN ai_moderation_raw TEXT",
"ALTER TABLE messages ADD COLUMN ai_analysis TEXT",
"ALTER TABLE messages ADD COLUMN ai_analyzed_at INTEGER",
"ALTER TABLE messages ADD COLUMN ai_error TEXT",
];
for (const migration of migrations) {
try {
adapter.exec(migration);
} catch {
// Column already exists on databases initialized after schema updates.
} }
}
return adapter;
}
async function getDatabaseAdapterInternal(): Promise<DatabaseAdapter> {
if (!dbAdapter) {
dbAdapter = await initializeDatabase();
}
return dbAdapter;
}
// Export as getDatabase for backward compatibility
export const getDatabase = getDatabaseAdapterInternal;
export async function getPersistedValue<T>( export async function getPersistedValue<T>(
key: string, key: string,
fallback: T, fallback: T,
): Promise<T> { ): Promise<T> {
const adapter = await getDatabaseAdapterInternal(); await initializeDatabase();
const row = adapter const db = getDrizzleDatabase() as any;
.prepare("SELECT value FROM ui_state WHERE key = ?")
.get(key) as { value: string } | undefined; const row = await db
if (!row) return fallback; .select()
.from(uiStateTable)
.where(eq(uiStateTable.key, key))
.limit(1);
if (!row || row.length === 0) return fallback;
try { try {
return JSON.parse(row.value) as T; return JSON.parse(row[0].value) as T;
} catch { } catch {
return fallback; return fallback;
} }
@@ -160,28 +57,45 @@ export async function setPersistedValue(
key: string, key: string,
value: unknown, value: unknown,
): Promise<void> { ): Promise<void> {
const adapter = await getDatabaseAdapterInternal(); await initializeDatabase();
adapter const db = getDrizzleDatabase() as any;
.prepare(`
INSERT INTO ui_state (key, value, updated_at) await db
VALUES (?, ?, ?) .insert(uiStateTable)
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at .values({
`) key,
.run(key, JSON.stringify(value), Date.now()); value: JSON.stringify(value),
updated_at: Date.now(),
})
.onConflictDoUpdate({
target: uiStateTable.key,
set: {
value: JSON.stringify(value),
updated_at: Date.now(),
},
});
} }
export async function enqueueMuxerJob(data: MuxerJobData): Promise<string> { export async function enqueueMuxerJob(data: MuxerJobData): Promise<string> {
try { try {
const adapter = await getDatabaseAdapterInternal(); await initializeDatabase();
const db = getDrizzleDatabase() as any;
const jobId = `${data.userId}-${data.sessionId}`; const jobId = `${data.userId}-${data.sessionId}`;
const now = Date.now(); const now = Date.now();
const stmt = adapter.prepare(` await db
INSERT INTO muxer_jobs (id, data, status, attempts, maxAttempts, createdAt, updatedAt) .insert(muxerJobsTable)
VALUES (?, ?, ?, ?, ?, ?, ?) .values({
`); id: jobId,
data: JSON.stringify(data),
stmt.run(jobId, JSON.stringify(data), "pending", 0, 3, now, now); status: "pending",
attempts: 0,
maxAttempts: 3,
createdAt: now,
updatedAt: now,
})
.onConflictDoNothing();
logger.info( logger.info(
{ jobId, userId: data.userId, sessionId: data.sessionId }, { jobId, userId: data.userId, sessionId: data.sessionId },
@@ -202,29 +116,25 @@ export async function enqueueMuxerJob(data: MuxerJobData): Promise<string> {
} }
export async function getPendingJobs(): Promise<StoredJob[]> { export async function getPendingJobs(): Promise<StoredJob[]> {
const adapter = await getDatabaseAdapterInternal(); await initializeDatabase();
const stmt = adapter.prepare(` const db = getDrizzleDatabase() as any;
SELECT id, data, status, attempts, maxAttempts, createdAt, updatedAt, error
FROM muxer_jobs
WHERE status = 'pending'
ORDER BY createdAt ASC
LIMIT 10
`);
const rows = stmt.all() as Array<{ const rows = await db
id: string; .select()
data: string; .from(muxerJobsTable)
status: string; .where(eq(muxerJobsTable.status, "pending"))
attempts: number; .orderBy(asc(muxerJobsTable.createdAt))
maxAttempts: number; .limit(10);
createdAt: number;
updatedAt: number;
error?: string;
}>;
return rows.map((row) => ({ return rows.map((row: any) => ({
...row, id: row.id,
data: row.data,
status: row.status as "pending" | "processing" | "completed" | "failed", status: row.status as "pending" | "processing" | "completed" | "failed",
attempts: row.attempts,
maxAttempts: row.maxAttempts,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
error: row.error || undefined,
})); }));
} }
@@ -233,34 +143,44 @@ export async function updateJobStatus(
status: "processing" | "completed" | "failed", status: "processing" | "completed" | "failed",
error?: string, error?: string,
): Promise<void> { ): Promise<void> {
const adapter = await getDatabaseAdapterInternal(); await initializeDatabase();
const db = getDrizzleDatabase() as any;
const now = Date.now(); const now = Date.now();
if (status === "failed") { if (status === "failed") {
const stmt = adapter.prepare(` await db
UPDATE muxer_jobs .update(muxerJobsTable)
SET status = ?, attempts = attempts + 1, updatedAt = ?, error = ? .set({
WHERE id = ? status,
`); attempts: sql`${muxerJobsTable.attempts} + 1`,
stmt.run(status, now, error || null, jobId); updatedAt: now,
error: error || null,
})
.where(eq(muxerJobsTable.id, jobId));
} else { } else {
const stmt = adapter.prepare(` await db
UPDATE muxer_jobs .update(muxerJobsTable)
SET status = ?, updatedAt = ? .set({
WHERE id = ? status,
`); updatedAt: now,
stmt.run(status, now, jobId); })
.where(eq(muxerJobsTable.id, jobId));
} }
logger.info({ jobId, status, error }, "Job status updated"); logger.info({ jobId, status, error }, "Job status updated");
} }
export async function retryFailedJob(jobId: string): Promise<boolean> { export async function retryFailedJob(jobId: string): Promise<boolean> {
const adapter = await getDatabaseAdapterInternal(); await initializeDatabase();
const db = getDrizzleDatabase() as any;
const job = adapter const jobs = await db
.prepare("SELECT * FROM muxer_jobs WHERE id = ?") .select()
.get(jobId) as StoredJob | undefined; .from(muxerJobsTable)
.where(eq(muxerJobsTable.id, jobId))
.limit(1);
const job = jobs[0];
if (!job) { if (!job) {
logger.warn({ jobId }, "Job not found"); logger.warn({ jobId }, "Job not found");
@@ -275,13 +195,14 @@ export async function retryFailedJob(jobId: string): Promise<boolean> {
return false; return false;
} }
const stmt = adapter.prepare(` await db
UPDATE muxer_jobs .update(muxerJobsTable)
SET status = 'pending', updatedAt = ? .set({
WHERE id = ? status: "pending",
`); updatedAt: Date.now(),
})
.where(eq(muxerJobsTable.id, jobId));
stmt.run(Date.now(), jobId);
logger.info({ jobId, attempt: job.attempts + 1 }, "Job retried"); logger.info({ jobId, attempt: job.attempts + 1 }, "Job retried");
return true; return true;
@@ -290,18 +211,26 @@ export async function retryFailedJob(jobId: string): Promise<boolean> {
export async function cleanupCompletedJobs( export async function cleanupCompletedJobs(
olderThanMs: number = 24 * 60 * 60 * 1000, olderThanMs: number = 24 * 60 * 60 * 1000,
): Promise<number> { ): Promise<number> {
const adapter = await getDatabaseAdapterInternal(); await initializeDatabase();
const db = getDrizzleDatabase() as any;
const cutoffTime = Date.now() - olderThanMs; const cutoffTime = Date.now() - olderThanMs;
const stmt = adapter.prepare(` const result = await db
DELETE FROM muxer_jobs .delete(muxerJobsTable)
WHERE status = 'completed' AND updatedAt < ? .where(
`); and(
eq(muxerJobsTable.status, "completed"),
lt(muxerJobsTable.updatedAt, cutoffTime),
),
);
const result = stmt.run(cutoffTime); const deletedCount = typeof result === "object" && "rowsAffected" in result
logger.info({ deletedCount: result.changes }, "Cleaned up completed jobs"); ? result.rowsAffected
: 0;
return result.changes; logger.info({ deletedCount }, "Cleaned up completed jobs");
return deletedCount;
} }
export async function getJobStats(): Promise<{ export async function getJobStats(): Promise<{
@@ -310,36 +239,37 @@ export async function getJobStats(): Promise<{
completed: number; completed: number;
failed: number; failed: number;
}> { }> {
const adapter = await getDatabaseAdapterInternal(); await initializeDatabase();
const db = getDrizzleDatabase() as any;
const stats = adapter const rows = await db
.prepare(` .select({
SELECT status: muxerJobsTable.status,
SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending, count: sql<number>`COUNT(*)`,
SUM(CASE WHEN status = 'processing' THEN 1 ELSE 0 END) as processing, })
SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) as completed, .from(muxerJobsTable)
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed .groupBy(muxerJobsTable.status);
FROM muxer_jobs
`) const stats = {
.get() as { pending: 0,
pending: number | null; processing: 0,
processing: number | null; completed: 0,
completed: number | null; failed: 0,
failed: number | null;
}; };
return { for (const row of rows) {
pending: stats.pending || 0, const count = typeof row.count === "object" && "count" in row.count
processing: stats.processing || 0, ? (row.count as any).count
completed: stats.completed || 0, : Number(row.count);
failed: stats.failed || 0, if (row.status === "pending") stats.pending = count;
}; else if (row.status === "processing") stats.processing = count;
else if (row.status === "completed") stats.completed = count;
else if (row.status === "failed") stats.failed = count;
}
return stats;
} }
export async function closeQueue(): Promise<void> { export async function closeQueue(): Promise<void> {
if (dbAdapter) {
await dbAdapter.close();
dbAdapter = null;
logger.info("Muxer queue closed"); logger.info("Muxer queue closed");
} }
}