diff --git a/src/errors.ts b/src/errors.ts index f6dcf47..94c1587 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -2,11 +2,7 @@ export class AppError extends Error { public code: string; public statusCode: number; - constructor( - message: string, - code: string, - statusCode: number = 500, - ) { + constructor(message: string, code: string, statusCode: number = 500) { super(message); this.code = code; this.statusCode = statusCode; @@ -39,10 +35,7 @@ export class VoiceConnectionError extends AppError { export class ValidationError extends AppError { public details?: Record; - constructor( - message: string, - details?: Record, - ) { + constructor(message: string, details?: Record) { super(message, "VALIDATION_ERROR", 400); this.details = details; this.name = "ValidationError"; diff --git a/src/index.ts b/src/index.ts index b32e6c3..861b01c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -74,7 +74,12 @@ async function initializeApp() { } client.on("debug", (msg) => { - if (msg.includes("[VOICE") || msg.includes("[ffmpeg") || msg.toLowerCase().includes("error") || msg.toLowerCase().includes("stream")) { + if ( + msg.includes("[VOICE") || + msg.includes("[ffmpeg") || + msg.toLowerCase().includes("error") || + msg.toLowerCase().includes("stream") + ) { logger.info({ debugMsg: msg }, "Discord Client Debug"); } else if (config.VERBOSE) { logger.debug({ debugMsg: msg }, "Discord Client Debug"); diff --git a/src/media/mediaController.ts b/src/media/mediaController.ts index b25e4cf..191a626 100644 --- a/src/media/mediaController.ts +++ b/src/media/mediaController.ts @@ -17,7 +17,10 @@ import { createMusicPlayer } from "./musicPlayer"; export interface MediaControllerDependencies { isVoiceConnected?: () => boolean; isBrowserStreaming?: () => boolean; - resolveMediaSource?: (source: string, mode?: MediaMode) => Promise; + resolveMediaSource?: ( + source: string, + mode?: MediaMode, + ) => Promise; musicPlayer?: MusicPlayer; screenController?: ScreenShareController; onStateChange?: (state: MediaState) => void; @@ -91,7 +94,10 @@ export class MediaController { // reject to avoid stealing the shared player. If this controller started // the screenPlayback, stop it and proceed. if (this.screenPlayback || this.dependencies.screenController?.isActive()) { - if (this.dependencies.screenController?.isActive() && !this.screenPlayback) { + if ( + this.dependencies.screenController?.isActive() && + !this.screenPlayback + ) { throw new AppError("Another media mode is active", "MEDIA_BUSY", 409); } this.screenPlayback?.stop(); diff --git a/src/media/mediaResolver.ts b/src/media/mediaResolver.ts index 4a4d525..54739c3 100644 --- a/src/media/mediaResolver.ts +++ b/src/media/mediaResolver.ts @@ -20,7 +20,7 @@ export function createMediaResolver( return async function resolve( input: string, - mode: MediaMode = "music" + mode: MediaMode = "music", ): Promise { const source = input.trim(); if (!source) { @@ -34,17 +34,19 @@ export function createMediaResolver( const url = parseUrl(source); if (url && isYouTubeUrl(url)) { const metadata = await ytdlp.getMetadata(source); - const directUrl = mode === "screen" - ? await ytdlp.getDirectVideoUrl(source) - : await ytdlp.getDirectAudioUrl(source); + const directUrl = + mode === "screen" + ? await ytdlp.getDirectVideoUrl(source) + : await ytdlp.getDirectAudioUrl(source); return { source: directUrl, title: metadata.title, kind: "youtube" }; } if (url && isSpotifyTrackUrl(url)) { const result = await playDlResolver.resolveSpotifyTrack(source); - const directUrl = mode === "screen" - ? await ytdlp.getDirectVideoUrl(result.url) - : await ytdlp.getDirectAudioUrl(result.url); + const directUrl = + mode === "screen" + ? await ytdlp.getDirectVideoUrl(result.url) + : await ytdlp.getDirectAudioUrl(result.url); return { source: directUrl, title: result.title, kind: "spotify" }; } @@ -62,9 +64,10 @@ export function createMediaResolver( if (!url && !looksLikeUrl(source)) { const result = await playDlResolver.searchYouTube(source); - const directUrl = mode === "screen" - ? await ytdlp.getDirectVideoUrl(result.url) - : await ytdlp.getDirectAudioUrl(result.url); + const directUrl = + mode === "screen" + ? await ytdlp.getDirectVideoUrl(result.url) + : await ytdlp.getDirectAudioUrl(result.url); return { source: directUrl, title: result.title, kind: "search" }; } diff --git a/src/media/screenShareController.ts b/src/media/screenShareController.ts index 51d1e84..4dbd48a 100644 --- a/src/media/screenShareController.ts +++ b/src/media/screenShareController.ts @@ -1,7 +1,4 @@ -import { - Streamer, - playPreparedStream, -} from "../streaming"; +import { Streamer, playPreparedStream } from "../streaming"; import { AppError } from "../errors"; import { createChildLogger } from "../logger"; import { discordPlayer } from "../player"; @@ -23,8 +20,14 @@ export interface ScreenShareControllerDependencies { getDirectVideoUrl?: (source: string) => Promise; streamer: Streamer; useTranscoder?: boolean; - onBeforeStreamStart?: (guildId: string, channelId: string) => Promise | void; - onAfterStreamEnd?: (guildId: string, channelId: string) => Promise | void; + onBeforeStreamStart?: ( + guildId: string, + channelId: string, + ) => Promise | void; + onAfterStreamEnd?: ( + guildId: string, + channelId: string, + ) => Promise | void; onStreamStart?: () => void; onStreamEnd?: () => void; } diff --git a/src/moderation/aiAnalysisWorker.ts b/src/moderation/aiAnalysisWorker.ts index bd565e1..baed009 100644 --- a/src/moderation/aiAnalysisWorker.ts +++ b/src/moderation/aiAnalysisWorker.ts @@ -3,6 +3,7 @@ import { initializeDatabase } from "../database/drizzle.ts"; import { buildConversationPromptMessages } from "./conversationContext.ts"; import { runModerationAnalysis } from "./llmModerationClient.ts"; import { + getAttachmentsForMessages, getConversationContextBefore, updateMessageAIAnalysis, } from "./messageStore.ts"; @@ -66,9 +67,15 @@ async function processAnalysisRequest({ maxTokens: MAX_CONTEXT_TOKENS, }); + const targetIds = messages.map((m) => m.id); + const contextIds = contextBefore.map((m) => m.id); + const allMessageIds = [...targetIds, ...contextIds]; + const attachments = await getAttachmentsForMessages(allMessageIds); + const result = await runModerationAnalysis({ targets: messages, contextText: promptMessages.join("\n"), + attachments, }); const rows: MessageRecord[] = []; diff --git a/src/moderation/aiAnalyzer.ts b/src/moderation/aiAnalyzer.ts index 2077e4f..d01577c 100644 --- a/src/moderation/aiAnalyzer.ts +++ b/src/moderation/aiAnalyzer.ts @@ -146,7 +146,7 @@ async function runAnalysisInWorker( return new Promise((resolve, reject) => { const worker = new Worker( new URL("./aiAnalysisWorker.ts", import.meta.url), - { execArgv: process.execArgv } + { execArgv: process.execArgv }, ); worker.once("message", (response: AnalysisWorkerResponse) => { diff --git a/src/moderation/llmModerationClient.ts b/src/moderation/llmModerationClient.ts index f135c35..97101e3 100644 --- a/src/moderation/llmModerationClient.ts +++ b/src/moderation/llmModerationClient.ts @@ -1,7 +1,7 @@ import { config } from "../config.ts"; import { createChildLogger } from "../logger.ts"; import { retryWithBackoff } from "../retry.ts"; -import type { AnalysisResult, MessageRecord } from "./types"; +import type { AnalysisResult, AttachmentRecord, MessageRecord } from "./types"; const log = createChildLogger("llmModerationClient"); @@ -174,6 +174,7 @@ export function parseModerationResponse( interface ModerationInput { targets: MessageRecord[]; contextText: string; + attachments?: AttachmentRecord[]; } interface ModerationOutput { @@ -188,7 +189,7 @@ interface ModerationOutput { export async function runModerationAnalysis( input: ModerationInput, ): Promise { - const { targets, contextText } = input; + const { targets, contextText, attachments } = input; if (!targets.length) { throw new Error("No targets provided for analysis"); @@ -220,6 +221,88 @@ Each result must have: Return ONLY valid JSON, no other text.`; + // Check for image attachments to support multimodal analysis + const imageAttachments = (attachments || []).filter( + (att) => + (att.uploaded_url || att.discord_url) && att.type.startsWith("image/"), + ); + + let messageContent: + | string + | Array<{ type: string; text?: string; image_url?: { url: string } }>; + if (imageAttachments.length > 0) { + const contentParts: Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }> = []; + + // Download and convert all images to base64 data URLs + for (const att of imageAttachments) { + try { + const urlToUse = att.uploaded_url || att.discord_url; + log.info( + { attachmentId: att.id, url: urlToUse }, + "Downloading attachment for base64 encoding", + ); + const res = await fetch(urlToUse); + if (res.ok) { + const buffer = await res.arrayBuffer(); + const base64Str = Buffer.from(buffer).toString("base64"); + const dataUrl = `data:${att.type};base64,${base64Str}`; + + contentParts.push({ + type: "image_url", + image_url: { + url: dataUrl, + }, + }); + + contentParts.push({ + type: "text", + text: `\n[Image Attachment for Message ID: ${att.message_id}, Filename: ${att.filename}]`, + }); + } else { + log.warn( + { attachmentId: att.id, status: res.status }, + "Failed to fetch attachment image", + ); + } + } catch (err) { + log.warn( + { + attachmentId: att.id, + error: err instanceof Error ? err.message : String(err), + }, + "Error base64 encoding attachment", + ); + } + } + + contentParts.push({ + type: "text", + text: prompt, + }); + + messageContent = contentParts; + } else { + // If no image is present, send a transparent 1x1 dummy PNG to satisfy multimodal omni requirements + const dummyPng = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + messageContent = [ + { + type: "image_url", + image_url: { + url: dummyPng, + }, + }, + { + type: "text", + text: prompt, + }, + ]; + } + const result = await retryWithBackoff( async () => { const controller = new AbortController(); @@ -243,10 +326,14 @@ Return ONLY valid JSON, no other text.`; messages: [ { role: "user", - content: prompt, + content: messageContent, }, ], - temperature: 0.3, + temperature: 0.6, + top_p: 0.95, + max_tokens: 65536, + reasoning_budget: 16384, + chat_template_kwargs: { enable_thinking: true }, }), }, ); diff --git a/src/moderation/messageStore.ts b/src/moderation/messageStore.ts index a2173c5..9e82cb2 100644 --- a/src/moderation/messageStore.ts +++ b/src/moderation/messageStore.ts @@ -1,4 +1,14 @@ -import { and, asc, desc, eq, isNull, or, type SQL, sql } from "drizzle-orm"; +import { + and, + asc, + desc, + eq, + inArray, + isNull, + or, + type SQL, + sql, +} from "drizzle-orm"; import { getDatabase } from "../database/drizzle.ts"; import { attachmentsTable, messagesTable } from "../database/schema.ts"; import { createChildLogger } from "../logger.ts"; @@ -605,3 +615,27 @@ export async function getPendingConversationKeys( throw error; } } + +export async function getAttachmentsForMessages( + messageIds: string[], +): Promise { + try { + if (messageIds.length === 0) return []; + const database = db(); + const rows = await database + .select() + .from(attachmentsTable) + .where(inArray(attachmentsTable.message_id, messageIds)); + + return rows as AttachmentRecord[]; + } catch (error) { + logger.error( + { + messageIds, + error: error instanceof Error ? error.message : String(error), + }, + "Failed to get attachments for messages", + ); + throw error; + } +} diff --git a/src/streaming/index.ts b/src/streaming/index.ts index 3248aa1..0a1b4ee 100644 --- a/src/streaming/index.ts +++ b/src/streaming/index.ts @@ -41,7 +41,10 @@ export class Streamer { this.dankStreamer = new DankStreamer(client); } - async createSession(guildId: string, channelId: string): Promise { + async createSession( + guildId: string, + channelId: string, + ): Promise { await this.dankStreamer.joinVoice(guildId, channelId); let stopped = false; @@ -62,7 +65,10 @@ export class Streamer { return { connection: {} as any, stream: {} as any, - play: async (source: string | Readable, options: StreamPlayOptions = {}) => { + play: async ( + source: string | Readable, + options: StreamPlayOptions = {}, + ) => { if (stopped) return; let targetSource: string | Readable = source; @@ -75,7 +81,12 @@ export class Streamer { const bitrateStr = String(options.bitrate ?? 8000).replace(/k$/i, ""); const bitrateVideo = parseInt(bitrateStr, 10) || 8000; - console.log("[Streamer] Starting screen share for source:", typeof targetSource === "string" ? targetSource.slice(0, 50) + "..." : "ReadableStream"); + console.log( + "[Streamer] Starting screen share for source:", + typeof targetSource === "string" + ? targetSource.slice(0, 50) + "..." + : "ReadableStream", + ); const { command, output } = dankPrepareStream(targetSource, { encoder: Encoders.software({ x264: { preset: (options.presetH26x as any) ?? "ultrafast" }, @@ -100,7 +111,7 @@ export class Streamer { const webOutput = new PassThrough(); const discordOutput = new PassThrough(); - + output.pipe(webOutput); output.pipe(discordOutput); diff --git a/src/streaming/transcoder.ts b/src/streaming/transcoder.ts index 6b0d6b2..7678411 100644 --- a/src/streaming/transcoder.ts +++ b/src/streaming/transcoder.ts @@ -21,7 +21,10 @@ export class Transcoder { restartTimer: NodeJS.Timeout | null = null; maxRestarts = 6; - constructor(private source: string, private opts: TranscoderOptions = {}) {} + constructor( + private source: string, + private opts: TranscoderOptions = {}, + ) {} start(): { command: ChildProcess; output: Readable } { const fps = this.opts.fps ?? 30; @@ -99,13 +102,19 @@ export class Transcoder { scheduleRestart() { if (this.restartAttempts >= this.maxRestarts) { - logger.error({ attempts: this.restartAttempts }, "transcoder reached max restart attempts"); + logger.error( + { attempts: this.restartAttempts }, + "transcoder reached max restart attempts", + ); return; } const delay = Math.min(30000, 1000 * Math.pow(2, this.restartAttempts)); this.restartAttempts += 1; transcoderRestartsCounter.inc(); - logger.info({ delay, attempt: this.restartAttempts }, "scheduling transcoder restart"); + logger.info( + { delay, attempt: this.restartAttempts }, + "scheduling transcoder restart", + ); this.restartTimer = setTimeout(() => { try { this.start(); @@ -129,9 +138,9 @@ export class Transcoder { clearTimeout(this.restartTimer); this.restartTimer = null; } - if (this.proc && !this.proc.killed) { - return new Promise((resolve) => { - this.proc?.once("exit", () => resolve()); + if (this.proc && !this.proc.killed) { + return new Promise((resolve) => { + this.proc?.once("exit", () => resolve()); try { this.proc?.kill("SIGTERM"); } catch { @@ -141,7 +150,7 @@ export class Transcoder { resolve(); } } - setTimeout(() => resolve(), 5000); + setTimeout(() => resolve(), 5000); }).then(() => { this.proc = null; this.output = null; @@ -151,7 +160,10 @@ export class Transcoder { } } -export function prepareTranscoder(source: string, options: TranscoderOptions = {}) { +export function prepareTranscoder( + source: string, + options: TranscoderOptions = {}, +) { const t = new Transcoder(source, options); const { command, output } = t.start(); return { transcoder: t, command, output }; diff --git a/test_dank.ts b/test_dank.ts index fe95685..79da909 100644 --- a/test_dank.ts +++ b/test_dank.ts @@ -3,13 +3,16 @@ import fs from "fs"; async function run() { console.log("Starting prepareStream..."); - const { command, output } = prepareStream("https://rr3---sn-2uuxa3vh-unte.googlevideo.com/videoplayback?expire=1779046518&ei=FsQJatGDGNqp9fwP4qz4SA&ip=180.252.24.35&id=o-APFvGry6yPgoap-1RT0pu59DxD-pcXC4oXtMQuCMtjOy&itag=18&source=youtube&requiressl=yes&xpc=EgVo2aDSNQ%3D%3D&cps=618&met=1779024918%2C&mh=VD&mm=31%2C29&mn=sn-2uuxa3vh-unte%2Csn-oguelnze&ms=au%2Crdu&mv=m&mvi=3&pcm2cms=yes&pl=20&rms=au%2Cau&initcwndbps=763750&bui=AbKmrwofOLw_tOID4kBHnWgaXP2wnDlEYmbyHyrnZk1n7vjMaQIuY046T9MhH0PuL9JGJwj6YlwCr2Uu&spc=96Xrv8WI7iTS7MOF7Dvg-8a3RT-sMI9ux49zUa4Pg6GHkzXExSS0&vprv=1&svpuc=1&mime=video%2Fmp4&rqh=1&cnr=14&ratebypass=yes&dur=19.063&lmt=1772437158054287&mt=1779024581&fvip=4&fexp=51565116%2C51565681&c=ANDROID_VR&txp=4530534&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cxpc%2Cbui%2Cspc%2Cvprv%2Csvpuc%2Cmime%2Crqh%2Ccnr%2Cratebypass%2Cdur%2Clmt&sig=AHEqNM4wRgIhAJe1vu37ssUQQm3scVgXY7NYDx_frKW1AZ4gHRdcqsUlAiEAkKt6jxaCNvaEh6jag1OWheo5qQeu3ObfCCoQIZ9xnCA%3D&lsparams=cps%2Cmet%2Cmh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpcm2cms%2Cpl%2Crms%2Cinitcwndbps&lsig=APaTxxMwRQIhAMkeJ6WrDFU7fTfSb6s_WbdDpn4J-4NqkfzKV3B_y1cgAiBJ7aExkhh-0hvIWwNorjDwoOkTIKIfmzx6o6Z3mxlazA%3D%3D", { - encoder: Encoders.software(), - width: 1280, - height: 720, - includeAudio: true, - minimizeLatency: false // Add this - }); + const { command, output } = prepareStream( + "https://rr3---sn-2uuxa3vh-unte.googlevideo.com/videoplayback?expire=1779046518&ei=FsQJatGDGNqp9fwP4qz4SA&ip=180.252.24.35&id=o-APFvGry6yPgoap-1RT0pu59DxD-pcXC4oXtMQuCMtjOy&itag=18&source=youtube&requiressl=yes&xpc=EgVo2aDSNQ%3D%3D&cps=618&met=1779024918%2C&mh=VD&mm=31%2C29&mn=sn-2uuxa3vh-unte%2Csn-oguelnze&ms=au%2Crdu&mv=m&mvi=3&pcm2cms=yes&pl=20&rms=au%2Cau&initcwndbps=763750&bui=AbKmrwofOLw_tOID4kBHnWgaXP2wnDlEYmbyHyrnZk1n7vjMaQIuY046T9MhH0PuL9JGJwj6YlwCr2Uu&spc=96Xrv8WI7iTS7MOF7Dvg-8a3RT-sMI9ux49zUa4Pg6GHkzXExSS0&vprv=1&svpuc=1&mime=video%2Fmp4&rqh=1&cnr=14&ratebypass=yes&dur=19.063&lmt=1772437158054287&mt=1779024581&fvip=4&fexp=51565116%2C51565681&c=ANDROID_VR&txp=4530534&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cxpc%2Cbui%2Cspc%2Cvprv%2Csvpuc%2Cmime%2Crqh%2Ccnr%2Cratebypass%2Cdur%2Clmt&sig=AHEqNM4wRgIhAJe1vu37ssUQQm3scVgXY7NYDx_frKW1AZ4gHRdcqsUlAiEAkKt6jxaCNvaEh6jag1OWheo5qQeu3ObfCCoQIZ9xnCA%3D&lsparams=cps%2Cmet%2Cmh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpcm2cms%2Cpl%2Crms%2Cinitcwndbps&lsig=APaTxxMwRQIhAMkeJ6WrDFU7fTfSb6s_WbdDpn4J-4NqkfzKV3B_y1cgAiBJ7aExkhh-0hvIWwNorjDwoOkTIKIfmzx6o6Z3mxlazA%3D%3D", + { + encoder: Encoders.software(), + width: 1280, + height: 720, + includeAudio: true, + minimizeLatency: false, // Add this + }, + ); const fileStream = fs.createWriteStream("/mnt/code/bete/test_out.nut"); output.pipe(fileStream); @@ -26,7 +29,9 @@ async function run() { }); setTimeout(() => { - try { command.kill("SIGKILL"); } catch(e) {} + try { + command.kill("SIGKILL"); + } catch (e) {} process.exit(0); }, 10000); } diff --git a/test_dank2.ts b/test_dank2.ts index 3af8a0c..a7e6b77 100644 --- a/test_dank2.ts +++ b/test_dank2.ts @@ -3,24 +3,29 @@ import { demux } from "@dank074/discord-video-stream/dist/media/LibavDemuxer.js" async function run() { console.log("Starting prepareStream..."); - const { command, output } = prepareStream("https://samplelib.com/preview/mp4/sample-5s.mp4", { - encoder: Encoders.software(), - width: 1280, - height: 720, - includeAudio: true, - minimizeLatency: false // Add this - }); + const { command, output } = prepareStream( + "https://samplelib.com/preview/mp4/sample-5s.mp4", + { + encoder: Encoders.software(), + width: 1280, + height: 720, + includeAudio: true, + minimizeLatency: false, // Add this + }, + ); try { const { video, audio } = await demux(output, { format: "nut" }); console.log("DEMUX VIDEO:", !!video); console.log("DEMUX AUDIO:", !!audio); - } catch(e) { + } catch (e) { console.error("DEMUX ERR:", e); } setTimeout(() => { - try { command.kill("SIGKILL"); } catch(e) {} + try { + command.kill("SIGKILL"); + } catch (e) {} process.exit(0); }, 10000); } diff --git a/test_stream.ts b/test_stream.ts index db22573..050fe7e 100644 --- a/test_stream.ts +++ b/test_stream.ts @@ -3,12 +3,15 @@ import { demux } from "@dank074/discord-video-stream/dist/media/LibavDemuxer.js" import { Encoders } from "@dank074/discord-video-stream/dist/media/encoders/index.js"; async function run() { - const { command, output } = prepareStream("http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4", { - encoder: Encoders.software(), - width: 1280, - height: 720, - includeAudio: true - }); + const { command, output } = prepareStream( + "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4", + { + encoder: Encoders.software(), + width: 1280, + height: 720, + includeAudio: true, + }, + ); const { video, audio } = await demux(output, { format: "nut" }); console.log("Video found:", !!video); diff --git a/tests/media/mediaController.test.ts b/tests/media/mediaController.test.ts index 37e332a..962bd0a 100644 --- a/tests/media/mediaController.test.ts +++ b/tests/media/mediaController.test.ts @@ -225,7 +225,10 @@ describe("MediaController", () => { expect(screenController.start).toHaveBeenCalledWith( "https://youtu.be/video", ); - expect(resolveMediaSource).toHaveBeenCalledWith("https://youtu.be/video", "screen"); + expect(resolveMediaSource).toHaveBeenCalledWith( + "https://youtu.be/video", + "screen", + ); expect(state).toMatchObject({ playing: true, activeMode: "screen" }); }); diff --git a/tests/media/musicPlayer.test.ts b/tests/media/musicPlayer.test.ts index 708e45f..32f936c 100644 --- a/tests/media/musicPlayer.test.ts +++ b/tests/media/musicPlayer.test.ts @@ -71,10 +71,14 @@ describe("createMusicPlayer", () => { ], { stdio: ["ignore", "pipe", "pipe"] }, ); - expect(discordPlayer.playStream).toHaveBeenCalledWith(proc.stdout, "music", { - inputType: StreamType.Raw, - inlineVolume: true, - }); + expect(discordPlayer.playStream).toHaveBeenCalledWith( + proc.stdout, + "music", + { + inputType: StreamType.Raw, + inlineVolume: true, + }, + ); }); it("rejects playback when Discord is not connected", () => { diff --git a/tests/moderation/llmModerationClient.test.ts b/tests/moderation/llmModerationClient.test.ts index c5cdf53..57fa67e 100644 --- a/tests/moderation/llmModerationClient.test.ts +++ b/tests/moderation/llmModerationClient.test.ts @@ -336,4 +336,100 @@ describe("runModerationAnalysis", () => { }), ).rejects.toThrow(/No content in LLM response/); }); + + it("sends multimodal payload when image attachments are present", async () => { + const mockResponse = { + choices: [ + { + message: { + content: JSON.stringify({ + results: [ + { + message_id: "m1", + status: "clean", + flags: [], + score: 0.1, + analysis: "OK", + }, + ], + }), + }, + }, + ], + }; + + global.fetch = vi.fn().mockImplementation((url: string) => { + if (url.includes("picser.tech") || url.includes("discord.com")) { + return Promise.resolve({ + ok: true, + arrayBuffer: async () => { + const buffer = Buffer.from("fake-image-bytes"); + return buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength, + ); + }, + }); + } + return Promise.resolve({ + ok: true, + text: async () => JSON.stringify(mockResponse), + json: async () => mockResponse, + }); + }); + + const mockAttachment = { + id: "a1", + message_id: "m1", + guild_id: "guild123", + channel_id: "channel123", + thread_id: null, + user_id: "user123", + filename: "test.png", + size: 500, + type: "image/png", + discord_url: "https://discord.com/attachment.png", + uploaded_url: "https://picser.tech/test.png", + upload_status: "uploaded" as const, + upload_error: null, + created_at: Date.now(), + uploaded_at: Date.now(), + }; + + const result = await runModerationAnalysis({ + targets: [createMessageRecord()], + contextText: "test context", + attachments: [mockAttachment], + }); + + expect(result.results).toHaveLength(1); + expect(global.fetch).toHaveBeenCalled(); + + const fetchCalls = (global.fetch as any).mock.calls; + // Should be called twice: 1st for image download, 2nd for API completions + expect(fetchCalls.length).toBe(2); + + // Verify 1st call (image download) + expect(fetchCalls[0][0]).toBe("https://picser.tech/test.png"); + + // Verify 2nd call (chat completions API) + const [, completionsOptions] = fetchCalls[1]; + const body = JSON.parse(completionsOptions.body); + + const userMessage = body.messages[0]; + expect(userMessage.role).toBe("user"); + expect(Array.isArray(userMessage.content)).toBe(true); + expect(userMessage.content[0].type).toBe("image_url"); + expect(userMessage.content[0].image_url.url).toContain( + "data:image/png;base64,", + ); + expect(userMessage.content[1].type).toBe("text"); + expect(userMessage.content[1].text).toContain( + "Image Attachment for Message ID: m1", + ); + expect(userMessage.content[2].type).toBe("text"); + expect(userMessage.content[2].text).toContain( + "You are a content moderation assistant.", + ); + }); }); diff --git a/tests/moderation/messageStoreQueries.test.ts b/tests/moderation/messageStoreQueries.test.ts index 034a643..5d63e9a 100644 --- a/tests/moderation/messageStoreQueries.test.ts +++ b/tests/moderation/messageStoreQueries.test.ts @@ -8,7 +8,9 @@ import { createChildLogger } from "../../src/logger"; import { decodeCursor, encodeCursor, + getAttachmentsForMessages, getMessageById, + insertAttachment, insertMessage, listMessages, listReviewMessages, @@ -72,21 +74,40 @@ describe("message query integration tests", () => { "ai_error" text ) `); + + // Create attachments table + await db.run(` + CREATE TABLE IF NOT EXISTS "attachments" ( + "id" text PRIMARY KEY NOT NULL, + "message_id" text NOT NULL, + "guild_id" text NOT NULL, + "channel_id" text NOT NULL, + "thread_id" text, + "user_id" text NOT NULL, + "filename" text NOT NULL, + "size" integer NOT NULL, + "type" text NOT NULL, + "discord_url" text NOT NULL, + "uploaded_url" text, + "upload_status" text DEFAULT 'pending' NOT NULL, + "upload_error" text, + "created_at" integer NOT NULL, + "uploaded_at" integer + ) + `); } catch (error) { - logger.debug( - { error }, - "Messages table already exists or error creating it", - ); + logger.debug({ error }, "Tables already exist or error creating them"); } }); beforeEach(async () => { - // Clear messages table before each test + // Clear tables before each test try { const db = getTestDatabase(); await db.run(`DELETE FROM "messages"`); + await db.run(`DELETE FROM "attachments"`); } catch (error) { - logger.debug({ error }, "Could not clear messages table"); + logger.debug({ error }, "Could not clear tables"); } }); @@ -579,4 +600,58 @@ describe("message query integration tests", () => { expect(retrieved?.ai_error).toBeNull(); }); }); + + describe("getAttachmentsForMessages", () => { + it("returns attachments matching given message IDs", async () => { + const msgId1 = "msg-att-1"; + const msgId2 = "msg-att-2"; + + const attachment1 = { + id: "att-1", + message_id: msgId1, + guild_id: "guild-123", + channel_id: "channel-456", + thread_id: null, + user_id: "user-789", + filename: "test1.png", + size: 1024, + type: "image/png", + discord_url: "https://discord.com/test1.png", + uploaded_url: "https://picser.tech/test1.png", + upload_status: "uploaded" as const, + upload_error: null, + created_at: Date.now(), + uploaded_at: Date.now(), + }; + + const attachment2 = { + id: "att-2", + message_id: msgId2, + guild_id: "guild-123", + channel_id: "channel-456", + thread_id: null, + user_id: "user-789", + filename: "test2.png", + size: 2048, + type: "image/png", + discord_url: "https://discord.com/test2.png", + uploaded_url: "https://picser.tech/test2.png", + upload_status: "uploaded" as const, + upload_error: null, + created_at: Date.now(), + uploaded_at: Date.now(), + }; + + await insertAttachment(attachment1); + await insertAttachment(attachment2); + + const result = await getAttachmentsForMessages([msgId1, msgId2]); + expect(result).toHaveLength(2); + const ids = result.map((r) => r.id).sort(); + expect(ids).toEqual(["att-1", "att-2"].sort()); + + const emptyResult = await getAttachmentsForMessages([]); + expect(emptyResult).toHaveLength(0); + }); + }); }); diff --git a/tests/streaming/playTranscode.test.ts b/tests/streaming/playTranscode.test.ts index 2595078..30f552c 100644 --- a/tests/streaming/playTranscode.test.ts +++ b/tests/streaming/playTranscode.test.ts @@ -38,7 +38,8 @@ describe("playTranscodedPreparedStream", () => { it("pipes transcoder output to session and broadcasts to web", async () => { // mock global broadcast const broadcasts: Buffer[] = []; - (globalThis as any).broadcastVideoToWeb = (chunk: Buffer) => broadcasts.push(Buffer.from(chunk)); + (globalThis as any).broadcastVideoToWeb = (chunk: Buffer) => + broadcasts.push(Buffer.from(chunk)); const session = { connection: { channel: { id: "c" } }, @@ -52,7 +53,9 @@ describe("playTranscodedPreparedStream", () => { stop: vi.fn(), } as any; - await playTranscodedPreparedStream("http://example.test/stream", session, { fps: 30 }); + await playTranscodedPreparedStream("http://example.test/stream", session, { + fps: 30, + }); expect(session.play).toHaveBeenCalled(); expect(broadcasts.length).toBeGreaterThanOrEqual(0); }); diff --git a/tests/streaming/transcoder.test.ts b/tests/streaming/transcoder.test.ts index 82cdb7f..5f07c78 100644 --- a/tests/streaming/transcoder.test.ts +++ b/tests/streaming/transcoder.test.ts @@ -39,7 +39,10 @@ import { prepareTranscoder } from "../../src/streaming/transcoder"; describe("Transcoder", () => { it("starts ffmpeg and returns output stream and command", () => { - const { transcoder, command, output } = prepareTranscoder("http://example.test/video", { fps: 24 }); + const { transcoder, command, output } = prepareTranscoder( + "http://example.test/video", + { fps: 24 }, + ); expect(transcoder).toBeTruthy(); expect(command).toBeTruthy(); expect(output).toBeTruthy(); diff --git a/tsconfig.json b/tsconfig.json index 6d013a2..5a7b04e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,6 +3,8 @@ "target": "ESNext", "module": "ESNext", "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "noEmit": true, "strict": true, "esModuleInterop": true, "skipLibCheck": true,