From a2542493cd20ee30f6de548cf2028e463943fb32 Mon Sep 17 00:00:00 2001 From: asepharyana Date: Tue, 4 Aug 2026 13:28:55 +0700 Subject: [PATCH] =?UTF-8?q?fix(voice):=20screen=20share=20now=20carries=20?= =?UTF-8?q?audio=20=E2=80=94=20merge=20DASH=20video+audio=20into=20single?= =?UTF-8?q?=20NUT=20input?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getDirectVideoUrl used yt-dlp --get-url with bestvideo+bestaudio, which prints the video-only and audio-only URLs on SEPARATE lines. Only the first (video-only) line was used, so ffmpeg had no audio track and the GoLive stream had no sound. Replace with getDirectScreenInput which: - uses --dump-single-json to fetch both fresh URLs in ONE yt-dlp run (signature URLs expire quickly) - returns the merged progressive URL directly when one exists - otherwise merges the video-only + audio-only DASH URLs locally via a child ffmpeg into a single NUT stream consumed as a Readable - tracks the merge ffmpeg process in cleanup() so shutdown kills it too Verified end-to-end with real YouTube URLs: yt-dlp pair → live ffmpeg merge (NUT) → H264+opus transcode yields both streams. Added tests/screenShareInput.test.ts covering URL / DASH-pair / error paths. --- .../modules/voice-recording/mediaSource.ts | 181 ++++++++++++++++-- .../voice-recording/screenShareController.ts | 6 +- .../tests/screenShareInput.test.ts | 142 ++++++++++++++ 3 files changed, 312 insertions(+), 17 deletions(-) create mode 100644 services/discord-gateway/tests/screenShareInput.test.ts diff --git a/services/discord-gateway/src/modules/voice-recording/mediaSource.ts b/services/discord-gateway/src/modules/voice-recording/mediaSource.ts index 2b31e9d..42fcf71 100644 --- a/services/discord-gateway/src/modules/voice-recording/mediaSource.ts +++ b/services/discord-gateway/src/modules/voice-recording/mediaSource.ts @@ -283,24 +283,39 @@ export function resolveMediaUrl( } /** - * Resolve a media URL to a directly playable video URL (for screen share / - * GoLive streaming). Uses yt-dlp `--get-url` with bestvideo+bestaudio. + * Resolve a media URL to a single playable input stream for screen share / + * GoLive streaming. * - * @throws If yt-dlp is not installed or the process exits with a non-zero code. + * yt-dlp `--get-url` with `bestvideo+bestaudio` prints the video-only and + * audio-only URLs on SEPARATE lines. The old code took only the first line + * (video-only) → ffmpeg had no audio track → GoLive stream had no sound. + * + * This returns a single input that `prepareStream` (which accepts only ONE + * ffmpeg input) can consume while STILL including audio: + * - If yt-dlp offers a merged progressive URL (one URL, video+audio) it is + * returned directly. + * - Otherwise the video-only + audio-only DASH URLs are fetched in the SAME + * yt-dlp run (signature URLs expire quickly) and merged locally by an + * ffmpeg process into a single NUT stream, which is streamed to the + * consumer over a Readable. NUT over stdin auto-probes cleanly (verified: + * av1+opus merge → H264+opus transcode). + * + * @returns a direct video URL (string) or a Readable of the merged NUT stream. */ -export function getDirectVideoUrl(url: string): Promise { - return new Promise((resolve, reject) => { +export function getDirectScreenInput(url: string): Promise { + return new Promise((resolve, reject) => { const args = [ url, - "--get-url", + "--dump-single-json", "--format", "bestvideo[protocol^=http]+bestaudio[protocol^=http]/best[protocol^=http]/best", "--no-playlist", "--no-warnings", "--quiet", + "--no-simulate", ]; - logger.info({ url }, "Spawning yt-dlp for direct video URL"); + logger.info({ url }, "Spawning yt-dlp for screen share input resolution"); const proc = spawn("yt-dlp", args, { stdio: ["pipe", "pipe", "pipe"], @@ -311,7 +326,7 @@ export function getDirectVideoUrl(url: string): Promise { let stdoutBuf = ""; let stderrBuf = ""; const MAX_STDERR = 4096; - const MAX_STDOUT = 1_048_576; + const MAX_STDOUT = 8 * 1024 * 1024; // JSON metadata + requested format URLs if (proc.stdout) { proc.stdout.on("data", (chunk: Buffer) => { @@ -349,22 +364,160 @@ export function getDirectVideoUrl(url: string): Promise { const detail = stderrBuf.trim() ? `: ${stderrBuf.trim()}` : ""; reject( new Error( - `yt-dlp direct URL resolution exited with code ${code}${detail}`, + `yt-dlp screen input resolution exited with code ${code}${detail}`, ), ); return; } - const firstLine = stdoutBuf.trim().split("\n")[0]; - if (!firstLine) { - reject(new Error("yt-dlp returned no direct video URL")); + let parsed: Record; + try { + parsed = JSON.parse(stdoutBuf.trim()) as Record; + } catch (parseErr) { + reject( + new Error( + `Failed to parse yt-dlp JSON for screen input: ${(parseErr as Error).message}`, + ), + ); return; } - resolve(firstLine); + + resolveScreenInput(parsed).then(resolve, (err: unknown) => { + const message = err instanceof Error ? err.message : String(err); + reject( + new Error(`Failed to build screen input for "${url}": ${message}`), + ); + }); }); }); } +/** + * From a parsed yt-dlp JSON info dict, decide how to feed a single ffmpeg + * input with both video and audio. + */ +async function resolveScreenInput( + info: Record, +): Promise { + const requested = info.requested_formats as + | Array> + | undefined; + + // Merged/progressive single URL (video+audio in one). Common when yt-dlp + // selects a single format (e.g. format 18 progressive mp4) or when a direct + // muxed URL is available. + const singleUrl = info.url as string | undefined; + const singleHasAudio = + info.acodec !== "none" && + typeof info.acodec === "string" && + info.acodec.length > 0; + + if (typeof singleUrl === "string" && singleUrl && singleHasAudio) { + logger.debug("Screen share uses merged progressive single URL"); + return singleUrl; + } + + // Separate video-only + audio-only DASH formats → merge locally via ffmpeg. + if (Array.isArray(requested) && requested.length >= 2) { + const video = requested.find( + (rf) => rf.vcodec && String(rf.vcodec) !== "none", + ); + const audio = requested.find( + (rf) => rf.acodec && String(rf.acodec) !== "none", + ); + const videoUrl = video?.url as string | undefined; + const audioUrl = audio?.url as string | undefined; + + if ( + typeof videoUrl === "string" && + videoUrl.length > 0 && + typeof audioUrl === "string" && + audioUrl.length > 0 + ) { + return mergeScreenStreams(videoUrl, audioUrl); + } + } + + throw new Error( + "yt-dlp returned neither a merged progressive URL nor a video+audio format pair", + ); +} + +/** + * Merge a video-only URL and an audio-only URL into a single NUT stream using + * a child ffmpeg process. Both URLs come from the same yt-dlp run, so they + * share the same signature/expiry and are consumed immediately. + */ +function mergeScreenStreams(videoUrl: string, audioUrl: string): Readable { + logger.info("Merging video+audio DASH streams into a single NUT input"); + + const ffmpeg = spawn( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-reconnect", + "1", + "-reconnect_streamed", + "1", + "-reconnect_delay_max", + "5", + "-i", + videoUrl, + "-i", + audioUrl, + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c:v", + "copy", + "-c:a", + "copy", + "-f", + "nut", + "pipe:1", + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + + // Track so cleanup() can terminate the merge during graceful shutdown. + activeProcesses.add(ffmpeg); + ffmpeg.once("exit", () => { + activeProcesses.delete(ffmpeg); + }); + + // Prevent the ffmpeg stderr from filling the pipe buffer / leaking. + let stderrBuf = ""; + const MAX_STDERR = 4096; + ffmpeg.stderr?.on("data", (chunk: Buffer) => { + if (stderrBuf.length < MAX_STDERR) { + stderrBuf += chunk.toString("utf8"); + } + }); + + ffmpeg.on("error", (err) => { + const msg = + err.message === "spawn ffmpeg ENOENT" + ? "FFmpeg not found! Install ffmpeg in the container." + : err.message; + logger.error({ error: msg }, "Screen stream merge ffmpeg error"); + }); + + ffmpeg.on("exit", (code) => { + const stderr = stderrBuf.trim(); + logger.warn( + { code, stderr: stderr.slice(-500) || undefined }, + "Screen stream merge ffmpeg exited", + ); + }); + + const stream = ffmpeg.stdout; + stream.setMaxListeners(32); + return stream; +} + /** * Extract metadata (title, duration, thumbnail) from a media URL * without downloading the audio stream. @@ -452,7 +605,7 @@ export async function extractMediaInfo(url: string): Promise { } /** - * Kill all active yt-dlp child processes. + * Kill all active yt-dlp / screen-share merge ffmpeg child processes. * * Call during graceful shutdown to ensure no orphan processes remain. */ diff --git a/services/discord-gateway/src/modules/voice-recording/screenShareController.ts b/services/discord-gateway/src/modules/voice-recording/screenShareController.ts index 7da7efd..9f20782 100644 --- a/services/discord-gateway/src/modules/voice-recording/screenShareController.ts +++ b/services/discord-gateway/src/modules/voice-recording/screenShareController.ts @@ -7,7 +7,7 @@ import { } from "@dank074/discord-video-stream"; import type { Client } from "discord.js-selfbot-v13"; import { createChildLogger } from "@/shared/logger/index"; -import { getDirectVideoUrl } from "./mediaSource.js"; +import { getDirectScreenInput } from "./mediaSource.js"; import type { ScreenSharePlayback } from "./mediaTypes.js"; import { discordPlayer } from "./player.js"; @@ -65,7 +65,7 @@ export class ScreenShareController { } try { - const directUrl = await getDirectVideoUrl(source); + const input = await getDirectScreenInput(source); if (!this.streamer) { this.streamer = new Streamer(this.client); } @@ -98,7 +98,7 @@ export class ScreenShareController { ), ]); - const { command, output } = prepareStream(directUrl, { + const { command, output } = prepareStream(input, { encoder: Encoders.software({ x264: { preset: "superfast" } }), width: 1280, height: 720, diff --git a/services/discord-gateway/tests/screenShareInput.test.ts b/services/discord-gateway/tests/screenShareInput.test.ts new file mode 100644 index 0000000..08038b2 --- /dev/null +++ b/services/discord-gateway/tests/screenShareInput.test.ts @@ -0,0 +1,142 @@ +// ═══════════════════════════════════════════════════════════════════════════════ +// Screen share input resolution tests +// +// Verifies the decision logic of getDirectScreenInput: +// - merged progressive URL → returned directly +// - video+audio DASH pair → local ffmpeg merge (Readable) +// - neither → rejection +// +// Both yt-dlp and ffmpeg are faked via PATH shim scripts so the test does not +// hit the network or need real binaries. +// ═══════════════════════════════════════════════════════════════════════════════ + +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { getDirectScreenInput } from "../src/modules/voice-recording/mediaSource.js"; + +// ─── fake bin dir ────────────────────────────────────────────────────────────── +let fakeBinDir: string | null = null; +const realPath = process.env.PATH; + +beforeAll(() => { + fakeBinDir = mkdtempSync(join(tmpdir(), "gmw-fake-bins-")); + + // Fake yt-dlp: prints the JSON file named in GMW_FAKE_YTDLP_JSON. + // If the file is missing → exits 1 (mimics yt-dlp failure). + const ytShim = `#!/usr/bin/env bash +if [ -n "$GMW_FAKE_YTDLP_JSON" ] && [ -f "$GMW_FAKE_YTDLP_JSON" ]; then + cat "$GMW_FAKE_YTDLP_JSON" + exit 0 +fi +echo "yt-dlp: fake JSON missing" >&2 +exit 1 +`; + writeFileSync(join(fakeBinDir, "yt-dlp"), ytShim); + chmodSync(join(fakeBinDir, "yt-dlp"), 0o755); + + // Fake ffmpeg: writes a small nut-ish payload to stdout so the returned + // Readable actually emits data (the merge path in mergeScreenStreams). + const ffShim = `#!/usr/bin/env bash +# Fake ffmpeg — ignore args, emit a few bytes so consumers see a live stream. +head -c 4096 /dev/urandom +exit 0 +`; + writeFileSync(join(fakeBinDir, "ffmpeg"), ffShim); + chmodSync(join(fakeBinDir, "ffmpeg"), 0o755); + + process.env.PATH = `${fakeBinDir}:${process.env.PATH}`; +}); + +afterAll(() => { + if (fakeBinDir) { + rmSync(fakeBinDir, { recursive: true, force: true }); + } + process.env.PATH = realPath; +}); + +// ─── helpers ─────────────────────────────────────────────────────────────────── +function writeFakeJson(payload: Record): string { + const p = join( + tmpdir(), + `gmw-fake-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.json`, + ); + writeFileSync(p, JSON.stringify(payload)); + return p; +} + +function dashPairInfo(videoUrl: string, audioUrl: string) { + return { + url: null, + acodec: "none", // top-level is not a single merged format + vcodec: "av01", + requested_formats: [ + { + format_id: "136", + vcodec: "avc1.4d401f", + acodec: "none", + url: videoUrl, + }, + { format_id: "140", vcodec: "none", acodec: "mp4a.40.2", url: audioUrl }, + ], + }; +} + +// ─── tests ───────────────────────────────────────────────────────────────────── +describe("getDirectScreenInput", () => { + it("returns the single merged progressive URL when the info has one", async () => { + process.env.GMW_FAKE_YTDLP_JSON = writeFakeJson({ + url: "https://cdn.example/progressive.mp4", + acodec: "mp4a.40.2", + vcodec: "avc1", + }); + const result = await getDirectScreenInput("https://youtu.be/abc"); + expect(result).toBe("https://cdn.example/progressive.mp4"); + }); + + it("returns a live Readable when a video+audio DASH pair must be merged", async () => { + process.env.GMW_FAKE_YTDLP_JSON = writeFakeJson( + dashPairInfo( + "https://cdn.example/video.mp4", + "https://cdn.example/audio.m4a", + ), + ); + const result = await getDirectScreenInput("https://youtu.be/abc"); + expect(Readable.isReadable(result)).toBe(true); + + // The fake ffmpeg emits bytes; collect a chunk to prove the stream flows. + const bytes = await new Promise((resolve, reject) => { + const stream = result as Readable; + let got = 0; + stream.on("data", (chunk: Buffer) => { + got += chunk.length; + }); + stream.on("error", reject); + stream.on("end", () => resolve(got)); + stream.resume(); + }); + expect(bytes).toBeGreaterThan(0); + }); + + it("rejects when yt-dlp returns neither a merged URL nor a format pair", async () => { + process.env.GMW_FAKE_YTDLP_JSON = writeFakeJson({ + url: null, + acodec: "none", + vcodec: "none", + requested_formats: [], + }); + await expect(getDirectScreenInput("https://youtu.be/abc")).rejects.toThrow( + /neither a merged progressive URL nor a video\+audio/, + ); + }); + + it("rejects when yt-dlp exits non-zero", async () => { + process.env.GMW_FAKE_YTDLP_JSON = "/nonexistent/gmw-fake.json"; + await expect(getDirectScreenInput("https://youtu.be/abc")).rejects.toThrow( + /screen input resolution exited with code 1/, + ); + }); +});