Compare commits

...
5 Commits
Author SHA1 Message Date
asepharyana 493bca590d fix(infra): build native voice deps (opus, datachannel) in Nix closure
Build & Deploy (Nix) / build-and-deploy (proxy) (push) Successful in 4m21s
Build & Deploy (Nix) / build-and-deploy (backend) (push) Successful in 4m22s
Build & Deploy (Nix) / build-and-deploy (discord-gateway) (push) Successful in 11m31s
pnpm 11 rebuild hanya jalanin script package yang di-approve via
pnpm-workspace.yaml (allowBuilds) DAN abort di kegagalan pertama.
Yaml harus tracked (flake source cuma ikut file git). Tapi pnpm rebuild
tetap gagal karena: (1) node-crc MSRV cargo:: error — dead dep, dihapus
dari deps+patch; (2) sharp install script gagal di sandbox — binary-nya
prebuilt @img, script cuma validasi — dikeluarkan dari approval list;
(3) node-datachannel prebuild CLI TypeError + cmake-js butuh cwd benar.

Fix: loop eksplisit di buildPhase gateway yang jalankan install script
tiap native dep (opus/node-datachannel/zeromq) dengan cwd package dir,
+ cmake (dontUseCmakeConfigure biar stdenv nggak auto-configure),
+ opensslDevEnv = symlinkJoin pkgsStatic.openssl.out (libcrypto.a —
CMakeLists set OPENSSL_USE_STATIC_LIBS=TRUE; pkgs.openssl default
output = bin tanpa lib) + openssl.dev (headers),
+ git (libdatachannel FetchContent clone dari GitHub).

Verifikasi: result store punya opus.node (compile source), node_datachannel.node
(compile source), node-av prebuilt, zeromq prebuilt; runtime smoke test:
PeerConnection instantiate+close OK, OpusEncoder encode OK, dank074
Streamer/prepareStream/playStream load OK.
2026-08-01 12:44:01 +07:00
asepharyana 823b484497 chore(gateway): pnpm 11 build-script approvals (allowBuilds) for native voice deps
pnpm 11.17 mengabaikan field pnpm.onlyBuiltDependencies di package.json.
Native deps voice (@discordjs/opus, @lng2004/node-datachannel, zeromq, dll)
tidak pernah kebangun di Nix store karena flake pnpmInstall pakai
--ignore-scripts dan pnpm rebuild tanpa approval. Hasil: receiver/rekaman/
GoLive diam-diam tanpa decoder/encoder native.

pnpm approve-builds --all menulis allowBuilds:true per package di
pnpm-workspace.yaml (harus tracked — flake source cuma ikut file git).
node-crc tetap gagal build (MSRV cargo:: check) tapi tidak pernah
di-import di source — harmless.
2026-08-01 11:47:01 +07:00
asepharyana 891c1305f0 feat(gateway): restore Discord GoLive screenshare (dulu pernah ada, hilang saat split microservices)
User: 'dulu sharescreen juga bisa'. Terbukti: commit d50ce86 (Mei 2026)
punya src/media/screenShareController.ts + vendor @dank074/discord-video-stream,
hilang saat rombak monolith -> microservices. Interface ScreenShareController
masih ada di mediaTypes.ts tapi implementasinya tidak.

Restore:
- dep @dank074/discord-video-stream@6.0.0 (npm, dibangun untuk
  discord.js-selfbot-v13 — cocok dengan stack gateway)
- mediaSource.getDirectVideoUrl (yt-dlp --get-url bestvideo+bestaudio)
- screenShareController.ts (BARU): Streamer(client) + prepareStream H264
  720p30 + playStream go-live; owner check via discordPlayer
- media.handler: mode:'screen' di media:queue -> screen path; status
  expose activeMode; stop matiin screen
- FE: tombol Screen di MusicPlayer + hook useMediaQueue({url, mode})

Verifikasi: gateway tsc PASS, FE tsc PASS, biome 0 error, next build PASS.
Nix build pending (dep native @lng2004/node-datachannel butuh pnpm rebuild).
2026-08-01 11:36:40 +07:00
asepharyana 189ab1c1f6 feat(fe): real mic capture for voice transmit (kirim suara)
Sebelumnya tombol Live/Muted cuma kirim voice:transmit:start/stop ke
gateway — TIDAK ADA audio yang dikirim (0 getUserMedia/AudioContext di
frontend). MicControl cuma toggle state kosong.

- lib/audio/mic-transmit.ts (BARU): getUserMedia → AudioContext 48kHz →
  AudioWorklet (downsample 24kHz mono s16le + volume + chunk 20ms) →
  frame 'PCM\0' + Int16LE → ws.sendBinary. Worklet inline via Blob URL
  (aman untuk static export).
- useMicTransmit(ws): aktif = start capture + voice:transmit:start;
  nonaktif = stop capture + voice:transmit:stop; setVolume untuk slider.
- voice page: volume slider sekarang beneran ngatur gain mic; disconnect
  ikut matiin mic.

Verifikasi: tsc PASS, biome 0 error, next build PASS. Test mic butuh
real device (headless browser tidak punya mic) — protokol: connect voice
→ Live → ngomong → orang di channel denger.
2026-08-01 11:26:18 +07:00
asepharyana 9d60f00934 fix(infra): add ffmpeg + yt-dlp to gmw-discord-gateway runtime
Root cause voice tidak berfungsi di produksi: ffmpeg & yt-dlp cuma ada
di devShell, bukan di package discord-gateway. Bukti dari log gateway:
'FFmpeg/avconv not found!' saat voice:transmit:start (mic -> Discord),
yang juga mematikan music playback (StreamType.Arbitrary butuh ffmpeg)
dan segment muxing rekaman.

- buildInputs: pkgs.ffmpeg-headless + pkgs.yt-dlp
- wrapper export PATH ke keduanya sebelum exec node
Verifikasi: nix build PASS; closure berisi ffmpeg-8.1.2 + yt-dlp-2026.07.04;
wrapper PATH mengarah ke keduanya; ffmpeg/yt-dlp jalan.
2026-08-01 11:22:20 +07:00
15 changed files with 5697 additions and 41 deletions
+46 -1
View File
@@ -11,6 +11,15 @@
let
pkgs = import nixpkgs { inherit system; };
# OpenSSL headers (.dev output) + STATIC libs (pkgsStatic.openssl.out —
# node-datachannel's CMakeLists sets OPENSSL_USE_STATIC_LIBS=TRUE, and
# the default `pkgs.openssl` resolves to `bin` which has no lib/) merged
# into one tree so FindOpenSSL resolves both via OPENSSL_ROOT_DIR.
opensslDevEnv = pkgs.symlinkJoin {
name = "openssl-dev-env";
paths = [ pkgs.pkgsStatic.openssl.out pkgs.openssl.dev ];
};
# ---- Shared build tools ----
nodejs = pkgs.nodejs_22;
pnpm = pkgs.pnpm.override { nodejs = nodejs; };
@@ -105,14 +114,49 @@ WRAPPER
nativeBuildInputs = [
nodejs pnpm
pkgs.python3 pkgs.gnumake pkgs.gcc
pkgs.python3 pkgs.gnumake pkgs.gcc pkgs.cmake
pkgs.rustc pkgs.cargo
pkgs.pkg-config
pkgs.openssl
pkgs.openssl.dev
pkgs.git # libdatachannel FetchContent clones from GitHub
pkgs.cacert
];
# Runtime tools for the voice pipeline: ffmpeg (mic transmit encode,
# music stream decode, segment muxing) and yt-dlp (YouTube/Spotify/
# search media resolution). Must be on PATH inside the wrapper below.
buildInputs = [ pkgs.ffmpeg-headless pkgs.yt-dlp ];
# cmake is only needed for node-datachannel's postinstall build —
# do NOT let stdenv run its own cmake configure phase on the source.
dontUseCmakeConfigure = true;
buildPhase = pnpmInstall + ''
echo "=== Building native voice deps ==="
# pnpm rebuild aborts on the first failing package and runs scripts
# from the wrong cwd build each native dep explicitly with its own
# install script. Each failure is tolerated (|| true); the packages
# that matter (opus, datachannel, node-av) are verified at runtime.
for pkg in \
node_modules/.pnpm/@discordjs+opus@*/node_modules/@discordjs/opus \
node_modules/.pnpm/@lng2004+node-datachannel@*/node_modules/@lng2004/node-datachannel \
node_modules/.pnpm/zeromq@*/node_modules/zeromq
do
if [ -d "$pkg" ]; then
echo "--- native build: $pkg ---"
(cd "$pkg" && npm run install 2>&1 || true)
# node-datachannel's `prebuild -r napi` CLI is broken (TypeError:
# expected first argument to be an array) the install fallback
# populates devDeps incl. cmake-js; build directly via cmake-js.
if [ "$(basename "$pkg")" = "node-datachannel" ]; then
echo "--- datachannel cmake-js compile ---"
# Nix splits OpenSSL headers/libs across outputs merge them
# (opensslDevEnv) so FindOpenSSL finds both include + libcrypto.
(cd "$pkg" && OPENSSL_ROOT_DIR="${opensslDevEnv}" npm run compile 2>&1 || true)
fi
fi
done
echo "=== Compiling TypeScript ==="
npx tsc 2>&1
echo "=== Fixing @/ path aliases to relative paths ==="
@@ -154,6 +198,7 @@ WRAPPER
cat > $out/bin/gmw-discord-gateway << WRAPPER
#!${pkgs.runtimeShell}
cd $out/lib/gmw-discord-gateway
export PATH=${pkgs.ffmpeg-headless}/bin:${pkgs.yt-dlp}/bin:\$PATH
exec ${nodejs}/bin/node dist/index.js
WRAPPER
chmod +x $out/bin/gmw-discord-gateway
+2 -7
View File
@@ -10,13 +10,9 @@
"@lng2004/node-datachannel",
"esbuild",
"node-av",
"node-crc",
"sharp",
"zeromq"
],
"patchedDependencies": {
"node-crc@4.0.0": "./patches/node-crc@4.0.0.patch"
}
]
},
"scripts": {
"dev": "tsx watch src/index.ts",
@@ -28,19 +24,18 @@
"test": "vitest run"
},
"dependencies": {
"@dank074/discord-video-stream": "6.0.0",
"@discordjs/opus": "^0.10.0",
"@discordjs/voice": "^0.19.2",
"@snazzah/davey": "^0.1.11",
"axios": "^1.16.1",
"discord.js-selfbot-v13": "^3.7.1",
"@dank074/discord-video-stream": "latest",
"dotenv": "^17.4.2",
"drizzle-orm": "^0.45.2",
"imghash": "^1.1.4",
"ioredis": "^5.11.0",
"libsodium-wrappers": "^0.8.4",
"lru-cache": "^11.5.1",
"node-crc": "^4.0.0",
"openai": "^6.38.0",
"opusscript": "^0.0.8",
"p-limit": "^7.3.0",
@@ -1,13 +0,0 @@
diff --git a/Cargo.toml b/Cargo.toml
index a967508960d8b6b686b23401ea14333b858a675c..d0282a30ee931563a65d774ed783c83d6d2fdd5e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,7 +3,7 @@ name = "node-crc"
version = "4.0.0"
authors = ["Magic Len <len@magiclen.org>"]
edition = "2021"
-rust-version = "1.65"
+rust-version = "1.77.0"
repository = "https://github.com/magiclen/node-crc"
homepage = "https://magiclen.org/node-js-crc/"
keywords = ["nodejs", "crc8", "crc16", "crc32", "crc64"]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,18 @@
allowBuilds:
"@discordjs/opus": true
"@lng2004/node-datachannel": true
esbuild: true
node-av: true
zeromq: true
# pnpm 11 requires build-script approvals here (the legacy `pnpm` field in
# package.json is ignored). Native voice deps need their postinstall build.
# NOTE: sharp sengaja TIDAK ada — binary-nya dari @img/sharp-linux-x64
# (prebuilt), install script-nya cuma validasi dan gagal di Nix sandbox.
# Kalau script sharp dijalankan pnpm rebuild abort sebelum opus/datachannel
# kebangun. node-crc dihapus dari deps (tidak pernah di-import).
onlyBuiltDependencies:
- "@discordjs/opus"
- "@lng2004/node-datachannel"
- esbuild
- node-av
- zeromq
@@ -82,7 +82,7 @@ export class CommandHandler {
// Create domain-specific handlers with their dependencies
this.voiceHandler = new VoiceHandler(client, voiceController);
this.mediaHandler = new MediaHandler();
this.mediaHandler = new MediaHandler(client, () => voiceController.getStatus());
this.guildHandler = new GuildHandler(client);
this.moderationHandler = new ModerationHandler(client);
@@ -1,4 +1,5 @@
import { randomUUID } from "node:crypto";
import type { Client } from "discord.js-selfbot-v13";
import type { CommandMessage, CommandReply } from "../../shared/index.js";
import { createChildLogger } from "../../shared/logger/index.js";
import { StreamType } from "@discordjs/voice";
@@ -11,6 +12,10 @@ import type {
MediaQueueItem,
} from "../voice-recording/mediaTypes.js";
import { discordPlayer } from "../voice-recording/player.js";
import {
ScreenShareController,
type ScreenShareVoiceStatus,
} from "../voice-recording/screenShareController.js";
// ---------------------------------------------------------------------------
// Types
@@ -27,6 +32,7 @@ interface MediaStatusItem {
export interface MediaStatusPayload {
playing: boolean;
activeMode: MediaMode | null;
musicVolume: number;
current: MediaStatusItem | null;
queue: MediaStatusItem[];
@@ -58,6 +64,7 @@ function buildStatusPayload(): MediaStatusPayload {
return {
playing:
currentTrackItem !== null && discordPlayer.getStatus() === "playing",
activeMode: currentTrackItem?.mode ?? null,
musicVolume: discordPlayer.getMusicVolume(),
current: currentTrackItem ? mapToStatusItem(currentTrackItem) : null,
queue: mediaQueue.map(mapToStatusItem),
@@ -70,8 +77,17 @@ function buildStatusPayload(): MediaStatusPayload {
export class MediaHandler {
private logger = createChildLogger("media-handler");
private screenController: ScreenShareController | null = null;
private screenPlayback: { stop(): void } | null = null;
constructor() {
constructor(
private readonly client: Client | null = null,
private readonly getVoiceStatus: () => ScreenShareVoiceStatus = () => ({
connected: false,
activeGuildId: null,
activeChannelId: null,
}),
) {
// Register auto-advance on natural track end
discordPlayer.onIdle(() => {
this.advanceQueue().catch((err) => {
@@ -109,6 +125,50 @@ export class MediaHandler {
};
}
// Screen share (GoLive) path — bypasses the audio queue entirely.
if (mode === "screen") {
try {
if (!this.client) {
return {
id: cmd.id,
success: false,
data: null,
error: "Gateway client not initialized",
};
}
if (!this.screenController) {
this.screenController = new ScreenShareController(
this.client,
this.getVoiceStatus,
);
}
const playback = await this.screenController.start(url);
this.screenPlayback = playback;
currentTrackItem = {
id: randomUUID(),
source: url,
title: url,
kind: "url",
mode: "screen",
requestedBy,
addedAt: Date.now(),
status: "playing",
};
playback.done.finally(() => {
this.screenPlayback = null;
if (currentTrackItem?.mode === "screen") {
currentTrackItem = null;
}
});
this.logger.info({ url }, "Screen share started");
return { id: cmd.id, success: true, data: buildStatusPayload() };
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
this.logger.error({ error: message }, "Screen share failed to start");
return { id: cmd.id, success: false, data: null, error: message };
}
}
// Lightweight metadata fetch for display — the full resolve happens in playNext
let title: string = url;
let duration: number | undefined;
@@ -174,6 +234,12 @@ export class MediaHandler {
}
async handleMediaStop(cmd: CommandMessage): Promise<CommandReply<unknown>> {
// Stop screen share if active — playback.done.finally clears the item.
this.screenPlayback?.stop();
this.screenPlayback = null;
if (currentTrackItem?.mode === "screen") {
currentTrackItem = null;
}
discordPlayer.stop("music");
currentTrackItem = null;
mediaQueue.length = 0; // Clear entire queue
@@ -282,6 +282,87 @@ export function resolveMediaUrl(
});
}
/**
* Resolve a media URL to a directly playable video URL (for screen share /
* GoLive streaming). Uses yt-dlp `--get-url` with bestvideo+bestaudio.
*
* @throws If yt-dlp is not installed or the process exits with a non-zero code.
*/
export function getDirectVideoUrl(url: string): Promise<string> {
return new Promise<string>((resolve, reject) => {
const args = [
url,
"--get-url",
"--format",
"bestvideo[protocol^=http]+bestaudio[protocol^=http]/best[protocol^=http]/best",
"--no-playlist",
"--no-warnings",
"--quiet",
];
logger.info({ url }, "Spawning yt-dlp for direct video URL");
const proc = spawn("yt-dlp", args, {
stdio: ["pipe", "pipe", "pipe"],
});
activeProcesses.add(proc);
let stdoutBuf = "";
let stderrBuf = "";
const MAX_STDERR = 4096;
const MAX_STDOUT = 1_048_576;
if (proc.stdout) {
proc.stdout.on("data", (chunk: Buffer) => {
if (stdoutBuf.length < MAX_STDOUT) {
stdoutBuf += chunk
.toString("utf8")
.slice(0, MAX_STDOUT - stdoutBuf.length);
}
});
}
if (proc.stderr) {
proc.stderr.on("data", (chunk: Buffer) => {
if (stderrBuf.length < MAX_STDERR) {
stderrBuf += chunk
.toString("utf8")
.slice(0, MAX_STDERR - stderrBuf.length);
}
});
}
proc.on("error", (err: NodeJS.ErrnoException) => {
activeProcesses.delete(proc);
if (err.code === "ENOENT") {
reject(buildNotInstalledError());
} else {
reject(new Error(`yt-dlp failed to start: ${err.message}`));
}
});
proc.on("close", (code) => {
activeProcesses.delete(proc);
if (code !== 0) {
const detail = stderrBuf.trim() ? `: ${stderrBuf.trim()}` : "";
reject(
new Error(`yt-dlp direct URL resolution exited with code ${code}${detail}`),
);
return;
}
const firstLine = stdoutBuf.trim().split("\n")[0];
if (!firstLine) {
reject(new Error("yt-dlp returned no direct video URL"));
return;
}
resolve(firstLine);
});
});
}
/**
* Extract metadata (title, duration, thumbnail) from a media URL
* without downloading the audio stream.
@@ -0,0 +1,105 @@
import {
Encoders,
prepareStream,
playStream,
Streamer,
Utils,
} from "@dank074/discord-video-stream";
import type { Client } from "discord.js-selfbot-v13";
import { createChildLogger } from "@/shared/logger/index";
import type { ScreenSharePlayback } from "./mediaTypes.js";
import { getDirectVideoUrl } from "./mediaSource.js";
import { discordPlayer } from "./player.js";
const logger = createChildLogger("screen-share");
export interface ScreenShareVoiceStatus {
connected: boolean;
activeGuildId: string | null;
activeChannelId: string | null;
}
/**
* Discord Go Live (screenshare) via @dank074/discord-video-stream.
*
* Pipeline:
* URL (YouTube, dll.) → yt-dlp direct video URL → ffmpeg (H264 720p30) →
* playStream({ type: "go-live" }) → Discord voice channel as Go Live.
*
* Restored from the pre-microservices implementation (commit d50ce86,
* src/media/screenShareController.ts) — the interface survived in
* mediaTypes.ts but the implementation was lost during the split.
*/
export class ScreenShareController {
private logger = createChildLogger("screen-share");
private streamer: Streamer | null = null;
private active: ScreenSharePlayback | null = null;
constructor(
private readonly client: Client,
private readonly getVoiceStatus: () => ScreenShareVoiceStatus,
) {}
isActive(): boolean {
return this.active !== null;
}
async start(source: string): Promise<ScreenSharePlayback> {
const status = this.getVoiceStatus();
if (
!status.connected ||
!status.activeGuildId ||
!status.activeChannelId
) {
throw new Error("Connect to a voice channel before sharing screen");
}
if (this.active || discordPlayer.getOwner() !== "none") {
throw new Error("Another media mode is active");
}
try {
const directUrl = await getDirectVideoUrl(source);
if (!this.streamer) {
this.streamer = new Streamer(this.client);
}
const { command, output } = prepareStream(directUrl, {
encoder: Encoders.software({ x264: { preset: "superfast" } }),
width: 1280,
height: 720,
frameRate: 30,
bitrateVideo: 2500,
bitrateVideoMax: 4000,
includeAudio: true,
videoCodec: Utils.normalizeVideoCodec("H264"),
});
let stopped = false;
const done = playStream(output, this.streamer, {
type: "go-live",
}).finally(() => {
this.active = null;
});
const controller = this;
this.active = {
done,
stop: () => {
if (stopped) return;
stopped = true;
command.kill("SIGTERM");
controller.active = null;
},
};
logger.info({ source }, "Screen share started");
return this.active;
} catch (error) {
this.active = null;
const message = error instanceof Error ? error.message : String(error);
logger.error({ error: message, source }, "Screen stream failed");
throw error;
}
}
}
@@ -28,7 +28,7 @@ export default function VoicePage() {
const { speakers, subscribe } = useSpeakers();
const connectMut = useVoiceConnect();
const disconnectMut = useVoiceDisconnect();
const micMut = useMicTransmit();
const micMut = useMicTransmit(ws);
const [selectedChannel, setSelectedChannel] = useState("");
const [micActive, setMicActive] = useState(false);
const [volume, setVolume] = useState(75);
@@ -41,16 +41,33 @@ export default function VoicePage() {
const handleMicToggle = useCallback(
async (checked: boolean) => {
setMicActive(checked);
try {
await micMut.mutateAsync(checked);
} catch {
setMicActive(!checked);
if (checked) {
try {
await micMut.mutateAsync(true);
setMicActive(true);
} catch {
setMicActive(false);
}
} else {
setMicActive(false);
try {
await micMut.mutateAsync(false);
} catch {
// Stop already tore down the local transmitter — ignore remote errors
}
}
},
[micMut],
);
const handleVolumeChange = useCallback(
(v: number) => {
setVolume(v);
micMut.setVolume(v);
},
[micMut],
);
const handleGuildChange = useCallback((guildId: string | null) => {
if (!guildId) {
setSelectedGuild("");
@@ -89,7 +106,13 @@ export default function VoicePage() {
channelId: selectedChannel,
})
}
onDisconnect={() => disconnectMut.mutate(undefined)}
onDisconnect={() => {
if (micActive) {
setMicActive(false);
void micMut.mutateAsync(false).catch(() => {});
}
disconnectMut.mutate(undefined);
}}
connecting={connectMut.isPending}
/>
@@ -101,7 +124,7 @@ export default function VoicePage() {
active={micActive}
onToggle={handleMicToggle}
volume={volume}
onVolumeChange={setVolume}
onVolumeChange={handleVolumeChange}
/>
</div>
)}
@@ -29,15 +29,19 @@ export function MusicPlayer({ ws }: MusicPlayerProps) {
const stopMut = useMediaStop();
const volumeMut = useMediaVolume();
const [queueUrl, setQueueUrl] = useState("");
const [screenMode, setScreenMode] = useState(false);
// Sync WS media_state into the query cache
useMediaWsSync(ws);
const handleQueue = useCallback(() => {
if (!queueUrl.trim()) return;
queueMut.mutate(queueUrl.trim());
queueMut.mutate({
url: queueUrl.trim(),
mode: screenMode ? "screen" : "music",
});
setQueueUrl("");
}, [queueUrl, queueMut]);
}, [queueUrl, queueMut, screenMode]);
const handleVolume = useCallback(
(value: number | readonly number[]) => {
@@ -64,6 +68,15 @@ export function MusicPlayer({ ws }: MusicPlayerProps) {
onKeyDown={(e) => e.key === "Enter" && handleQueue()}
className="flex-1 h-9"
/>
<Button
variant={screenMode ? "default" : "secondary"}
size="sm"
onClick={() => setScreenMode((v) => !v)}
title="Queue as Discord GoLive screenshare instead of audio playback"
className="h-9"
>
Screen
</Button>
<Button
onClick={handleQueue}
disabled={!queueUrl.trim() || queueMut.isPending}
@@ -73,6 +86,14 @@ export function MusicPlayer({ ws }: MusicPlayerProps) {
</Button>
</div>
{mediaState?.activeMode && (
<p className="text-[10px] font-mono text-primary/80 uppercase tracking-wider">
{mediaState.activeMode === "screen"
? "Screen share active"
: "Music playing"}
</p>
)}
{mediaState?.current ? (
<div className="rounded-lg bg-gradient-to-br from-primary/5 to-primary/[0.02] border border-primary/10 p-4 space-y-2">
<p className="text-xs text-muted-foreground font-medium uppercase tracking-wider flex items-center gap-1.5">
+4 -1
View File
@@ -24,7 +24,10 @@ function useMediaAction<TArgs>(fn: (args: TArgs) => Promise<MediaState>) {
}
export function useMediaQueue() {
return useMediaAction((url: string) => mediaApi.queue(url, "music"));
return useMediaAction(
(input: { url: string; mode?: "music" | "screen" }) =>
mediaApi.queue(input.url, input.mode ?? "music"),
);
}
export function useMediaSkip() {
+25 -7
View File
@@ -1,7 +1,8 @@
import { useCallback, useState } from "react";
import { useCallback, useRef, useState } from "react";
import useSWR, { useSWRConfig } from "swr";
import { useAction } from "@/hooks/use-action";
import { voiceApi } from "@/lib/api";
import { MicTransmitter } from "@/lib/audio/mic-transmit";
import type { ActiveSpeaker, Channel, VoiceStatus } from "@/lib/types";
import type { WsHook } from "@/lib/ws-hook";
@@ -65,10 +66,27 @@ export function useVoiceDisconnect() {
return useAction(() => voiceApi.disconnect(), { onSuccess: invalidate });
}
export function useMicTransmit() {
return useAction((active: boolean) =>
voiceApi.sendCommand(
active ? "voice:transmit:start" : "voice:transmit:stop",
),
);
export function useMicTransmit(ws: {
sendBinary: (data: ArrayBufferLike) => void;
}) {
const transmitterRef = useRef<MicTransmitter | null>(null);
const action = useAction(async (active: boolean) => {
if (active) {
const transmitter = new MicTransmitter((frame) => ws.sendBinary(frame));
transmitterRef.current = transmitter;
await transmitter.start();
await voiceApi.sendCommand("voice:transmit:start");
} else {
transmitterRef.current?.stop();
transmitterRef.current = null;
await voiceApi.sendCommand("voice:transmit:stop");
}
});
const setVolume = useCallback((volume: number) => {
transmitterRef.current?.setVolume(volume / 100);
}, []);
return { ...action, setVolume };
}
@@ -0,0 +1,147 @@
/**
* Browser mic → Discord voice transmit.
*
* Pipeline: getUserMedia → AudioContext (48kHz) → AudioWorklet (downsample to
* 24kHz mono s16le, apply volume, chunk 20ms) → binary WS frames.
*
* The backend expects each binary frame to start with a 4-byte magic "PCM\0"
* followed by raw Int16LE PCM; it base64s the payload and publishes to Redis,
* where the gateway's VoiceTransmitter feeds it into FFmpeg (24kHz mono s16le
* → OggOpus) and plays it in the voice channel.
*/
const PCM_MAGIC = new Uint8Array([0x50, 0x43, 0x4d, 0x00]); // "PCM\0"
const TARGET_RATE = 24000;
const CHUNK_MS = 20;
// Inline AudioWorklet processor (Blob URL — works with Next static export,
// no asset pipeline needed).
const WORKLET_SRC = `
class PcmDownsampler extends AudioWorkletProcessor {
constructor(options) {
super();
const opts = options.processorOptions || {};
this.targetRate = opts.targetRate || 24000;
this.ratio = sampleRate / this.targetRate;
this.chunkSamples = Math.floor((this.targetRate * (opts.chunkMs || 20)) / 1000);
this.phase = 0;
this.buffer = new Int16Array(this.chunkSamples);
this.bufferLen = 0;
this.volume = typeof opts.volume === 'number' ? opts.volume : 1;
this.port.onmessage = (e) => {
if (e.data && e.data.type === 'volume') this.volume = e.data.value;
};
}
process(inputs) {
const input = inputs[0];
if (!input || input.length === 0) return true;
// Mixdown: average available channels
const chans = input.filter((c) => c && c.length > 0);
if (chans.length === 0) return true;
const len = chans[0].length;
for (let i = 0; i < len; i++) {
let s = 0;
for (let c = 0; c < chans.length; c++) s += chans[c][i];
s /= chans.length;
this.phase += 1;
if (this.phase >= this.ratio) {
this.phase -= this.ratio;
const v = Math.max(-1, Math.min(1, s * this.volume));
this.buffer[this.bufferLen++] = (v * 32767) | 0;
if (this.bufferLen >= this.chunkSamples) {
const out = new Int16Array(this.buffer);
this.port.postMessage(out.buffer, [out.buffer]);
this.buffer = new Int16Array(this.chunkSamples);
this.bufferLen = 0;
}
}
}
return true;
}
}
registerProcessor('pcm-downsampler', PcmDownsampler);
`;
export class MicTransmitter {
private ctx: AudioContext | null = null;
private stream: MediaStream | null = null;
private node: AudioWorkletNode | null = null;
private active = false;
private volume = 1;
constructor(private readonly onChunk: (frame: ArrayBuffer) => void) {}
get isActive(): boolean {
return this.active;
}
async start(volume = 1): Promise<void> {
if (this.active) return;
this.volume = volume;
if (!navigator.mediaDevices?.getUserMedia) {
throw new Error("getUserMedia is not available (insecure context?)");
}
this.stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
this.ctx = new AudioContext({ sampleRate: 48000 });
const blob = new Blob([WORKLET_SRC], { type: "application/javascript" });
const workletUrl = URL.createObjectURL(blob);
try {
await this.ctx.audioWorklet.addModule(workletUrl);
} finally {
URL.revokeObjectURL(workletUrl);
}
const source = this.ctx.createMediaStreamSource(this.stream);
this.node = new AudioWorkletNode(this.ctx, "pcm-downsampler", {
processorOptions: {
targetRate: TARGET_RATE,
chunkMs: CHUNK_MS,
volume: this.volume,
},
});
this.node.port.onmessage = (e: MessageEvent<ArrayBuffer>) => {
if (!this.active || !(e.data instanceof ArrayBuffer)) return;
const frame = new Uint8Array(PCM_MAGIC.length + e.data.byteLength);
frame.set(PCM_MAGIC, 0);
frame.set(new Uint8Array(e.data), PCM_MAGIC.length);
this.onChunk(frame.buffer);
};
source.connect(this.node);
// Keep the graph alive with an inaudible tail (silent gain) so the
// worklet keeps pulling mic data without audible feedback.
const silent = this.ctx.createGain();
silent.gain.value = 0;
this.node.connect(silent);
silent.connect(this.ctx.destination);
this.active = true;
}
setVolume(volume: number): void {
this.volume = volume;
this.node?.port.postMessage({ type: "volume", value: volume });
}
stop(): void {
this.active = false;
this.node?.port.postMessage({ type: "volume", value: 0 });
this.node?.disconnect();
this.node = null;
this.stream?.getTracks().forEach((t) => t.stop());
this.stream = null;
this.ctx?.close().catch(() => {});
this.ctx = null;
}
}
+1
View File
@@ -11,6 +11,7 @@ export interface MediaItem {
export interface MediaState {
playing: boolean;
activeMode?: MediaMode | null;
musicVolume: number;
current: MediaItem | null;
queue: MediaItem[];