From 189ab1c1f6721f0f96479547b3cccaa2177b68d9 Mon Sep 17 00:00:00 2001 From: asepharyana Date: Sat, 1 Aug 2026 11:26:18 +0700 Subject: [PATCH] feat(fe): real mic capture for voice transmit (kirim suara) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sebelumnya tombol Live/Muted cuma kirim voice:transmit:start/stop ke gateway — TIDAK ADA audio yang dikirim (0 getUserMedia/AudioContext di frontend). MicControl cuma toggle state kosong. - lib/audio/mic-transmit.ts (BARU): getUserMedia → AudioContext 48kHz → AudioWorklet (downsample 24kHz mono s16le + volume + chunk 20ms) → frame 'PCM\0' + Int16LE → ws.sendBinary. Worklet inline via Blob URL (aman untuk static export). - useMicTransmit(ws): aktif = start capture + voice:transmit:start; nonaktif = stop capture + voice:transmit:stop; setVolume untuk slider. - voice page: volume slider sekarang beneran ngatur gain mic; disconnect ikut matiin mic. Verifikasi: tsc PASS, biome 0 error, next build PASS. Test mic butuh real device (headless browser tidak punya mic) — protokol: connect voice → Live → ngomong → orang di channel denger. --- .../src/app/(dashboard)/voice/page.tsx | 39 ++++- services/frontend/src/hooks/use-voice.ts | 32 +++- .../frontend/src/lib/audio/mic-transmit.ts | 147 ++++++++++++++++++ 3 files changed, 203 insertions(+), 15 deletions(-) create mode 100644 services/frontend/src/lib/audio/mic-transmit.ts diff --git a/services/frontend/src/app/(dashboard)/voice/page.tsx b/services/frontend/src/app/(dashboard)/voice/page.tsx index 9b46e1a..7306154 100644 --- a/services/frontend/src/app/(dashboard)/voice/page.tsx +++ b/services/frontend/src/app/(dashboard)/voice/page.tsx @@ -28,7 +28,7 @@ export default function VoicePage() { const { speakers, subscribe } = useSpeakers(); const connectMut = useVoiceConnect(); const disconnectMut = useVoiceDisconnect(); - const micMut = useMicTransmit(); + const micMut = useMicTransmit(ws); const [selectedChannel, setSelectedChannel] = useState(""); const [micActive, setMicActive] = useState(false); const [volume, setVolume] = useState(75); @@ -41,16 +41,33 @@ export default function VoicePage() { const handleMicToggle = useCallback( async (checked: boolean) => { - setMicActive(checked); - try { - await micMut.mutateAsync(checked); - } catch { - setMicActive(!checked); + if (checked) { + try { + await micMut.mutateAsync(true); + setMicActive(true); + } catch { + setMicActive(false); + } + } else { + setMicActive(false); + try { + await micMut.mutateAsync(false); + } catch { + // Stop already tore down the local transmitter — ignore remote errors + } } }, [micMut], ); + const handleVolumeChange = useCallback( + (v: number) => { + setVolume(v); + micMut.setVolume(v); + }, + [micMut], + ); + const handleGuildChange = useCallback((guildId: string | null) => { if (!guildId) { setSelectedGuild(""); @@ -89,7 +106,13 @@ export default function VoicePage() { channelId: selectedChannel, }) } - onDisconnect={() => disconnectMut.mutate(undefined)} + onDisconnect={() => { + if (micActive) { + setMicActive(false); + void micMut.mutateAsync(false).catch(() => {}); + } + disconnectMut.mutate(undefined); + }} connecting={connectMut.isPending} /> @@ -101,7 +124,7 @@ export default function VoicePage() { active={micActive} onToggle={handleMicToggle} volume={volume} - onVolumeChange={setVolume} + onVolumeChange={handleVolumeChange} /> )} diff --git a/services/frontend/src/hooks/use-voice.ts b/services/frontend/src/hooks/use-voice.ts index d145c07..4ee57ac 100644 --- a/services/frontend/src/hooks/use-voice.ts +++ b/services/frontend/src/hooks/use-voice.ts @@ -1,7 +1,8 @@ -import { useCallback, useState } from "react"; +import { useCallback, useRef, useState } from "react"; import useSWR, { useSWRConfig } from "swr"; import { useAction } from "@/hooks/use-action"; import { voiceApi } from "@/lib/api"; +import { MicTransmitter } from "@/lib/audio/mic-transmit"; import type { ActiveSpeaker, Channel, VoiceStatus } from "@/lib/types"; import type { WsHook } from "@/lib/ws-hook"; @@ -65,10 +66,27 @@ export function useVoiceDisconnect() { return useAction(() => voiceApi.disconnect(), { onSuccess: invalidate }); } -export function useMicTransmit() { - return useAction((active: boolean) => - voiceApi.sendCommand( - active ? "voice:transmit:start" : "voice:transmit:stop", - ), - ); +export function useMicTransmit(ws: { + sendBinary: (data: ArrayBufferLike) => void; +}) { + const transmitterRef = useRef(null); + + const action = useAction(async (active: boolean) => { + if (active) { + const transmitter = new MicTransmitter((frame) => ws.sendBinary(frame)); + transmitterRef.current = transmitter; + await transmitter.start(); + await voiceApi.sendCommand("voice:transmit:start"); + } else { + transmitterRef.current?.stop(); + transmitterRef.current = null; + await voiceApi.sendCommand("voice:transmit:stop"); + } + }); + + const setVolume = useCallback((volume: number) => { + transmitterRef.current?.setVolume(volume / 100); + }, []); + + return { ...action, setVolume }; } diff --git a/services/frontend/src/lib/audio/mic-transmit.ts b/services/frontend/src/lib/audio/mic-transmit.ts new file mode 100644 index 0000000..c06b47e --- /dev/null +++ b/services/frontend/src/lib/audio/mic-transmit.ts @@ -0,0 +1,147 @@ +/** + * Browser mic → Discord voice transmit. + * + * Pipeline: getUserMedia → AudioContext (48kHz) → AudioWorklet (downsample to + * 24kHz mono s16le, apply volume, chunk 20ms) → binary WS frames. + * + * The backend expects each binary frame to start with a 4-byte magic "PCM\0" + * followed by raw Int16LE PCM; it base64s the payload and publishes to Redis, + * where the gateway's VoiceTransmitter feeds it into FFmpeg (24kHz mono s16le + * → OggOpus) and plays it in the voice channel. + */ + +const PCM_MAGIC = new Uint8Array([0x50, 0x43, 0x4d, 0x00]); // "PCM\0" +const TARGET_RATE = 24000; +const CHUNK_MS = 20; + +// Inline AudioWorklet processor (Blob URL — works with Next static export, +// no asset pipeline needed). +const WORKLET_SRC = ` +class PcmDownsampler extends AudioWorkletProcessor { + constructor(options) { + super(); + const opts = options.processorOptions || {}; + this.targetRate = opts.targetRate || 24000; + this.ratio = sampleRate / this.targetRate; + this.chunkSamples = Math.floor((this.targetRate * (opts.chunkMs || 20)) / 1000); + this.phase = 0; + this.buffer = new Int16Array(this.chunkSamples); + this.bufferLen = 0; + this.volume = typeof opts.volume === 'number' ? opts.volume : 1; + this.port.onmessage = (e) => { + if (e.data && e.data.type === 'volume') this.volume = e.data.value; + }; + } + process(inputs) { + const input = inputs[0]; + if (!input || input.length === 0) return true; + // Mixdown: average available channels + const chans = input.filter((c) => c && c.length > 0); + if (chans.length === 0) return true; + const len = chans[0].length; + for (let i = 0; i < len; i++) { + let s = 0; + for (let c = 0; c < chans.length; c++) s += chans[c][i]; + s /= chans.length; + this.phase += 1; + if (this.phase >= this.ratio) { + this.phase -= this.ratio; + const v = Math.max(-1, Math.min(1, s * this.volume)); + this.buffer[this.bufferLen++] = (v * 32767) | 0; + if (this.bufferLen >= this.chunkSamples) { + const out = new Int16Array(this.buffer); + this.port.postMessage(out.buffer, [out.buffer]); + this.buffer = new Int16Array(this.chunkSamples); + this.bufferLen = 0; + } + } + } + return true; + } +} +registerProcessor('pcm-downsampler', PcmDownsampler); +`; + +export class MicTransmitter { + private ctx: AudioContext | null = null; + private stream: MediaStream | null = null; + private node: AudioWorkletNode | null = null; + private active = false; + private volume = 1; + + constructor(private readonly onChunk: (frame: ArrayBuffer) => void) {} + + get isActive(): boolean { + return this.active; + } + + async start(volume = 1): Promise { + if (this.active) return; + this.volume = volume; + + if (!navigator.mediaDevices?.getUserMedia) { + throw new Error("getUserMedia is not available (insecure context?)"); + } + + this.stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + + this.ctx = new AudioContext({ sampleRate: 48000 }); + + const blob = new Blob([WORKLET_SRC], { type: "application/javascript" }); + const workletUrl = URL.createObjectURL(blob); + try { + await this.ctx.audioWorklet.addModule(workletUrl); + } finally { + URL.revokeObjectURL(workletUrl); + } + + const source = this.ctx.createMediaStreamSource(this.stream); + this.node = new AudioWorkletNode(this.ctx, "pcm-downsampler", { + processorOptions: { + targetRate: TARGET_RATE, + chunkMs: CHUNK_MS, + volume: this.volume, + }, + }); + + this.node.port.onmessage = (e: MessageEvent) => { + if (!this.active || !(e.data instanceof ArrayBuffer)) return; + const frame = new Uint8Array(PCM_MAGIC.length + e.data.byteLength); + frame.set(PCM_MAGIC, 0); + frame.set(new Uint8Array(e.data), PCM_MAGIC.length); + this.onChunk(frame.buffer); + }; + + source.connect(this.node); + // Keep the graph alive with an inaudible tail (silent gain) so the + // worklet keeps pulling mic data without audible feedback. + const silent = this.ctx.createGain(); + silent.gain.value = 0; + this.node.connect(silent); + silent.connect(this.ctx.destination); + + this.active = true; + } + + setVolume(volume: number): void { + this.volume = volume; + this.node?.port.postMessage({ type: "volume", value: volume }); + } + + stop(): void { + this.active = false; + this.node?.port.postMessage({ type: "volume", value: 0 }); + this.node?.disconnect(); + this.node = null; + this.stream?.getTracks().forEach((t) => t.stop()); + this.stream = null; + this.ctx?.close().catch(() => {}); + this.ctx = null; + } +}