feat(voice): implement full voice transmit (Browser → Discord)
Frontend (useAudioTransmit.ts): - Capture microphone via getUserMedia - Convert Float32 to Int16 PCM at 24kHz mono - Base64 encode and send as JSON via WebSocket - Send voice:transmit:start/stop commands on start/stop Backend (ws/server.ts): - Handle voice_transmit messages from browser - Forward PCM data to Redis channel backend:voice:transmit - Handle voice_command messages and forward to backend:command - discord-gateway VoiceTransmitter receives and plays to voice channel Flow: Browser Mic → getUserMedia → ScriptProcessor → Int16 PCM → base64 → WebSocket JSON → Backend → Redis → VoiceTransmitter → Discord Voice Channel Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
2e9e9c1a06
commit
314d2baec4
@@ -12,6 +12,14 @@ export function useAudioTransmit(socketRef: {
|
||||
const processorRef = useRef<ScriptProcessorNode | null>(null);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
// Send voice:transmit:stop command to backend
|
||||
if (socketRef.current && socketRef.current.readyState === WebSocket.OPEN) {
|
||||
socketRef.current.send(JSON.stringify({
|
||||
type: 'voice_command',
|
||||
command: 'voice:transmit:stop'
|
||||
}));
|
||||
}
|
||||
|
||||
setIsStreaming(false);
|
||||
if (processorRef.current) {
|
||||
processorRef.current.disconnect();
|
||||
@@ -25,9 +33,17 @@ export function useAudioTransmit(socketRef: {
|
||||
for (const track of streamRef.current.getTracks()) track.stop();
|
||||
streamRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
}, [socketRef]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
// Send voice:transmit:start command to backend
|
||||
if (socketRef.current && socketRef.current.readyState === WebSocket.OPEN) {
|
||||
socketRef.current.send(JSON.stringify({
|
||||
type: 'voice_command',
|
||||
command: 'voice:transmit:start'
|
||||
}));
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
setIsStreaming(true);
|
||||
@@ -49,8 +65,20 @@ export function useAudioTransmit(socketRef: {
|
||||
const pcmData = new Int16Array(inputData.length);
|
||||
for (let i = 0; i < inputData.length; i++)
|
||||
pcmData[i] = Math.max(-1, Math.min(1, inputData[i])) * 32767;
|
||||
// BUG 2 FIX: slice() to create independent copy of the ArrayBuffer
|
||||
socketRef.current.send(pcmData.buffer.slice(0));
|
||||
|
||||
// Convert to base64
|
||||
const bytes = new Uint8Array(pcmData.buffer);
|
||||
let binary = '';
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
const base64 = btoa(binary);
|
||||
|
||||
// Send as JSON for backend to forward to Redis
|
||||
socketRef.current.send(JSON.stringify({
|
||||
type: 'voice_transmit',
|
||||
buffer: base64
|
||||
}));
|
||||
};
|
||||
}, [socketRef]);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user