feat(voice): implement full voice transmit (Browser → Discord)
Frontend (useAudioTransmit.ts): - Capture microphone via getUserMedia - Convert Float32 to Int16 PCM at 24kHz mono - Base64 encode and send as JSON via WebSocket - Send voice:transmit:start/stop commands on start/stop Backend (ws/server.ts): - Handle voice_transmit messages from browser - Forward PCM data to Redis channel backend:voice:transmit - Handle voice_command messages and forward to backend:command - discord-gateway VoiceTransmitter receives and plays to voice channel Flow: Browser Mic → getUserMedia → ScriptProcessor → Int16 PCM → base64 → WebSocket JSON → Backend → Redis → VoiceTransmitter → Discord Voice Channel Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
2e9e9c1a06
commit
314d2baec4
@@ -65,13 +65,40 @@ export function createWebSocketServer(server: Server): WebSocketServer {
|
|||||||
);
|
);
|
||||||
|
|
||||||
ws.on("message", (data: Buffer) => {
|
ws.on("message", (data: Buffer) => {
|
||||||
// Binary PCM data received from browser.
|
// Handle JSON messages from browser
|
||||||
// Since backend has no Discord client to relay to, drop it.
|
if (typeof data === 'string' || (Buffer.isBuffer(data) && data.length > 0 && data[0] === 0x7B)) {
|
||||||
if (Buffer.isBuffer(data) && data.length > 0) {
|
try {
|
||||||
logger.debug(
|
const message = JSON.parse(data.toString());
|
||||||
{ bytes: data.length },
|
|
||||||
"Dropping binary PCM (no Discord client)",
|
if (message.type === 'voice_transmit' && message.buffer) {
|
||||||
);
|
// Forward PCM data to Redis for discord-gateway
|
||||||
|
import('../shared/redis/index.js').then(({ getCommandPublisher }) => {
|
||||||
|
const publisher = getCommandPublisher();
|
||||||
|
publisher.publish('backend:voice:transmit', JSON.stringify({
|
||||||
|
type: 'pcm',
|
||||||
|
buffer: message.buffer
|
||||||
|
})).catch((err: Error) => {
|
||||||
|
logger.error({ err }, 'Failed to publish voice transmit to Redis');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
} else if (message.type === 'voice_command' && message.command) {
|
||||||
|
// Forward voice commands to discord-gateway
|
||||||
|
import('../shared/redis/index.js').then(({ getCommandPublisher }) => {
|
||||||
|
const publisher = getCommandPublisher();
|
||||||
|
const commandId = `cmd-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
||||||
|
publisher.publish('backend:command', JSON.stringify({
|
||||||
|
id: commandId,
|
||||||
|
type: message.command,
|
||||||
|
payload: {},
|
||||||
|
replyChannel: `reply:${commandId}`
|
||||||
|
})).catch((err: Error) => {
|
||||||
|
logger.error({ err }, 'Failed to publish voice command to Redis');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.debug({ err }, 'Failed to parse WebSocket message as JSON');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,14 @@ export function useAudioTransmit(socketRef: {
|
|||||||
const processorRef = useRef<ScriptProcessorNode | null>(null);
|
const processorRef = useRef<ScriptProcessorNode | null>(null);
|
||||||
|
|
||||||
const stop = useCallback(() => {
|
const stop = useCallback(() => {
|
||||||
|
// Send voice:transmit:stop command to backend
|
||||||
|
if (socketRef.current && socketRef.current.readyState === WebSocket.OPEN) {
|
||||||
|
socketRef.current.send(JSON.stringify({
|
||||||
|
type: 'voice_command',
|
||||||
|
command: 'voice:transmit:stop'
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
setIsStreaming(false);
|
setIsStreaming(false);
|
||||||
if (processorRef.current) {
|
if (processorRef.current) {
|
||||||
processorRef.current.disconnect();
|
processorRef.current.disconnect();
|
||||||
@@ -25,9 +33,17 @@ export function useAudioTransmit(socketRef: {
|
|||||||
for (const track of streamRef.current.getTracks()) track.stop();
|
for (const track of streamRef.current.getTracks()) track.stop();
|
||||||
streamRef.current = null;
|
streamRef.current = null;
|
||||||
}
|
}
|
||||||
}, []);
|
}, [socketRef]);
|
||||||
|
|
||||||
const start = useCallback(async () => {
|
const start = useCallback(async () => {
|
||||||
|
// Send voice:transmit:start command to backend
|
||||||
|
if (socketRef.current && socketRef.current.readyState === WebSocket.OPEN) {
|
||||||
|
socketRef.current.send(JSON.stringify({
|
||||||
|
type: 'voice_command',
|
||||||
|
command: 'voice:transmit:start'
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||||
streamRef.current = stream;
|
streamRef.current = stream;
|
||||||
setIsStreaming(true);
|
setIsStreaming(true);
|
||||||
@@ -49,8 +65,20 @@ export function useAudioTransmit(socketRef: {
|
|||||||
const pcmData = new Int16Array(inputData.length);
|
const pcmData = new Int16Array(inputData.length);
|
||||||
for (let i = 0; i < inputData.length; i++)
|
for (let i = 0; i < inputData.length; i++)
|
||||||
pcmData[i] = Math.max(-1, Math.min(1, inputData[i])) * 32767;
|
pcmData[i] = Math.max(-1, Math.min(1, inputData[i])) * 32767;
|
||||||
// BUG 2 FIX: slice() to create independent copy of the ArrayBuffer
|
|
||||||
socketRef.current.send(pcmData.buffer.slice(0));
|
// Convert to base64
|
||||||
|
const bytes = new Uint8Array(pcmData.buffer);
|
||||||
|
let binary = '';
|
||||||
|
for (let i = 0; i < bytes.length; i++) {
|
||||||
|
binary += String.fromCharCode(bytes[i]);
|
||||||
|
}
|
||||||
|
const base64 = btoa(binary);
|
||||||
|
|
||||||
|
// Send as JSON for backend to forward to Redis
|
||||||
|
socketRef.current.send(JSON.stringify({
|
||||||
|
type: 'voice_transmit',
|
||||||
|
buffer: base64
|
||||||
|
}));
|
||||||
};
|
};
|
||||||
}, [socketRef]);
|
}, [socketRef]);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user