From 314d2baec44be229179e011a03035ab86161d07d Mon Sep 17 00:00:00 2001
From: MythEclipse <asepharyanalm001@gmail.com>
Date: Mon, 8 Jun 2026 21:29:41 +0700
Subject: [PATCH] =?UTF-8?q?feat(voice):=20implement=20full=20voice=20trans?=
 =?UTF-8?q?mit=20(Browser=20=E2=86=92=20Discord)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Frontend (useAudioTransmit.ts):
- Capture microphone via getUserMedia
- Convert Float32 to Int16 PCM at 24kHz mono
- Base64 encode and send as JSON via WebSocket
- Send voice:transmit:start/stop commands on start/stop

Backend (ws/server.ts):
- Handle voice_transmit messages from browser
- Forward PCM data to Redis channel backend:voice:transmit
- Handle voice_command messages and forward to backend:command
- discord-gateway VoiceTransmitter receives and plays to voice channel

Flow:
   Browser Mic → getUserMedia → ScriptProcessor → Int16 PCM → base64 →
   WebSocket JSON → Backend → Redis → VoiceTransmitter → Discord Voice Channel

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 services/backend/src/ws/server.ts             | 41 +++++++++++++++----
 .../src/shared/hooks/useAudioTransmit.ts      | 34 +++++++++++++--
 2 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/services/backend/src/ws/server.ts b/services/backend/src/ws/server.ts
index 58988d3..34b73e8 100644
--- a/services/backend/src/ws/server.ts
+++ b/services/backend/src/ws/server.ts
@@ -65,13 +65,40 @@ export function createWebSocketServer(server: Server): WebSocketServer {
     );
 
     ws.on("message", (data: Buffer) => {
-      // Binary PCM data received from browser.
-      // Since backend has no Discord client to relay to, drop it.
-      if (Buffer.isBuffer(data) && data.length > 0) {
-        logger.debug(
-          { bytes: data.length },
-          "Dropping binary PCM (no Discord client)",
-        );
+      // Handle JSON messages from browser
+      if (typeof data === 'string' || (Buffer.isBuffer(data) && data.length > 0 && data[0] === 0x7B)) {
+        try {
+          const message = JSON.parse(data.toString());
+
+          if (message.type === 'voice_transmit' && message.buffer) {
+            // Forward PCM data to Redis for discord-gateway
+            import('../shared/redis/index.js').then(({ getCommandPublisher }) => {
+              const publisher = getCommandPublisher();
+              publisher.publish('backend:voice:transmit', JSON.stringify({
+                type: 'pcm',
+                buffer: message.buffer
+              })).catch((err: Error) => {
+                logger.error({ err }, 'Failed to publish voice transmit to Redis');
+              });
+            });
+          } else if (message.type === 'voice_command' && message.command) {
+            // Forward voice commands to discord-gateway
+            import('../shared/redis/index.js').then(({ getCommandPublisher }) => {
+              const publisher = getCommandPublisher();
+              const commandId = `cmd-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
+              publisher.publish('backend:command', JSON.stringify({
+                id: commandId,
+                type: message.command,
+                payload: {},
+                replyChannel: `reply:${commandId}`
+              })).catch((err: Error) => {
+                logger.error({ err }, 'Failed to publish voice command to Redis');
+              });
+            });
+          }
+        } catch (err) {
+          logger.debug({ err }, 'Failed to parse WebSocket message as JSON');
+        }
       }
     });
 
diff --git a/services/frontend/src/shared/hooks/useAudioTransmit.ts b/services/frontend/src/shared/hooks/useAudioTransmit.ts
index a82bd6e..e17fabd 100644
--- a/services/frontend/src/shared/hooks/useAudioTransmit.ts
+++ b/services/frontend/src/shared/hooks/useAudioTransmit.ts
@@ -12,6 +12,14 @@ export function useAudioTransmit(socketRef: {
   const processorRef = useRef<ScriptProcessorNode | null>(null);
 
   const stop = useCallback(() => {
+    // Send voice:transmit:stop command to backend
+    if (socketRef.current && socketRef.current.readyState === WebSocket.OPEN) {
+      socketRef.current.send(JSON.stringify({
+        type: 'voice_command',
+        command: 'voice:transmit:stop'
+      }));
+    }
+
     setIsStreaming(false);
     if (processorRef.current) {
       processorRef.current.disconnect();
@@ -25,9 +33,17 @@ export function useAudioTransmit(socketRef: {
       for (const track of streamRef.current.getTracks()) track.stop();
       streamRef.current = null;
     }
-  }, []);
+  }, [socketRef]);
 
   const start = useCallback(async () => {
+    // Send voice:transmit:start command to backend
+    if (socketRef.current && socketRef.current.readyState === WebSocket.OPEN) {
+      socketRef.current.send(JSON.stringify({
+        type: 'voice_command',
+        command: 'voice:transmit:start'
+      }));
+    }
+
     const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
     streamRef.current = stream;
     setIsStreaming(true);
@@ -49,8 +65,20 @@ export function useAudioTransmit(socketRef: {
       const pcmData = new Int16Array(inputData.length);
       for (let i = 0; i < inputData.length; i++)
         pcmData[i] = Math.max(-1, Math.min(1, inputData[i])) * 32767;
-      // BUG 2 FIX: slice() to create independent copy of the ArrayBuffer
-      socketRef.current.send(pcmData.buffer.slice(0));
+
+      // Convert to base64
+      const bytes = new Uint8Array(pcmData.buffer);
+      let binary = '';
+      for (let i = 0; i < bytes.length; i++) {
+        binary += String.fromCharCode(bytes[i]);
+      }
+      const base64 = btoa(binary);
+
+      // Send as JSON for backend to forward to Redis
+      socketRef.current.send(JSON.stringify({
+        type: 'voice_transmit',
+        buffer: base64
+      }));
     };
   }, [socketRef]);