fix(frontend): handle voice PCM data as JSON instead of binary
- Add onVoicePcmData and onVoiceActiveUser handlers to WebSocket interface
- Add voice_pcm_data and voice_active_user event cases in socket message handler
- Update useAudioPlayback to decode base64 PCM from JSON format
- Connect onVoicePcmData to audio.handleIncomingPcm in App.tsx
Format change:
- Old: Binary (4 bytes userId + PCM buffer)
- New: JSON {userId: string, pcm: base64string}
This matches the format sent by discord-gateway via Redis.
Now PCM audio will flow correctly: Discord → Gateway → Redis → Backend → WebSocket → Browser!
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
1c7b7e6398
commit
2e9e9c1a06
@@ -13,11 +13,16 @@ export function useAudioPlayback() {
|
||||
const userTimelinesRef = useRef(new Map<number, number>());
|
||||
|
||||
const handleIncomingPcm = useCallback(
|
||||
(data: ArrayBuffer) => {
|
||||
const headerView = new DataView(data, 0, 4);
|
||||
const userIdHash = headerView.getInt32(0, true);
|
||||
const audioData = data.slice(4);
|
||||
const int16Array = new Int16Array(audioData);
|
||||
(data: { userId: string; pcm: string }) => {
|
||||
// Decode base64 PCM data
|
||||
const binaryString = atob(data.pcm);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
const int16Array = new Int16Array(bytes.buffer);
|
||||
|
||||
// Calculate audio levels for visualization
|
||||
let sum = 0;
|
||||
for (const sample of int16Array) sum += Math.abs(sample / 32768);
|
||||
const average = int16Array.length ? sum / int16Array.length : 0;
|
||||
@@ -34,18 +39,25 @@ export function useAudioPlayback() {
|
||||
|
||||
const audioContext = audioContextRef.current;
|
||||
if (!isListening || !audioContext) return;
|
||||
|
||||
// Convert to float32 for Web Audio API
|
||||
const float32Array = new Float32Array(int16Array.length);
|
||||
for (let i = 0; i < int16Array.length; i++)
|
||||
float32Array[i] = int16Array[i] / 32768;
|
||||
|
||||
const audioBuffer = audioContext.createBuffer(
|
||||
CHANNELS,
|
||||
float32Array.length / SAMPLE_RATE,
|
||||
float32Array.length,
|
||||
SAMPLE_RATE,
|
||||
);
|
||||
audioBuffer.getChannelData(0).set(float32Array);
|
||||
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(audioContext.destination);
|
||||
|
||||
// Schedule playback per user to avoid overlaps
|
||||
const userIdHash = parseInt(data.userId, 10);
|
||||
const currentTime = audioContext.currentTime;
|
||||
let nextStart = userTimelinesRef.current.get(userIdHash) || 0;
|
||||
if (nextStart < currentTime) nextStart = currentTime + 0.05;
|
||||
|
||||
@@ -18,6 +18,8 @@ export interface WsHandlers {
|
||||
onVoiceRecordingStarted?: (data: unknown) => void;
|
||||
onVoiceRecordingStopped?: (data: unknown) => void;
|
||||
onVoiceRecordingUploaded?: (data: unknown) => void;
|
||||
onVoicePcmData?: (data: unknown) => void;
|
||||
onVoiceActiveUser?: (data: unknown) => void;
|
||||
}
|
||||
|
||||
let _wsInstance: WebSocket | null = null;
|
||||
@@ -90,6 +92,12 @@ function doConnect(): WebSocket {
|
||||
case "voice_recording_stopped":
|
||||
h.onVoiceRecordingStopped?.(msg.data);
|
||||
break;
|
||||
case "voice_pcm_data":
|
||||
h.onVoicePcmData?.(msg.data);
|
||||
break;
|
||||
case "voice_active_user":
|
||||
h.onVoiceActiveUser?.(msg.data);
|
||||
break;
|
||||
case "attachment_created":
|
||||
// attachment_created is informational — same data shape as message_created
|
||||
h.onMessageCreated?.(msg.data);
|
||||
@@ -149,6 +157,10 @@ export function useDashboardSocket(handlers: WsHandlers) {
|
||||
handlersRef.current.onVoiceRecordingStarted?.(d),
|
||||
onVoiceRecordingStopped: (d) =>
|
||||
handlersRef.current.onVoiceRecordingStopped?.(d),
|
||||
onVoicePcmData: (d) =>
|
||||
handlersRef.current.onVoicePcmData?.(d),
|
||||
onVoiceActiveUser: (d) =>
|
||||
handlersRef.current.onVoiceActiveUser?.(d),
|
||||
};
|
||||
|
||||
_listeners.add(wrapper);
|
||||
|
||||
Reference in New Issue
Block a user