diff --git a/services/frontend/src/App.tsx b/services/frontend/src/App.tsx index e90d1ea..e396705 100644 --- a/services/frontend/src/App.tsx +++ b/services/frontend/src/App.tsx @@ -76,7 +76,7 @@ export default function App() { ); const socket = useDashboardSocket({ - onBinary: audio.handleIncomingPcm, + onVoicePcmData: (d) => audio.handleIncomingPcm(d as { userId: string; pcm: string }), onUserState: (users) => setActiveSpeakers(users as ActiveSpeaker[]), onMessageCreated: (m) => messages.setMessages((prev) => mergeMessages(prev, [m as MessageRecord])), diff --git a/services/frontend/src/shared/hooks/useAudioPlayback.ts b/services/frontend/src/shared/hooks/useAudioPlayback.ts index f26d31c..f840db1 100644 --- a/services/frontend/src/shared/hooks/useAudioPlayback.ts +++ b/services/frontend/src/shared/hooks/useAudioPlayback.ts @@ -13,11 +13,16 @@ export function useAudioPlayback() { const userTimelinesRef = useRef(new Map()); const handleIncomingPcm = useCallback( - (data: ArrayBuffer) => { - const headerView = new DataView(data, 0, 4); - const userIdHash = headerView.getInt32(0, true); - const audioData = data.slice(4); - const int16Array = new Int16Array(audioData); + (data: { userId: string; pcm: string }) => { + // Decode base64 PCM data + const binaryString = atob(data.pcm); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + const int16Array = new Int16Array(bytes.buffer); + + // Calculate audio levels for visualization let sum = 0; for (const sample of int16Array) sum += Math.abs(sample / 32768); const average = int16Array.length ? sum / int16Array.length : 0; @@ -34,18 +39,25 @@ export function useAudioPlayback() { const audioContext = audioContextRef.current; if (!isListening || !audioContext) return; + + // Convert to float32 for Web Audio API const float32Array = new Float32Array(int16Array.length); for (let i = 0; i < int16Array.length; i++) float32Array[i] = int16Array[i] / 32768; + const audioBuffer = audioContext.createBuffer( CHANNELS, - float32Array.length / SAMPLE_RATE, + float32Array.length, SAMPLE_RATE, ); audioBuffer.getChannelData(0).set(float32Array); + const source = audioContext.createBufferSource(); source.buffer = audioBuffer; source.connect(audioContext.destination); + + // Schedule playback per user to avoid overlaps + const userIdHash = parseInt(data.userId, 10); const currentTime = audioContext.currentTime; let nextStart = userTimelinesRef.current.get(userIdHash) || 0; if (nextStart < currentTime) nextStart = currentTime + 0.05; diff --git a/services/frontend/src/shared/ws/socket.ts b/services/frontend/src/shared/ws/socket.ts index c646e46..dd17f33 100644 --- a/services/frontend/src/shared/ws/socket.ts +++ b/services/frontend/src/shared/ws/socket.ts @@ -18,6 +18,8 @@ export interface WsHandlers { onVoiceRecordingStarted?: (data: unknown) => void; onVoiceRecordingStopped?: (data: unknown) => void; onVoiceRecordingUploaded?: (data: unknown) => void; + onVoicePcmData?: (data: unknown) => void; + onVoiceActiveUser?: (data: unknown) => void; } let _wsInstance: WebSocket | null = null; @@ -90,6 +92,12 @@ function doConnect(): WebSocket { case "voice_recording_stopped": h.onVoiceRecordingStopped?.(msg.data); break; + case "voice_pcm_data": + h.onVoicePcmData?.(msg.data); + break; + case "voice_active_user": + h.onVoiceActiveUser?.(msg.data); + break; case "attachment_created": // attachment_created is informational — same data shape as message_created h.onMessageCreated?.(msg.data); @@ -149,6 +157,10 @@ export function useDashboardSocket(handlers: WsHandlers) { handlersRef.current.onVoiceRecordingStarted?.(d), onVoiceRecordingStopped: (d) => handlersRef.current.onVoiceRecordingStopped?.(d), + onVoicePcmData: (d) => + handlersRef.current.onVoicePcmData?.(d), + onVoiceActiveUser: (d) => + handlersRef.current.onVoiceActiveUser?.(d), }; _listeners.add(wrapper);