refactor(voice): optimize recording pipeline and improve performance
Refactor the voice recording and playback systems to improve efficiency, reduce latency, and enhance Docker build performance. - **Infrastructure**: Optimize Dockerfiles using build mounts for pnpm cache and reorder layers for better caching of dependencies and build tools. - **Backend/Gateway**: - Refactor `broadcast` module to use a generic event-based system instead of hardcoded functions. - Simplify voice recording logic by merging metadata and segment management into a unified `segment.ts`. - Optimize audio downsampling in `streamSetup.ts` using `Int16Array` views for better performance. - Implement `withFallback` utility for more robust Redis/Database command execution. - **Frontend**: - Optimize audio playback visualization using pre-computed level shapes and efficient RMS calculation. - Reduce latency in voice commands by prioritizing WebSocket communication over HTTP. - Improve base64 encoding efficiency in audio transmission. - **General**: - Add default value for `ADMIN_PASSWORD` in shared config. - Fix Docker healthcheck to use `127.0.0.1` instead of `localhost`.
This commit is contained in:
MythEclipse
@@ -6,11 +6,18 @@ const logger = createLogger("use-audio-playback");
|
||||
|
||||
const SAMPLE_RATE = 24000;
|
||||
const CHANNELS = 1;
|
||||
const LEVEL_COUNT = 32;
|
||||
|
||||
// Pre-computed level distribution shape — computed once at module load, not per render
|
||||
const LEVEL_SHAPE = Array.from(
|
||||
{ length: LEVEL_COUNT },
|
||||
(_, i) => 0.3 + (Math.sin(i * 0.6) * 0.35 + 0.65) * 0.7,
|
||||
);
|
||||
|
||||
export function useAudioPlayback() {
|
||||
const [isListening, setIsListening] = useState(false);
|
||||
const [levels, setLevels] = useState<number[]>(
|
||||
Array.from({ length: 32 }, () => 0.04),
|
||||
Array.from({ length: LEVEL_COUNT }, () => 0.04),
|
||||
);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const userTimelinesRef = useRef(new Map<string, number>());
|
||||
@@ -19,36 +26,37 @@ export function useAudioPlayback() {
|
||||
(data: { userId: string; pcm: string }) => {
|
||||
// Decode base64 PCM data
|
||||
try {
|
||||
const binaryString = atob(data.pcm);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
const int16Array = new Int16Array(bytes.buffer);
|
||||
// 5a: Replace manual charCodeAt loop with Uint8Array.from
|
||||
const bytes = Uint8Array.from(atob(data.pcm), (c) => c.charCodeAt(0));
|
||||
if (bytes.length === 0) return;
|
||||
const int16Array = new Int16Array(
|
||||
bytes.buffer,
|
||||
bytes.byteOffset,
|
||||
bytes.byteLength / 2,
|
||||
);
|
||||
|
||||
// Calculate audio levels for visualization
|
||||
let sum = 0;
|
||||
for (const sample of int16Array) sum += Math.abs(sample / 32768);
|
||||
const average = int16Array.length ? sum / int16Array.length : 0;
|
||||
// 5b/5d: Real RMS calculation + Float32Array conversion in single pass
|
||||
let sumSquares = 0;
|
||||
const float32Array = new Float32Array(int16Array.length);
|
||||
for (let i = 0; i < int16Array.length; i++) {
|
||||
const normalized = int16Array[i] / 32768;
|
||||
float32Array[i] = normalized;
|
||||
sumSquares += normalized * normalized;
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / int16Array.length);
|
||||
// Scale RMS to a lively visualization range, clamp to [0.04, 1.0]
|
||||
const dbLevel = Math.min(1, Math.max(0.04, rms * 8));
|
||||
|
||||
// 5c: Use pre-computed LEVEL_SHAPE (no Date.now() per PCM frame)
|
||||
setLevels((prev) =>
|
||||
prev.map((_, index) =>
|
||||
Math.max(
|
||||
0.04,
|
||||
average *
|
||||
(0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) *
|
||||
5,
|
||||
),
|
||||
Math.max(0.04, dbLevel * LEVEL_SHAPE[index] * 5),
|
||||
),
|
||||
);
|
||||
|
||||
const audioContext = audioContextRef.current;
|
||||
if (!isListening || !audioContext) return;
|
||||
|
||||
// Convert to float32 for Web Audio API
|
||||
const float32Array = new Float32Array(int16Array.length);
|
||||
for (let i = 0; i < int16Array.length; i++)
|
||||
float32Array[i] = int16Array[i] / 32768;
|
||||
|
||||
const audioBuffer = audioContext.createBuffer(
|
||||
CHANNELS,
|
||||
float32Array.length,
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
// ─── Audio transmit hook — captures mic, encodes to PCM, sends via WebSocket ──
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
import { getAPIURL } from "../api/client";
|
||||
import { createChildLogger } from "../logger";
|
||||
import { getAPIURL } from "../api/client.js";
|
||||
import { createChildLogger } from "../logger.js";
|
||||
|
||||
const SAMPLE_RATE = 24000;
|
||||
const logger = createChildLogger("useAudioTransmit");
|
||||
|
||||
async function sendTransmitCommand(command: string): Promise<void> {
|
||||
// Send via HTTP API
|
||||
// Send via HTTP API (kept as exported function for backward compatibility)
|
||||
const resp = await fetch(`${getAPIURL()}/api/voice/command`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
@@ -24,6 +24,22 @@ async function sendTransmitCommand(command: string): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
function sendWsCommand(
|
||||
socketRef: { readonly current: WebSocket | null },
|
||||
command: string,
|
||||
): boolean {
|
||||
if (socketRef.current?.readyState === WebSocket.OPEN) {
|
||||
socketRef.current.send(
|
||||
JSON.stringify({
|
||||
type: "voice_command",
|
||||
command,
|
||||
}),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function useAudioTransmit(socketRef: {
|
||||
readonly current: WebSocket | null;
|
||||
}) {
|
||||
@@ -33,7 +49,10 @@ export function useAudioTransmit(socketRef: {
|
||||
const processorRef = useRef<ScriptProcessorNode | null>(null);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
sendTransmitCommand("voice:transmit:stop").catch(() => {});
|
||||
// 6c: Prefer WebSocket round-trip over HTTP for lower latency
|
||||
if (!sendWsCommand(socketRef, "voice:transmit:stop")) {
|
||||
sendTransmitCommand("voice:transmit:stop").catch(() => {});
|
||||
}
|
||||
|
||||
setIsStreaming(false);
|
||||
if (processorRef.current) {
|
||||
@@ -48,10 +67,13 @@ export function useAudioTransmit(socketRef: {
|
||||
for (const track of streamRef.current.getTracks()) track.stop();
|
||||
streamRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
}, [socketRef]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
await sendTransmitCommand("voice:transmit:start");
|
||||
// 6c: Prefer WebSocket round-trip over HTTP for lower latency
|
||||
if (!sendWsCommand(socketRef, "voice:transmit:start")) {
|
||||
await sendTransmitCommand("voice:transmit:start");
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
@@ -75,13 +97,10 @@ export function useAudioTransmit(socketRef: {
|
||||
for (let i = 0; i < inputData.length; i++)
|
||||
pcmData[i] = Math.max(-1, Math.min(1, inputData[i])) * 32767;
|
||||
|
||||
// Base64 encode
|
||||
// 6b: Replace string-concatenation loop with single call
|
||||
// 1024 samples → 2048 bytes — well within call-stack limits
|
||||
const bytes = new Uint8Array(pcmData.buffer);
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
const base64 = btoa(binary);
|
||||
const base64 = btoa(String.fromCharCode(...bytes));
|
||||
|
||||
socketRef.current.send(
|
||||
JSON.stringify({
|
||||
|
||||
Reference in New Issue
Block a user