From 62d131cf147128236a0d80afcc6a2fd5f9273f5b Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Sat, 16 May 2026 21:08:39 +0700 Subject: [PATCH] feat: implement local audio streaming with controls in voice components --- frontend/src/App.tsx | 83 +++++++++++++++++-- .../src/components/voice/VoiceControl.tsx | 7 ++ frontend/src/components/voice/VoicePanel.tsx | 2 + src/moderation/aiAnalysisWorker.ts | 7 ++ src/webserver.ts | 1 + 5 files changed, 95 insertions(+), 5 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c834064..62d8d1e 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -25,7 +25,11 @@ export default function App() { const [activeSpeakers, setActiveSpeakers] = useState([]); const [levels, setLevels] = useState(Array.from({ length: 32 }, () => 0.04)); const [isListening, setIsListening] = useState(false); - const audioContextRef = useRef(null); + const [isStreaming, setIsStreaming] = useState(false); + const audioContextListenRef = useRef(null); + const audioContextTransmitRef = useRef(null); + const streamRef = useRef(null); + const processorRef = useRef(null); const userTimelinesRef = useRef(new Map()); const activeTab = uiState.activeTab || "voice"; @@ -44,7 +48,7 @@ export default function App() { const average = int16Array.length ? sum / int16Array.length : 0; setLevels((prev) => prev.map((_, index) => Math.max(0.04, average * (0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) * 5))); - const audioContext = audioContextRef.current; + const audioContext = audioContextListenRef.current; if (!isListening || !audioContext) return; const float32Array = new Float32Array(int16Array.length); for (let i = 0; i < int16Array.length; i++) float32Array[i] = int16Array[i] / 32768; @@ -72,6 +76,73 @@ export default function App() { onPcm: handleIncomingPcm, }); + const stopStreamingLocal = useCallback(() => { + setIsStreaming(false); + if (processorRef.current) { + processorRef.current.disconnect(); + processorRef.current = null; + } + if (audioContextTransmitRef.current) { + audioContextTransmitRef.current.close(); + audioContextTransmitRef.current = null; + } + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) track.stop(); + streamRef.current = null; + } + setLevels(Array.from({ length: 32 }, () => 0.04)); + }, []); + + const startStreamingLocal = useCallback(async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + setIsStreaming(true); + + const AudioContextCtor = window.AudioContext || window.webkitAudioContext; + const audioContext = new AudioContextCtor({ sampleRate: SAMPLE_RATE }); + audioContextTransmitRef.current = audioContext; + + const source = audioContext.createMediaStreamSource(stream); + const processor = audioContext.createScriptProcessor(4096, 1, 1); + processorRef.current = processor; + + source.connect(processor); + processor.connect(audioContext.destination); + + processor.onaudioprocess = (event) => { + if (!socket.socketRef.current || socket.socketRef.current.readyState !== WebSocket.OPEN) return; + + const inputData = event.inputBuffer.getChannelData(0); + const pcmData = new Int16Array(inputData.length); + for (let i = 0; i < inputData.length; i++) { + pcmData[i] = Math.max(-1, Math.min(1, inputData[i])) * 32767; + } + socket.socketRef.current.send(pcmData.buffer); + + // Update local levels from mic + let sum = 0; + for (let i = 0; i < inputData.length; i++) sum += Math.abs(inputData[i]); + const average = inputData.length ? sum / inputData.length : 0; + setLevels((prev) => prev.map((_, index) => Math.max(0.04, average * (0.5 + Math.sin(index * 0.6 + Date.now() / 140) * 0.35 + 0.65) * 5))); + }; + } catch (err) { + console.error("Microphone access failed:", err); + setIsStreaming(false); + throw err; + } + }, [socket.socketRef]); + + const toggleStreaming = useCallback(async () => { + if (isStreaming) { + stopStreamingLocal(); + await patchUIState({ isStreaming: false }); + } else { + await startStreamingLocal(); + await patchUIState({ isStreaming: true }); + } + }, [isStreaming, startStreamingLocal, stopStreamingLocal, patchUIState]); + useEffect(() => { if (selectedVoiceGuild) voice.loadVoiceChannels(selectedVoiceGuild).catch(() => undefined); }, [selectedVoiceGuild, voice.loadVoiceChannels]); @@ -86,15 +157,15 @@ export default function App() { const toggleListening = useCallback(async () => { if (isListening) { - await audioContextRef.current?.suspend(); + await audioContextListenRef.current?.suspend(); userTimelinesRef.current.clear(); setIsListening(false); await patchUIState({ isListening: false }); return; } const AudioContextCtor = window.AudioContext || window.webkitAudioContext; - audioContextRef.current ??= new AudioContextCtor({ sampleRate: SAMPLE_RATE }); - await audioContextRef.current.resume(); + audioContextListenRef.current ??= new AudioContextCtor({ sampleRate: SAMPLE_RATE }); + await audioContextListenRef.current.resume(); setIsListening(true); await patchUIState({ isListening: true }); }, [isListening, patchUIState]); @@ -131,11 +202,13 @@ export default function App() { activeSpeakers={activeSpeakers} levels={levels} isListening={isListening} + isStreaming={isStreaming} onGuildChange={(guildId) => patchUIState({ selectedVoiceGuild: guildId, selectedVoiceChannel: "" })} onChannelChange={(channelId) => patchUIState({ selectedVoiceChannel: channelId })} onJoin={() => voice.joinVoice(selectedVoiceGuild, selectedVoiceChannel)} onDisconnect={() => voice.leaveVoice()} onListenToggle={toggleListening} + onStreamingToggle={toggleStreaming} /> diff --git a/frontend/src/components/voice/VoiceControl.tsx b/frontend/src/components/voice/VoiceControl.tsx index 7555230..d27cd94 100644 --- a/frontend/src/components/voice/VoiceControl.tsx +++ b/frontend/src/components/voice/VoiceControl.tsx @@ -15,7 +15,9 @@ interface VoiceControlProps { onJoin: () => void; onDisconnect: () => void; onListenToggle: () => void; + onStreamingToggle: () => void; isListening: boolean; + isStreaming: boolean; } export function VoiceControl({ @@ -30,7 +32,9 @@ export function VoiceControl({ onJoin, onDisconnect, onListenToggle, + onStreamingToggle, isListening, + isStreaming, }: VoiceControlProps) { return ( @@ -69,6 +73,9 @@ export function VoiceControl({ + diff --git a/frontend/src/components/voice/VoicePanel.tsx b/frontend/src/components/voice/VoicePanel.tsx index 7188345..dfa0f28 100644 --- a/frontend/src/components/voice/VoicePanel.tsx +++ b/frontend/src/components/voice/VoicePanel.tsx @@ -14,11 +14,13 @@ interface VoicePanelProps { activeSpeakers: ActiveSpeaker[]; levels: number[]; isListening: boolean; + isStreaming: boolean; onGuildChange: (guildId: string) => void; onChannelChange: (channelId: string) => void; onJoin: () => void; onDisconnect: () => void; onListenToggle: () => void; + onStreamingToggle: () => void; } export function VoicePanel(props: VoicePanelProps) { diff --git a/src/moderation/aiAnalysisWorker.ts b/src/moderation/aiAnalysisWorker.ts index cf9b246..24619ae 100644 --- a/src/moderation/aiAnalysisWorker.ts +++ b/src/moderation/aiAnalysisWorker.ts @@ -1,4 +1,5 @@ import { parentPort } from "node:worker_threads"; +import { initializeDatabase } from "../database/drizzle"; import { buildConversationPromptMessages } from "./conversationContext"; import { runModerationAnalysis } from "./llmModerationClient"; import { @@ -9,6 +10,8 @@ import type { MessageRecord } from "./types"; const MAX_CONTEXT_TOKENS = 8000; +let dbInitialized = false; + interface AnalysisWorkerRequest { conversationKey: string; messages: MessageRecord[]; @@ -32,6 +35,10 @@ async function processAnalysisRequest({ messages, }: AnalysisWorkerRequest): Promise { try { + if (!dbInitialized) { + await initializeDatabase(); + dbInitialized = true; + } const firstMessage = messages[0]; if (!firstMessage) return { ok: true, conversationKey, rows: [] }; diff --git a/src/webserver.ts b/src/webserver.ts index 8cada7d..c95ade3 100644 --- a/src/webserver.ts +++ b/src/webserver.ts @@ -218,6 +218,7 @@ export async function startWebserver( app.use(express.json()); app.use(express.static(path.join(__dirname, "../public"))); + app.use(express.static(path.join(__dirname, "../public/app"))); app.get("/", (_req: Request, res: Response) => { const reactIndex = path.join(__dirname, "../public/app/index.html");