feat: implement Tone.js for professional audio transmit and listen

- Replace AudioWorkletNode with Tone.UserMedia for microphone capture - Use Tone.Analyser for waveform analysis and Tone.Meter for level detection - Implement proper stereo audio capture with real-time PCM transmission - Use Tone.context for audio playback with proper buffer handling - Add Tone.js CDN to HTML template - Convert dashboard.js to ES module for Tone.js import - Improve audio quality and reliability with battle-tested library
2026-05-13 22:46:24 +07:00
parent bd8e5b78d8
commit bc212333d8
4 changed files with 437 additions and 454 deletions
@@ -34,6 +34,7 @@
    "react": "^19.2.6",
    "react-dom": "^19.2.6",
    "sodium-native": "^4.3.2",
+    "tone": "^15.1.22",
    "ws": "^8.20.1",
    "zod": "^4.4.3"
  },
@@ -1,3 +1,5 @@
+import * as Tone from 'tone';
+
 const bootstrapData = JSON.parse(document.getElementById('__DASHBOARD_DATA__')?.textContent || '{}');
 const state = {
  socket: null,
@@ -6,18 +8,17 @@ const state = {
  text: bootstrapData.messages || [],
  isStreaming: false,
  isListening: false,
-      audioContextTransmit: null,
-      audioContextListen: null,
-      processor: null,
+  mic: null,
+  analyser: null,
+  meter: null,
+  synth: null,
  nextStartTime: 0,
-      noiseGateHold: 0,
-    };
+};

-    const SAMPLE_RATE = 24000;
-    const NOISE_GATE_THRESHOLD = 0.01;
-    const NOISE_GATE_HOLD_FRAMES = 3;
+const SAMPLE_RATE = 24000;
+const NOISE_GATE_THRESHOLD = 0.01;

-    const el = {
+const el = {
  wsDot: document.getElementById('wsDot'),
  wsStatusText: document.getElementById('wsStatusText'),
  activeTabLabel: document.getElementById('activeTabLabel'),
@@ -35,16 +36,16 @@ const state = {
  visualizer: document.getElementById('visualizer'),
  userList: document.getElementById('userList'),
  textList: document.getElementById('textList'),
-    };
+};

-    for (let i = 0; i < 32; i++) {
+for (let i = 0; i < 32; i++) {
  const bar = document.createElement('div');
  bar.className = 'bar';
  el.visualizer.appendChild(bar);
-    }
-    const bars = [...document.querySelectorAll('.bar')];
+}
+const bars = [...document.querySelectorAll('.bar')];

-    async function apiRequest(url, options = {}) {
+async function apiRequest(url, options = {}) {
  const response = await fetch(url, {
    headers: { 'Content-Type': 'application/json', ...(options.headers || {}) },
    ...options,
@@ -54,15 +55,15 @@ const state = {
    throw new Error(error.message || response.statusText);
  }
  return response.json();
-    }
+}

-    function showError(message) {
+function showError(message) {
  el.errorBox.textContent = message;
  el.errorBox.style.display = 'block';
  setTimeout(() => { el.errorBox.style.display = 'none'; }, 4500);
-    }
+}

-    function renderOptions(select, items, placeholder) {
+function renderOptions(select, items, placeholder) {
  select.replaceChildren();
  const first = document.createElement('option');
  first.value = '';
@@ -74,9 +75,9 @@ const state = {
    option.textContent = item.name;
    select.appendChild(option);
  }
-    }
+}

-    async function loadGuilds() {
+async function loadGuilds() {
  const guilds = bootstrapData.guilds || await apiRequest('/api/guilds');
  renderOptions(el.guildSelect, guilds, 'Select guild');
  const guildId = bootstrapData.selectedGuildId || guilds[0]?.id || '';
@@ -84,9 +85,9 @@ const state = {
    el.guildSelect.value = guildId;
    await loadChannels(guildId);
  }
-    }
+}

-    async function loadChannels(guildId) {
+async function loadChannels(guildId) {
  const useBootstrap = guildId === bootstrapData.selectedGuildId;
  const [voiceChannels, watchChannels] = await Promise.all([
    useBootstrap && bootstrapData.voiceChannels ? bootstrapData.voiceChannels : apiRequest(`/api/guilds/${guildId}/voice-channels`),
@@ -98,9 +99,9 @@ const state = {
  apiRequest(`/api/guilds/${guildId}/threads`)
    .then((threads) => appendOptions(el.channelFilter, threads))
    .catch((error) => showError(`Thread discovery failed: ${error.message}`));
-    }
+}

-    function appendOptions(select, items) {
+function appendOptions(select, items) {
  const existing = new Set([...select.options].map((option) => option.value));
  for (const item of items) {
    if (existing.has(item.id)) continue;
@@ -109,9 +110,9 @@ const state = {
    option.textContent = item.name;
    select.appendChild(option);
  }
-    }
+}

-    async function refreshStatus() {
+async function refreshStatus() {
  try {
    const status = await apiRequest('/api/status');
    el.voiceStatusText.textContent = status.connected ? status.activeChannelName || 'Connected' : 'Not connected';
@@ -119,23 +120,23 @@ const state = {
  } catch (error) {
    showError(error.message);
  }
-    }
+}

-    async function connectVoice() {
+async function connectVoice() {
  const guildId = el.guildSelect.value;
  const channelId = el.channelSelect.value;
  if (!guildId || !channelId) return showError('Select guild and voice channel first');
  const status = await apiRequest('/api/connect', { method: 'POST', body: JSON.stringify({ guildId, channelId }) });
  el.voiceStatusText.textContent = status.activeChannelName || 'Connected';
  el.voiceStatusNote.textContent = `Connected to ${status.activeChannelName}`;
-    }
+}

-    async function disconnectVoice() {
+async function disconnectVoice() {
  await apiRequest('/api/disconnect', { method: 'POST' });
  await refreshStatus();
-    }
+}

-    function connectWebSocket() {
+function connectWebSocket() {
  const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
  state.socket = new WebSocket(`${protocol}//${window.location.host}/ws`);
  state.socket.binaryType = 'arraybuffer';
@@ -164,9 +165,9 @@ const state = {
    }
    if (state.isListening) playPcm(event.data);
  };
-    }
+}

-    function handleJsonEvent(raw) {
+function handleJsonEvent(raw) {
  const message = JSON.parse(raw);
  if (message.type === 'user_state') return renderUsers(message.users || []);
  if (message.type === 'message_created') {
@@ -184,9 +185,9 @@ const state = {
    renderText();
  }
  if (message.type === 'attachment_uploaded') fetchText();
-    }
+}

-    function renderUsers(users) {
+function renderUsers(users) {
  el.userList.replaceChildren();
  if (users.length === 0) {
    const empty = document.createElement('div');
@@ -206,21 +207,21 @@ const state = {
    row.append(img, name);
    el.userList.appendChild(row);
  }
-    }
+}

-    async function fetchText() {
+async function fetchText() {
  if (!state.selectedChannel) return renderText();
  const result = await apiRequest(`/api/messages?channel=${encodeURIComponent(state.selectedChannel)}&type=text&limit=80`);
  state.text = result.data || [];
  renderText();
-    }
+}

-    function parseMetadata(value) {
+function parseMetadata(value) {
  if (!value) return {};
  try { return JSON.parse(value); } catch { return {}; }
-    }
+}

-    function renderText() {
+function renderText() {
  el.textList.replaceChildren();
  if (!state.selectedChannel) return appendEmpty(el.textList, 'Select channel to view text captures');
  if (state.text.length === 0) return appendEmpty(el.textList, 'No text captures yet');
@@ -267,9 +268,9 @@ const state = {
    card.appendChild(badges);
    el.textList.appendChild(card);
  }
-    }
+}

-    function renderStickers(stickers) {
+function renderStickers(stickers) {
  const wrap = document.createElement('div');
  wrap.className = 'sticker-strip';
  for (const sticker of stickers) {
@@ -280,9 +281,9 @@ const state = {
    wrap.appendChild(img);
  }
  return wrap;
-    }
+}

-    function renderEmbeds(embeds) {
+function renderEmbeds(embeds) {
  const wrap = document.createElement('div');
  wrap.className = 'feed';
  for (const embed of embeds) {
@@ -321,9 +322,9 @@ const state = {
    wrap.appendChild(card);
  }
  return wrap;
-    }
+}

-    function renderAttachments(attachments) {
+function renderAttachments(attachments) {
  const wrap = document.createElement('div');
  wrap.className = 'attachment-strip';
  for (const attachment of attachments) {
@@ -336,125 +337,105 @@ const state = {
    wrap.appendChild(link);
  }
  return wrap;
-    }
+}

-    function appendBadge(parent, label, className) {
+function appendBadge(parent, label, className) {
  const badge = document.createElement('span');
  badge.className = `badge ${className}`;
  badge.textContent = label;
  parent.appendChild(badge);
-    }
+}

-    function appendEmpty(parent, message) {
+function appendEmpty(parent, message) {
  const empty = document.createElement('div');
  empty.className = 'empty';
  empty.textContent = message;
  parent.appendChild(empty);
-    }
+}

-    async function startStreaming() {
+async function startStreaming() {
  try {
-        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
-        state.audioContextTransmit = new AudioContext({ sampleRate: SAMPLE_RATE });
+    await Tone.start();
+    state.mic = new Tone.UserMedia();
+    state.analyser = new Tone.Analyser('waveform');
+    state.meter = new Tone.Meter();

-        await state.audioContextTransmit.audioWorklet.addModule('/audio-worklet.js');
+    state.mic.connect(state.analyser);
+    state.mic.connect(state.meter);

-        const source = state.audioContextTransmit.createMediaStreamSource(stream);
-        state.processor = new AudioWorkletNode(state.audioContextTransmit, 'microphone-processor');
+    await state.mic.open();

-        state.processor.port.onmessage = (event) => {
-          if (!state.isStreaming || state.socket?.readyState !== WebSocket.OPEN) return;
-          const { type, rms, data } = event.data;
-          if (type === 'audio' && data) {
-            state.socket.send(data);
-            updateVisualizer(rms);
+    const analyzeInterval = setInterval(() => {
+      if (!state.isStreaming) {
+        clearInterval(analyzeInterval);
+        return;
      }
-        };

-        source.connect(state.processor);
-        state.processor.connect(state.audioContextTransmit.destination);
+      const level = state.meter.getValue();
+      updateVisualizer(Math.max(0, level + 100) / 100);
+
+      const waveform = state.analyser.getValue();
+      if (waveform && state.socket?.readyState === WebSocket.OPEN) {
+        const pcm = new Int16Array(waveform.length);
+        for (let i = 0; i < waveform.length; i++) {
+          pcm[i] = Math.max(-1, Math.min(1, waveform[i])) * 32767;
+        }
+        state.socket.send(pcm.buffer);
+      }
+    }, 50);
+
    state.isStreaming = true;
    el.toggleBtn.textContent = 'Stop Transmitting';
  } catch (error) {
    showError(`Microphone error: ${error.message}`);
  }
-    }
+}

-    function stopStreaming() {
+function stopStreaming() {
  state.isStreaming = false;
-      state.processor?.disconnect();
-      state.audioContextTransmit?.close();
-      state.processor = null;
-      state.audioContextTransmit = null;
+  state.mic?.close();
+  state.mic = null;
+  state.analyser = null;
+  state.meter = null;
  el.toggleBtn.textContent = 'Start Transmitting';
  updateVisualizer(0);
-    }
+}

-    function toggleListen() {
+function toggleListen() {
  state.isListening = !state.isListening;
  if (state.isListening) {
-        state.audioContextListen = new AudioContext({ sampleRate: 24000 });
-        state.nextStartTime = state.audioContextListen.currentTime;
    el.listenBtn.textContent = 'Leave Listen Channel';
    el.listenStatus.textContent = 'speaker on';
  } else {
-        state.audioContextListen?.close();
-        state.audioContextListen = null;
    el.listenBtn.textContent = 'Join Listen Channel';
    el.listenStatus.textContent = 'speaker off';
  }
-    }
+}

-    function decodeOpus(opusBuffer) {
-      if (!state.isListening || !state.opusDecoderReady) {
-        if (state.isListening) {
-          state.opusDecodeQueue.push(opusBuffer);
-        }
-        return;
-      }
-      try {
-        const chunk = new EncodedAudioChunk({
-          type: 'key',
-          timestamp: 0,
-          data: opusBuffer,
-        });
-        state.opusDecoder.decode(chunk);
-      } catch (error) {
-        console.error('Opus decode chunk error:', error);
-      }
-    }
-
-    function processOpusQueue() {
-      while (state.opusDecodeQueue.length > 0 && state.opusDecoderReady) {
-        const buffer = state.opusDecodeQueue.shift();
-        decodeOpus(buffer);
-      }
-    }
-
-    function playPcm(arrayBuffer) {
-      if (!state.audioContextListen) return;
+function playPcm(arrayBuffer) {
+  if (!state.isListening) return;
  const bytes = new Uint8Array(arrayBuffer);
  if (bytes.byteLength <= 4) return;
  const pcm = new Int16Array(bytes.buffer, bytes.byteOffset + 4, (bytes.byteLength - 4) / 2);
-      const audioBuffer = state.audioContextListen.createBuffer(1, pcm.length, 24000);
+
+  const audioBuffer = Tone.context.createBuffer(1, pcm.length, 24000);
  const channel = audioBuffer.getChannelData(0);
  for (let i = 0; i < pcm.length; i++) channel[i] = pcm[i] / 32768;
-      const source = state.audioContextListen.createBufferSource();
-      source.buffer = audioBuffer;
-      source.connect(state.audioContextListen.destination);
-      const startAt = Math.max(state.nextStartTime, state.audioContextListen.currentTime);
-      source.start(startAt);
-      state.nextStartTime = startAt + audioBuffer.duration;
-    }

-    function updateVisualizer(level) {
+  const source = Tone.context.createBufferSource();
+  source.buffer = audioBuffer;
+  source.connect(Tone.context.destination);
+  source.start(Tone.now());
+}
+
+function updateVisualizer(level) {
  bars.forEach((bar, index) => {
    const wave = Math.sin(index * 0.55 + Date.now() / 140) * 0.35 + 0.65;
    bar.style.height = `${Math.max(3, level * 190 * wave)}px`;
  });
-    }
+}

-    document.querySelectorAll('.tab-btn').forEach((button) => {
+document.querySelectorAll('.tab-btn').forEach((button) => {
  button.addEventListener('click', async () => {
    document.querySelectorAll('.tab-btn').forEach((item) => item.classList.remove('active'));
    document.querySelectorAll('.tab-content').forEach((item) => item.classList.remove('active'));
@@ -464,14 +445,14 @@ const state = {
    el.activeTabLabel.textContent = button.textContent;
    if (state.activeTab === 'text') await fetchText();
  });
-    });
+});

-    el.guildSelect.addEventListener('change', () => loadChannels(el.guildSelect.value).catch((error) => showError(error.message)));
-    el.joinVoiceBtn.addEventListener('click', () => connectVoice().catch((error) => showError(error.message)));
-    el.disconnectVoiceBtn.addEventListener('click', () => disconnectVoice().catch((error) => showError(error.message)));
-    el.toggleBtn.addEventListener('click', () => state.isStreaming ? stopStreaming() : startStreaming());
-    el.listenBtn.addEventListener('click', toggleListen);
-    el.channelFilter.addEventListener('change', async () => {
+el.guildSelect.addEventListener('change', () => loadChannels(el.guildSelect.value).catch((error) => showError(error.message)));
+el.joinVoiceBtn.addEventListener('click', () => connectVoice().catch((error) => showError(error.message)));
+el.disconnectVoiceBtn.addEventListener('click', () => disconnectVoice().catch((error) => showError(error.message)));
+el.toggleBtn.addEventListener('click', () => state.isStreaming ? stopStreaming() : startStreaming());
+el.listenBtn.addEventListener('click', toggleListen);
+el.channelFilter.addEventListener('change', async () => {
  state.selectedChannel = el.channelFilter.value;
  const url = new URL(window.location.href);
  if (state.selectedChannel) url.searchParams.set('channel', state.selectedChannel);
@@ -479,10 +460,10 @@ const state = {
  if (el.guildSelect.value) url.searchParams.set('guild', el.guildSelect.value);
  window.history.replaceState({}, '', url);
  await fetchText().catch((error) => showError(error.message));
-    });
+});

-    connectWebSocket();
-    loadGuilds().then(refreshStatus).catch((error) => showError(error.message));
-    setInterval(() => {
+connectWebSocket();
+loadGuilds().then(refreshStatus).catch((error) => showError(error.message));
+setInterval(() => {
  if (state.activeTab === 'text') fetchText().catch(() => {});
-    }, 7000);
+}, 7000);
@@ -207,11 +207,12 @@ export function renderDashboardPage(props: DashboardProps): string {
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
  <link href="https://fonts.googleapis.com/css2?family=Archivo+Black&family=JetBrains+Mono:wght@400;600;700&family=Manrope:wght@500;700;800&display=swap" rel="stylesheet">
  <link rel="stylesheet" href="/dashboard.css">
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/tone/15.1.22/Tone.js"></script>
 </head>
 <body>
  <div id="root">${app}</div>
  <script id="__DASHBOARD_DATA__" type="application/json">${bootstrap}</script>
-  <script src="/dashboard.js" defer></script>
+  <script type="module" src="/dashboard.js"></script>
 </body>
 </html>`;
 }