feat(VoiceReceiver): Recording video with audio

Full implemented

src/client/voice/dispatcher/AudioDispatcher.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const BaseDispatcher = require('./BaseDispatcher');
+const Util = require('../../../util/Util');
 const Silence = require('../util/Silence');
 const VolumeInterface = require('../util/VolumeInterface');
 
@@ -24,7 +25,7 @@ const VolumeInterface = require('../util/VolumeInterface');
 class AudioDispatcher extends BaseDispatcher {
   constructor(player, { seek = 0, volume = 1, fec, plp, bitrate = 96, highWaterMark = 12 } = {}, streams) {
     const streamOptions = { seek, volume, fec, plp, bitrate, highWaterMark };
-    super(player, highWaterMark, 120, false, streams);
+    super(player, highWaterMark, Util.getPayloadType('opus'), false, streams);
 
     this.streamOptions = streamOptions;
 

src/client/voice/networking/VoiceUDPClient.js

@@ -126,15 +126,7 @@ class VoiceConnectionUDPClient extends EventEmitter {
         op: VoiceOpcodes.SELECT_PROTOCOL,
         d: {
           protocol: 'udp',
-          codecs: [
-            {
-              name: 'opus',
-              type: 'audio',
-              priority: 1000,
-              payload_type: 120,
-            },
-            ...Util.getAllPayloadType(),
-          ],
+          codecs: Util.getAllPayloadType(),
           data: {
             address: packet.address,
             port: packet.port,

src/client/voice/receiver/FFmpegHandler.js

@@ -15,9 +15,15 @@ const { StreamOutput } = require('../util/Socket');
  * @extends {EventEmitter}
  */
 class FFmpegHandler extends EventEmitter {
-  constructor(codec, portUdp, output) {
+  constructor(codec, portUdp, output, isEnableAudio) {
     super();
 
+    /**
+     * If the audio is enabled
+     * @type {boolean}
+     */
+    this.isEnableAudio = isEnableAudio;
+
     /**
      * The codec of the stream
      * @type {VideoCodec}
@@ -41,7 +47,8 @@ class FFmpegHandler extends EventEmitter {
      */
     this.output = output;
 
-    const sdpData = Util.getSDPCodecName(codec, portUdp);
+    const sdpData = Util.getSDPCodecName(portUdp, this.isEnableAudio);
+
     /**
      * The FFmpeg process is ready or not
      * @type {boolean}
@@ -70,8 +77,8 @@ class FFmpegHandler extends EventEmitter {
       '-max_delay',
       '500000',
       '-y',
-      '-f', // Specify the format
-      'mpegts', // MKV format
+      '-f',
+      'matroska',
       isStream ? this.outputStream.url : output,
     ]);
 
@@ -88,14 +95,17 @@ class FFmpegHandler extends EventEmitter {
       this.emit('ready');
     });
     this.socket = createSocket('udp4');
+    this.socketAudio = createSocket('udp4');
   }
   /**
    * Send a payload to FFmpeg via UDP
    * @param {Buffer} payload The payload
+   * @param {boolean} isAudio If the payload is audio
    * @param {*} callback Callback
    */
   sendPayloadToFFmpeg(
     payload,
+    isAudio = false,
     callback = e => {
       if (e) {
         console.error('Error sending packet:', e);
@@ -103,7 +113,14 @@ class FFmpegHandler extends EventEmitter {
     },
   ) {
     const message = Buffer.from(payload);
-    this.socket.send(message, 0, message.length, this.portUdp, '127.0.0.1', callback);
+    if (isAudio && !this.isEnableAudio) {
+      return;
+    }
+    if (isAudio) {
+      this.socketAudio.send(message, 0, message.length, this.portUdp + 2, '127.0.0.1', callback);
+    } else {
+      this.socket.send(message, 0, message.length, this.portUdp, '127.0.0.1', callback);
+    }
   }
 
   destroy() {

src/client/voice/receiver/PacketHandler.js

@@ -56,9 +56,9 @@ class PacketHandler extends EventEmitter {
     return stream;
   }
 
-  makeVideoStream(user, portUdp, codec = 'H264', output) {
+  makeVideoStream(user, portUdp, codec, output, isEnableAudio = false) {
     if (this.videoStreams.has(user)) return this.videoStreams.get(user);
-    const stream = new FFmpegHandler(codec, portUdp, output);
+    const stream = new FFmpegHandler(codec, portUdp, output, isEnableAudio);
     stream.on('ready', () => {
       this.videoStreams.set(user, stream);
     });
@@ -186,6 +186,21 @@ class PacketHandler extends EventEmitter {
     }
   }
 
+  audioReceiverForStream(buffer) {
+    const ssrc = buffer.readUInt32BE(8);
+    const userStat = this.connection.ssrcMap.get(ssrc); // Audio_ssrc
+    if (!userStat) return;
+    const streamInfo = this.videoStreams.get(userStat.userId);
+    if (!streamInfo) return;
+    const packet = this.parseBuffer(buffer, true);
+    if (packet instanceof Error) {
+      return;
+    }
+    if (streamInfo.isEnableAudio) {
+      streamInfo.sendPayloadToFFmpeg(Buffer.concat(packet), true);
+    }
+  }
+
   videoReceiver(buffer) {
     const ssrc = buffer.readUInt32BE(8);
     const userStat = this.connection.ssrcMap.get(ssrc - 1); // Video_ssrc
@@ -203,7 +218,7 @@ class PacketHandler extends EventEmitter {
         // If this is a silence frame, pretend we never received it
         return;
       }
-      this.receiver.emit('videoData', ssrc, userStat, header, videoPacket);
+      this.receiver.emit('videoData', ssrc - 1, userStat, header, videoPacket);
 
       if (streamInfo) {
         streamInfo.sendPayloadToFFmpeg(Buffer.concat(packet));
@@ -214,6 +229,7 @@ class PacketHandler extends EventEmitter {
   push(buffer) {
     this.audioReceiver(buffer);
     this.videoReceiver(buffer);
+    this.audioReceiverForStream(buffer);
   }
 }
 

src/client/voice/receiver/Receiver.js

@@ -57,10 +57,10 @@ class VoiceReceiver extends EventEmitter {
   /**
    * Options passed to `VoiceReceiver#createVideoStream`.
    * @typedef {Object} ReceiveVideoStreamOptions
-   * @property {number} [portUdp] The UDP port to use for the video stream (local stream).
-   * @property {string} [codec='H264'] The codec to use for encoding the video. Default is 'H264'.
-   * <info>H265 supported, but not implemented</info>
-   * @property {any} [output] Additional output options, as required.
+   * @property {number} portUdp The UDP port to use for the video stream (local stream).
+   * @property {WritableStream|string} output Output stream or file path to write the video stream to.
+   * @property {boolean} [isEnableAudio=false] Enable audio for the video stream.
+   * <info>If you intend to record the stream with audio, make sure that `portUdp` and `portUdp + 2` are not in use.</info>
    */
 
   /**
@@ -71,11 +71,10 @@ class VoiceReceiver extends EventEmitter {
    * @param {ReceiveVideoStreamOptions} options Options.
    * @returns {FFmpegHandler} The video stream for the specified user.
    */
-  createVideoStream(user, { portUdp, codec, output } = {}) {
+  createVideoStream(user, { portUdp, output, isEnableAudio = false } = {}) {
     user = this.connection.client.users.resolve(user);
     if (!user) throw new Error('VOICE_USER_MISSING');
-    codec = 'H264';
-    const stream = this.packets.makeVideoStream(user.id, portUdp, codec, output);
+    const stream = this.packets.makeVideoStream(user.id, portUdp, 'H264', output, isEnableAudio);
     return stream;
   }
 

src/util/Util.js

@@ -944,23 +944,23 @@ class Util extends null {
     return payloadTypes.find(p => p.name === codecName).payload_type;
   }
 
-  static getSDPCodecName(packet, portUdp) {
-    let payload, payloadType;
-    if (typeof packet === 'string') {
-      payload = payloadTypes.find(p => p.name === packet);
-      payloadType = payload.payload_type;
-    } else {
-      const payloadType = packet[1] > 120 ? packet[1] & 0x80 : packet[1];
-      payload = payloadTypes.find(p => p.payload_type === payloadType);
-    }
-    let sdpData = `o=- 0 0 IN IP4 127.0.0.1
-s=No Name
-c=IN IP4 127.0.0.1
+  static getSDPCodecName(portUdp, isEnableAudio) {
+    let sdpData = `v=0
+o=- 0 0 IN IP4 0.0.0.0
+s=-
+c=IN IP4 0.0.0.0
 t=0 0
 a=tool:libavformat 61.1.100
-m=video ${portUdp} RTP/AVP ${payloadType}
-a=rtpmap:${payloadType} ${payload.name}/90000
-#Placeholder
+m=video ${portUdp} RTP/AVP 105
+a=rtpmap:105 H264/90000
+a=fmtp:105 profile-level-id=42e01f;sprop-parameter-sets=Z0IAH6tAoAt2AtwEBAaQeJEV,aM4JyA==;packetization-mode=1
+${
+  isEnableAudio
+    ? `m=audio ${portUdp + 2} RTP/AVP 120
+a=rtpmap:120 opus/48000/2
+a=fmtp:120 minptime=10;useinbandfec=1`
+    : ''
+}
 a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level
 a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
 a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
@@ -974,12 +974,6 @@ a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
 a=extmap:13 urn:3gpp:video-orientation
 a=extmap:14 urn:ietf:params:rtp-hdrext:toffset
 `;
-    if (payload.name === 'H264') {
-      sdpData = sdpData.replace(
-        '#Placeholder',
-        `a=fmtp:${payloadType} profile-level-id=42e01f;sprop-parameter-sets=Z0IAH6tAoAt2AtwEBAaQeJEV,aM4JyA==;packetization-mode=1`,
-      );
-    }
     return sdpData;
   }
 }