feat(VoiceReceiver): Recording video with audio

Full implemented

examples/VoiceChannel/RecordingVideo.js

@@ -29,7 +29,9 @@ client.on('ready', async client => {
 
   const video = connectionStream.receiver.createVideoStream('user_id', {
     portUdp: 5004,
-    output: fs.createWriteStream('video.ts'), // Output file using MPEG-TS container
+    output: fs.createWriteStream('video.mkv'), // Output file using matroska container
+    // If you want video with audio, set isEnableAudio to true
+    isEnableAudio: false,
   });
 
   video.stream.stderr.on('data', data => {

src/client/voice/dispatcher/AudioDispatcher.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const BaseDispatcher = require('./BaseDispatcher');
+const Util = require('../../../util/Util');
 const Silence = require('../util/Silence');
 const VolumeInterface = require('../util/VolumeInterface');
 
@@ -24,7 +25,7 @@ const VolumeInterface = require('../util/VolumeInterface');
 class AudioDispatcher extends BaseDispatcher {
   constructor(player, { seek = 0, volume = 1, fec, plp, bitrate = 96, highWaterMark = 12 } = {}, streams) {
     const streamOptions = { seek, volume, fec, plp, bitrate, highWaterMark };
-    super(player, highWaterMark, 120, false, streams);
+    super(player, highWaterMark, Util.getPayloadType('opus'), false, streams);
 
     this.streamOptions = streamOptions;
 

src/client/voice/networking/VoiceUDPClient.js

@@ -126,15 +126,7 @@ class VoiceConnectionUDPClient extends EventEmitter {
         op: VoiceOpcodes.SELECT_PROTOCOL,
         d: {
           protocol: 'udp',
-          codecs: [
+          codecs: Util.getAllPayloadType(),
-            {
-              name: 'opus',
-              type: 'audio',
-              priority: 1000,
-              payload_type: 120,
-            },
-            ...Util.getAllPayloadType(),
-          ],
           data: {
             address: packet.address,
             port: packet.port,

src/client/voice/receiver/FFmpegHandler.js

@@ -15,9 +15,15 @@ const { StreamOutput } = require('../util/Socket');
  * @extends {EventEmitter}
  */
 class FFmpegHandler extends EventEmitter {
-  constructor(codec, portUdp, output) {
+  constructor(codec, portUdp, output, isEnableAudio) {
     super();
 
+    /**
+     * If the audio is enabled
+     * @type {boolean}
+     */
+    this.isEnableAudio = isEnableAudio;
+
     /**
      * The codec of the stream
      * @type {VideoCodec}
@@ -41,7 +47,8 @@ class FFmpegHandler extends EventEmitter {
      */
     this.output = output;
 
-    const sdpData = Util.getSDPCodecName(codec, portUdp);
+    const sdpData = Util.getSDPCodecName(portUdp, this.isEnableAudio);
+
     /**
      * The FFmpeg process is ready or not
      * @type {boolean}
@@ -70,8 +77,8 @@ class FFmpegHandler extends EventEmitter {
       '-max_delay',
       '500000',
       '-y',
-      '-f', // Specify the format
+      '-f',
-      'mpegts', // MKV format
+      'matroska',
       isStream ? this.outputStream.url : output,
     ]);
 
@@ -88,14 +95,17 @@ class FFmpegHandler extends EventEmitter {
       this.emit('ready');
     });
     this.socket = createSocket('udp4');
+    this.socketAudio = createSocket('udp4');
   }
   /**
    * Send a payload to FFmpeg via UDP
    * @param {Buffer} payload The payload
+   * @param {boolean} isAudio If the payload is audio
    * @param {*} callback Callback
    */
   sendPayloadToFFmpeg(
     payload,
+    isAudio = false,
     callback = e => {
       if (e) {
         console.error('Error sending packet:', e);
@@ -103,7 +113,14 @@ class FFmpegHandler extends EventEmitter {
     },
   ) {
     const message = Buffer.from(payload);
-    this.socket.send(message, 0, message.length, this.portUdp, '127.0.0.1', callback);
+    if (isAudio && !this.isEnableAudio) {
+      return;
+    }
+    if (isAudio) {
+      this.socketAudio.send(message, 0, message.length, this.portUdp + 2, '127.0.0.1', callback);
+    } else {
+      this.socket.send(message, 0, message.length, this.portUdp, '127.0.0.1', callback);
+    }
   }
 
   destroy() {

src/client/voice/receiver/PacketHandler.js

@@ -56,9 +56,9 @@ class PacketHandler extends EventEmitter {
     return stream;
   }
 
-  makeVideoStream(user, portUdp, codec = 'H264', output) {
+  makeVideoStream(user, portUdp, codec, output, isEnableAudio = false) {
     if (this.videoStreams.has(user)) return this.videoStreams.get(user);
-    const stream = new FFmpegHandler(codec, portUdp, output);
+    const stream = new FFmpegHandler(codec, portUdp, output, isEnableAudio);
     stream.on('ready', () => {
       this.videoStreams.set(user, stream);
     });
@@ -186,6 +186,21 @@ class PacketHandler extends EventEmitter {
     }
   }
 
+  audioReceiverForStream(buffer) {
+    const ssrc = buffer.readUInt32BE(8);
+    const userStat = this.connection.ssrcMap.get(ssrc); // Audio_ssrc
+    if (!userStat) return;
+    const streamInfo = this.videoStreams.get(userStat.userId);
+    if (!streamInfo) return;
+    const packet = this.parseBuffer(buffer, true);
+    if (packet instanceof Error) {
+      return;
+    }
+    if (streamInfo.isEnableAudio) {
+      streamInfo.sendPayloadToFFmpeg(Buffer.concat(packet), true);
+    }
+  }
+
   videoReceiver(buffer) {
     const ssrc = buffer.readUInt32BE(8);
     const userStat = this.connection.ssrcMap.get(ssrc - 1); // Video_ssrc
@@ -203,7 +218,7 @@ class PacketHandler extends EventEmitter {
         // If this is a silence frame, pretend we never received it
         return;
       }
-      this.receiver.emit('videoData', ssrc, userStat, header, videoPacket);
+      this.receiver.emit('videoData', ssrc - 1, userStat, header, videoPacket);
 
       if (streamInfo) {
         streamInfo.sendPayloadToFFmpeg(Buffer.concat(packet));
@@ -214,6 +229,7 @@ class PacketHandler extends EventEmitter {
   push(buffer) {
     this.audioReceiver(buffer);
     this.videoReceiver(buffer);
+    this.audioReceiverForStream(buffer);
   }
 }
 

src/client/voice/receiver/Receiver.js

@@ -57,10 +57,10 @@ class VoiceReceiver extends EventEmitter {
   /**
    * Options passed to `VoiceReceiver#createVideoStream`.
    * @typedef {Object} ReceiveVideoStreamOptions
-   * @property {number} [portUdp] The UDP port to use for the video stream (local stream).
+   * @property {number} portUdp The UDP port to use for the video stream (local stream).
-   * @property {string} [codec='H264'] The codec to use for encoding the video. Default is 'H264'.
+   * @property {WritableStream|string} output Output stream or file path to write the video stream to.
-   * <info>H265 supported, but not implemented</info>
+   * @property {boolean} [isEnableAudio=false] Enable audio for the video stream.
-   * @property {any} [output] Additional output options, as required.
+   * <info>If you intend to record the stream with audio, make sure that `portUdp` and `portUdp + 2` are not in use.</info>
    */
 
   /**
@@ -71,11 +71,10 @@ class VoiceReceiver extends EventEmitter {
    * @param {ReceiveVideoStreamOptions} options Options.
    * @returns {FFmpegHandler} The video stream for the specified user.
    */
-  createVideoStream(user, { portUdp, codec, output } = {}) {
+  createVideoStream(user, { portUdp, output, isEnableAudio = false } = {}) {
     user = this.connection.client.users.resolve(user);
     if (!user) throw new Error('VOICE_USER_MISSING');
-    codec = 'H264';
+    const stream = this.packets.makeVideoStream(user.id, portUdp, 'H264', output, isEnableAudio);
-    const stream = this.packets.makeVideoStream(user.id, portUdp, codec, output);
     return stream;
   }
 

src/util/Util.js

@@ -944,23 +944,23 @@ class Util extends null {
     return payloadTypes.find(p => p.name === codecName).payload_type;
   }
 
-  static getSDPCodecName(packet, portUdp) {
+  static getSDPCodecName(portUdp, isEnableAudio) {
-    let payload, payloadType;
+    let sdpData = `v=0
-    if (typeof packet === 'string') {
+o=- 0 0 IN IP4 0.0.0.0
-      payload = payloadTypes.find(p => p.name === packet);
+s=-
-      payloadType = payload.payload_type;
+c=IN IP4 0.0.0.0
-    } else {
-      const payloadType = packet[1] > 120 ? packet[1] & 0x80 : packet[1];
-      payload = payloadTypes.find(p => p.payload_type === payloadType);
-    }
-    let sdpData = `o=- 0 0 IN IP4 127.0.0.1
-s=No Name
-c=IN IP4 127.0.0.1
 t=0 0
 a=tool:libavformat 61.1.100
-m=video ${portUdp} RTP/AVP ${payloadType}
+m=video ${portUdp} RTP/AVP 105
-a=rtpmap:${payloadType} ${payload.name}/90000
+a=rtpmap:105 H264/90000
-#Placeholder
+a=fmtp:105 profile-level-id=42e01f;sprop-parameter-sets=Z0IAH6tAoAt2AtwEBAaQeJEV,aM4JyA==;packetization-mode=1
+${
+  isEnableAudio
+    ? `m=audio ${portUdp + 2} RTP/AVP 120
+a=rtpmap:120 opus/48000/2
+a=fmtp:120 minptime=10;useinbandfec=1`
+    : ''
+}
 a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level
 a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
 a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
@@ -974,12 +974,6 @@ a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
 a=extmap:13 urn:3gpp:video-orientation
 a=extmap:14 urn:ietf:params:rtp-hdrext:toffset
 `;
-    if (payload.name === 'H264') {
-      sdpData = sdpData.replace(
-        '#Placeholder',
-        `a=fmtp:${payloadType} profile-level-id=42e01f;sprop-parameter-sets=Z0IAH6tAoAt2AtwEBAaQeJEV,aM4JyA==;packetization-mode=1`,
-      );
-    }
     return sdpData;
   }
 }

typings/index.d.ts

@@ -1105,8 +1105,10 @@ export class FFmpegHandler extends EventEmitter {
   public ready: boolean;
   public stream: ChildProcessWithoutNullStreams;
   public socket: Socket;
+  public socketAudio: Socket;
   public output: Writable | string;
-  public sendPayloadToFFmpeg(payload: Buffer): void;
+  public isEnableAudio: boolean;
+  public sendPayloadToFFmpeg(payload: Buffer, isAudio?: boolean): void;
   public on(event: 'ready', listener: () => void): this;
   public once(event: 'ready', listener: () => void): this;
   public destroy(): void;
@@ -1117,7 +1119,11 @@ export class VoiceReceiver extends EventEmitter {
   public createStream(user: UserResolvable, options?: { mode?: 'opus' | 'pcm'; end?: 'silence' | 'manual' }): Readable;
   public createVideoStream(
     user: UserResolvable,
-    options?: { portUdp: number; codec: 'H264'; output: Writable | string },
+    options?: {
+      portUdp: number;
+      output: Writable | string;
+      isEnableAudio: boolean;
+    },
   ): FFmpegHandler;
 
   public on(event: 'debug', listener: (error: Error | string) => void): this;