feat: support jetson hw encoder for video recording

src/codecs/jetson/jetson_encoder.cpp

@@ -6,26 +6,44 @@
 #include "NvBuffer.h"
 #include <NvBufSurface.h>
 
-const int KEY_FRAME_INTERVAL = 256;
 const int BUFFER_NUM = 4;
 
+static std::atomic<uint32_t> global_enc_id{0};
+
 /* Only accept V4L2_PIX_FMT_YUV420M (multi-pnale yuv420) or dma source input */
 std::unique_ptr<JetsonEncoder> JetsonEncoder::Create(int width, int height, uint32_t dst_pix_fmt,
                                                      bool is_dma_src) {
-    auto ptr = std::make_unique<JetsonEncoder>(width, height, dst_pix_fmt, is_dma_src);
+    JetsonEncoderConfig config = {
+        .width = width,
+        .height = height,
+        .is_dma_src = is_dma_src,
+        .dst_pix_fmt = dst_pix_fmt,
+    };
+    return Create(config);
+}
+
+std::unique_ptr<JetsonEncoder> JetsonEncoder::Create(JetsonEncoderConfig config) {
+    char auto_name[16];
+    snprintf(auto_name, sizeof(auto_name), "enc%d", global_enc_id.fetch_add(1) % 10);
+    auto ptr = std::make_unique<JetsonEncoder>(config, auto_name);
     ptr->Start();
     return ptr;
 }
 
-JetsonEncoder::JetsonEncoder(int width, int height, uint32_t dst_pix_fmt, bool is_dma_src)
+JetsonEncoder::JetsonEncoder(JetsonEncoderConfig config, const char *name)
     : abort_(true),
-      width_(width),
-      height_(height),
-      framerate_(30),
-      bitrate_bps_(2 * 1024 * 1024),
+      encoder_(nullptr),
+      name_(name),
+      width_(config.width),
+      height_(config.height),
+      framerate_(config.fps),
+      bitrate_bps_(config.bitrate),
+      i_interval_(config.i_interval),
+      idr_interval_(config.idr_interval),
       src_pix_fmt_(V4L2_PIX_FMT_NV12M),
-      dst_pix_fmt_(dst_pix_fmt),
-      is_dma_src_(is_dma_src) {}
+      dst_pix_fmt_(config.dst_pix_fmt),
+      is_dma_src_(config.is_dma_src),
+      rate_control_mode_(config.rc_mode) {}
 
 JetsonEncoder::~JetsonEncoder() {
     abort_ = true;
@@ -49,7 +67,7 @@ JetsonEncoder::~JetsonEncoder() {
 bool JetsonEncoder::CreateVideoEncoder() {
     int ret = 0;
 
-    encoder_ = NvVideoEncoder::createVideoEncoder("enc0");
+    encoder_ = NvVideoEncoder::createVideoEncoder(name_);
     if (!encoder_)
         ORIGINATE_ERROR("Could not create encoder");
 
@@ -81,24 +99,24 @@ bool JetsonEncoder::CreateVideoEncoder() {
         if (ret < 0)
             ORIGINATE_ERROR("Could not set B frame number");
 
-        ret = encoder_->setInsertSpsPpsAtIdrEnabled(true);
-        if (ret < 0)
-            ORIGINATE_ERROR("Could not insert SPS PPS at every IDR");
-
         ret = encoder_->setInsertVuiEnabled(true);
         if (ret < 0)
             ORIGINATE_ERROR("Could not insert Video Usability Information");
     }
 
-    ret = encoder_->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR);
+    ret = encoder_->setInsertSpsPpsAtIdrEnabled(true);
+    if (ret < 0)
+        ORIGINATE_ERROR("Could not insert SPS PPS at every IDR");
+
+    ret = encoder_->setRateControlMode(rate_control_mode_);
     if (ret < 0)
         ORIGINATE_ERROR("Could not set rate control mode");
 
-    ret = encoder_->setIDRInterval(KEY_FRAME_INTERVAL);
+    ret = encoder_->setIDRInterval(idr_interval_);
     if (ret < 0)
         ORIGINATE_ERROR("Could not set IDR interval");
 
-    ret = encoder_->setIFrameInterval(0);
+    ret = encoder_->setIFrameInterval(i_interval_);
     if (ret < 0)
         ORIGINATE_ERROR("Could not set I-frame interval");
 
@@ -171,7 +189,7 @@ void JetsonEncoder::SetBitrate(int adjusted_bitrate_bps) {
 void JetsonEncoder::ForceKeyFrame() {
     int ret = encoder_->forceIDR();
     if (ret < 0)
-        ERROR_PRINT("Could not force set encoder tokey frame");
+        ERROR_PRINT("Could not force set encoder to key frame");
 }
 
 void JetsonEncoder::Start() {

src/codecs/jetson/jetson_encoder.h

@@ -9,29 +9,48 @@
 
 #include <NvVideoEncoder.h>
 
+struct JetsonEncoderConfig {
+    int width;
+    int height;
+    bool is_dma_src;
+    uint32_t dst_pix_fmt;
+
+    int fps = 30;
+    int bitrate = 2 * 1024 * 1024;
+    int i_interval = 0;
+    int idr_interval = 256;
+    v4l2_mpeg_video_bitrate_mode rc_mode = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR;
+};
+
 class JetsonEncoder : public IFrameProcessor {
   public:
-    JetsonEncoder(int width, int height, uint32_t dst_pix_fmt, bool is_dma_src);
-    ~JetsonEncoder() override;
-
     static std::unique_ptr<JetsonEncoder> Create(int width, int height, uint32_t dst_pix_fmt,
                                                  bool is_dma_src);
+    static std::unique_ptr<JetsonEncoder> Create(JetsonEncoderConfig config);
+
+    JetsonEncoder(JetsonEncoderConfig config, const char *name);
+    ~JetsonEncoder() override;
+
     void EmplaceBuffer(V4L2FrameBufferRef frame_buffer,
-                       std::function<void(V4L2FrameBufferRef)> on_capture);
+                       std::function<void(V4L2FrameBufferRef)> on_capture) override;
     void ForceKeyFrame();
     void SetFps(int adjusted_fps);
     void SetBitrate(int adjusted_bitrate_bps);
 
   private:
     NvVideoEncoder *encoder_;
     std::atomic<bool> abort_;
+    const char *name_;
     int width_;
     int height_;
     int framerate_;
     int bitrate_bps_;
+    int i_interval_;
+    int idr_interval_;
     uint32_t src_pix_fmt_;
     uint32_t dst_pix_fmt_;
     bool is_dma_src_;
+    v4l2_mpeg_video_bitrate_mode rate_control_mode_;
     ThreadSafeQueue<std::function<void(V4L2FrameBufferRef)>> capturing_tasks_;
 
     bool CreateVideoEncoder();

src/common/v4l2_frame_buffer.cpp

@@ -119,7 +119,7 @@ rtc::scoped_refptr<V4L2FrameBuffer> V4L2FrameBuffer::Clone() const {
     } else {
         memcpy(clone->data_.get(), buffer_.start, size_);
     }
-
+    clone->SetDmaFd(buffer_.dmafd);
     clone->flags_ = flags_;
     clone->timestamp_ = timestamp_;
 

src/recorder/CMakeLists.txt

@@ -2,7 +2,25 @@ project(recorder)
 
 set(FFMPEG_LINK_LIBS avformat)
 
-aux_source_directory(${PROJECT_SOURCE_DIR} RECORDER_FILES)
+set(RECORDER_FILES
+    ${PROJECT_SOURCE_DIR}/audio_recorder.cpp
+    ${PROJECT_SOURCE_DIR}/openh264_recorder.cpp
+    ${PROJECT_SOURCE_DIR}/raw_h264_recorder.cpp
+    ${PROJECT_SOURCE_DIR}/recorder_manager.cpp
+    ${PROJECT_SOURCE_DIR}/video_recorder.cpp
+)
+
+if(USE_RPI_HW_ENCODER)
+    list(APPEND RECORDER_FILES
+        ${PROJECT_SOURCE_DIR}/v4l2_h264_recorder.cpp
+    )
+endif()
+
+if(USE_JETSON_HW_ENCODER)
+    list(APPEND RECORDER_FILES
+        ${PROJECT_SOURCE_DIR}/jetson_recorder.cpp
+    )
+endif()
 
 add_library(${PROJECT_NAME} ${RECORDER_FILES})
 

src/recorder/jetson_recorder.cpp

@@ -0,0 +1,36 @@
+#include "recorder/jetson_recorder.h"
+#include "common/logging.h"
+
+const float bpp_factor = 0.06f;
+
+std::unique_ptr<JetsonRecorder> JetsonRecorder::Create(int width, int height, int fps) {
+    return std::make_unique<JetsonRecorder>(width, height, fps);
+}
+
+JetsonRecorder::JetsonRecorder(int width, int height, int fps)
+    : VideoRecorder(width, height, fps, AV_CODEC_ID_AV1) {}
+
+void JetsonRecorder::Encode(rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer) {
+    if (!encoder_) {
+        JetsonEncoderConfig config = {
+            .width = width,
+            .height = height,
+            .is_dma_src = true,
+            .dst_pix_fmt = V4L2_PIX_FMT_AV1,
+            .fps = fps,
+            .bitrate = static_cast<int>(width * height * fps * bpp_factor),
+            .i_interval = 0,
+            .idr_interval = fps,
+            .rc_mode = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
+        };
+
+        encoder_ = JetsonEncoder::Create(config);
+    }
+
+    encoder_->EmplaceBuffer(frame_buffer, [this, frame_buffer](V4L2FrameBufferRef encoded_buffer) {
+        OnEncoded((uint8_t *)encoded_buffer->Data(), encoded_buffer->size(),
+                  frame_buffer->timestamp(), encoded_buffer->flags());
+    });
+}
+
+void JetsonRecorder::ReleaseEncoder() { encoder_.reset(); }

src/recorder/jetson_recorder.h

@@ -0,0 +1,20 @@
+#ifndef V4L2_H264_RECORDER_H_
+#define V4L2_H264_RECORDER_H_
+
+#include "codecs/jetson/jetson_encoder.h"
+#include "recorder/video_recorder.h"
+
+class JetsonRecorder : public VideoRecorder {
+  public:
+    static std::unique_ptr<JetsonRecorder> Create(int width, int height, int fps);
+    JetsonRecorder(int width, int height, int fps);
+
+  protected:
+    void ReleaseEncoder() override;
+    void Encode(rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer) override;
+
+  private:
+    std::unique_ptr<JetsonEncoder> encoder_;
+};
+
+#endif

src/recorder/openh264_recorder.cpp

@@ -1,11 +1,11 @@
 #include "recorder/openh264_recorder.h"
 
 std::unique_ptr<Openh264Recorder> Openh264Recorder::Create(int width, int height, int fps) {
-    return std::make_unique<Openh264Recorder>(width, height, fps, "h264_v4l2m2m");
+    return std::make_unique<Openh264Recorder>(width, height, fps);
 }
 
-Openh264Recorder::Openh264Recorder(int width, int height, int fps, std::string encoder_name)
-    : VideoRecorder(width, height, fps, encoder_name) {}
+Openh264Recorder::Openh264Recorder(int width, int height, int fps)
+    : VideoRecorder(width, height, fps, AV_CODEC_ID_H264) {}
 
 void Openh264Recorder::Encode(rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer) {
     if (!encoder_) {

src/recorder/openh264_recorder.h

@@ -7,7 +7,7 @@
 class Openh264Recorder : public VideoRecorder {
   public:
     static std::unique_ptr<Openh264Recorder> Create(int width, int height, int fps);
-    Openh264Recorder(int width, int height, int fps, std::string encoder_name);
+    Openh264Recorder(int width, int height, int fps);
 
   protected:
     void ReleaseEncoder() override;

src/recorder/raw_h264_recorder.cpp

@@ -6,11 +6,11 @@
 #define NAL_UNIT_TYPE_PPS 8
 
 std::unique_ptr<RawH264Recorder> RawH264Recorder::Create(int width, int height, int fps) {
-    return std::make_unique<RawH264Recorder>(width, height, fps, "h264_v4l2m2m");
+    return std::make_unique<RawH264Recorder>(width, height, fps);
 }
 
-RawH264Recorder::RawH264Recorder(int width, int height, int fps, std::string encoder_name)
-    : VideoRecorder(width, height, fps, encoder_name),
+RawH264Recorder::RawH264Recorder(int width, int height, int fps)
+    : VideoRecorder(width, height, fps, AV_CODEC_ID_H264),
       has_sps_(false),
       has_pps_(false),
       has_first_keyframe_(false){};
@@ -34,7 +34,8 @@ void RawH264Recorder::Encode(rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer) {
     }
 
     if (has_first_keyframe_) {
-        OnEncoded((uint8_t *)frame_buffer->Data(), frame_buffer->size(), frame_buffer->timestamp());
+        OnEncoded((uint8_t *)frame_buffer->Data(), frame_buffer->size(), frame_buffer->timestamp(),
+                  frame_buffer->flags());
     }
 }
 

src/recorder/raw_h264_recorder.h

@@ -6,7 +6,7 @@
 class RawH264Recorder : public VideoRecorder {
   public:
     static std::unique_ptr<RawH264Recorder> Create(int width, int height, int fps);
-    RawH264Recorder(int width, int height, int fps, std::string encoder_name);
+    RawH264Recorder(int width, int height, int fps);
     ~RawH264Recorder();
     void OnStart() override;
 

src/recorder/recorder_manager.cpp

@@ -12,9 +12,11 @@
 #include "common/utils.h"
 #include "common/v4l2_frame_buffer.h"
 #include "recorder/openh264_recorder.h"
-#if defined(RPI_PLATFORM)
 #include "recorder/raw_h264_recorder.h"
+#if defined(USE_RPI_HW_ENCODER)
 #include "recorder/v4l2_h264_recorder.h"
+#elif defined(USE_JETSON_HW_ENCODER)
+#include "recorder/jetson_recorder.h"
 #endif
 
 const int ROTATION_PERIOD = 60;
@@ -92,17 +94,15 @@ void RecorderManager::CreateVideoRecorder(std::shared_ptr<VideoCapturer> capture
         if (config.record_mode == RecordMode::Snapshot) {
             return nullptr;
         }
-#if defined(RPI_PLATFORM)
         if (capturer->format() == V4L2_PIX_FMT_H264) {
             return RawH264Recorder::Create(width, height, fps);
         } else if (config.hw_accel) {
+#if defined(USE_RPI_HW_ENCODER)
             return V4L2H264Recorder::Create(width, height, fps);
-        }
-#endif
-
-#if defined(JETSON_PLATFORM)
-        // todo: use jetson hw video encoder
+#elif defined(USE_JETSON_HW_ENCODER)
+            return JetsonRecorder::Create(width, height, fps);
 #endif
+        }
         return Openh264Recorder::Create(width, height, fps);
     })();
 }

src/recorder/v4l2_h264_recorder.cpp

@@ -1,11 +1,11 @@
 #include "recorder/v4l2_h264_recorder.h"
 
 std::unique_ptr<V4L2H264Recorder> V4L2H264Recorder::Create(int width, int height, int fps) {
-    return std::make_unique<V4L2H264Recorder>(width, height, fps, "h264_v4l2m2m");
+    return std::make_unique<V4L2H264Recorder>(width, height, fps);
 }
 
-V4L2H264Recorder::V4L2H264Recorder(int width, int height, int fps, std::string encoder_name)
-    : VideoRecorder(width, height, fps, encoder_name) {}
+V4L2H264Recorder::V4L2H264Recorder(int width, int height, int fps)
+    : VideoRecorder(width, height, fps, AV_CODEC_ID_H264) {}
 
 void V4L2H264Recorder::Encode(rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer) {
     if (!encoder_) {
@@ -20,7 +20,7 @@ void V4L2H264Recorder::Encode(rtc::scoped_refptr<V4L2FrameBuffer> frame_buffer)
 
     encoder_->EmplaceBuffer(frame_buffer, [this, frame_buffer](V4L2FrameBufferRef encoded_buffer) {
         OnEncoded((uint8_t *)encoded_buffer->Data(), encoded_buffer->size(),
-                  frame_buffer->timestamp());
+                  frame_buffer->timestamp(), encoded_buffer->flags());
     });
 }
 

src/recorder/v4l2_h264_recorder.h

@@ -7,7 +7,7 @@
 class V4L2H264Recorder : public VideoRecorder {
   public:
     static std::unique_ptr<V4L2H264Recorder> Create(int width, int height, int fps);
-    V4L2H264Recorder(int width, int height, int fps, std::string encoder_name);
+    V4L2H264Recorder(int width, int height, int fps);
 
   protected:
     void ReleaseEncoder() override;