feat: add sub stream

src/args.h

@@ -49,6 +49,7 @@ template <typename DEFAULT> struct TimeVal {
 
 struct Args {
     // video input
+    int num_streams = 1;
     int camera_id = 0;
     int fps = 30;
     int width = 640;
@@ -60,6 +61,15 @@ struct Args {
     std::string camera = "libcamera:0";
     std::string v4l2_format = "mjpeg";
 
+    // sub stream for multiple resolution capture
+    int sub_width = 0;
+    int sub_height = 0;
+
+    // stream usage, 0: main stream, 1: sub stream
+    int record_stream_idx = 0; // recording stream index
+    int live_stream_idx = 0;   // webrtc live stream index
+    int ai_stream_idx = 0;     // ai stream index
+
     // audio input
     int sample_rate = 44100;
     bool no_audio = false;

src/capturer/CMakeLists.txt

@@ -15,7 +15,6 @@ if(USE_LIBARGUS_CAPTURE)
 
     list(APPEND CAPTURE_FILES
         libargus_egl_capturer.cpp
-        libargus_buffer_capturer.cpp
 
         ${MULTIMEDIA_API_BASE}/samples/common/classes/NvBuffer.cpp
         ${MULTIMEDIA_API_BASE}/samples/common/classes/NvLogging.cpp

src/capturer/libargus_egl_capturer.cpp

@@ -3,45 +3,74 @@
 #include <iostream>
 #include <sys/mman.h>
 
-#include "common/logging.h"
 #include "common/utils.h"
 
 static constexpr uint64_t kAcquireFrameTimeoutNs = 3'000'000'000;
 
 std::shared_ptr<LibargusEglCapturer> LibargusEglCapturer::Create(Args args) {
     auto ptr = std::make_shared<LibargusEglCapturer>(args);
-    ptr->InitCamera();
+    ptr->Initialize();
     ptr->StartCapture();
     return ptr;
 }
 
 LibargusEglCapturer::LibargusEglCapturer(Args args)
     : camera_id_(args.camera_id),
-      dma_fd_(-1),
+      num_streams_(args.num_streams),
       fps_(args.fps),
-      width_(args.width),
-      height_(args.height),
       format_(args.format),
-      config_(args),
-      camera_provider_(Argus::CameraProvider::create()) {}
+      config_(args) {}
 
 LibargusEglCapturer::~LibargusEglCapturer() {
-    DestroyNvBufferFromFd(dma_fd_);
-
     if (icapture_session_) {
         icapture_session_->stopRepeat();
         icapture_session_->waitForIdle();
     }
+
+    stream_handlers_.clear();
+
+    for (int i = 0; i < num_streams_; i++) {
+        output_streams_[i].reset();
+    }
+
+    capture_session_.reset();
     camera_provider_.reset();
 }
 
-void LibargusEglCapturer::DestroyNvBufferFromFd(int &fd) {
-    if (fd >= 0) {
+void LibargusEglCapturer::Initialize() {
+    output_streams_.resize(num_streams_);
+
+    stream_handlers_.emplace_back(
+        StreamHandler::Create(0, Argus::Size2D<uint32_t>(config_.width, config_.height)));
+    int frame_size =
+        config_.width * config_.height + ((config_.width + 1) / 2) * ((config_.height + 1) / 2) * 2;
+    stream_handlers_[0]->SetFrameBuffer(
+        V4L2FrameBuffer::Create(config_.width, config_.height, frame_size, format_));
+
+    if (has_sub_stream()) {
+        stream_handlers_.emplace_back(StreamHandler::Create(
+            1, Argus::Size2D<uint32_t>(config_.sub_width, config_.sub_height)));
+        frame_size = config_.sub_width * config_.sub_height +
+                     ((config_.sub_width + 1) / 2) * ((config_.sub_height + 1) / 2) * 2;
+        stream_handlers_[1]->SetFrameBuffer(
+            V4L2FrameBuffer::Create(config_.sub_width, config_.sub_height, frame_size, format_));
+    }
+
+    camera_provider_.reset(Argus::CameraProvider::create());
+    if (!camera_provider_) {
+        throw std::runtime_error("Failed to create CameraProvider");
+    }
+
+    InitStreams();
+}
+
+void StreamHandler::DestroyNvBufferFromFd() {
+    if (dma_fd_ >= 0) {
         NvBufSurface *nvbuf = nullptr;
-        if (NvBufSurfaceFromFd(fd, (void **)(&nvbuf)) == 0 && nvbuf) {
+        if (NvBufSurfaceFromFd(dma_fd_, (void **)(&nvbuf)) == 0 && nvbuf) {
             NvBufSurfaceDestroy(nvbuf);
         }
-        fd = -1;
+        dma_fd_ = -1;
     }
 }
 
@@ -157,11 +186,8 @@ void LibargusEglCapturer::PrintCameraDeviceInfo(Argus::CameraDevice *cameraDevic
     }
 }
 
-void LibargusEglCapturer::InitCamera() {
-    if (!camera_provider_) {
-        throw std::runtime_error("Failed to create CameraProvider");
-    }
-    icamera_provider_ = Argus::interface_cast<Argus::ICameraProvider>(camera_provider_);
+void LibargusEglCapturer::InitStreams() {
+    auto icamera_provider_ = Argus::interface_cast<Argus::ICameraProvider>(camera_provider_);
     if (!icamera_provider_) {
         throw std::runtime_error("Failed to create CameraProvider");
     }
@@ -183,8 +209,8 @@ void LibargusEglCapturer::InitCamera() {
     }
 
     camera_device_ = camera_devices[camera_id_];
-    auto capture_session(icamera_provider_->createCaptureSession(camera_device_));
-    icapture_session_ = Argus::interface_cast<Argus::ICaptureSession>(capture_session);
+    capture_session_.reset(icamera_provider_->createCaptureSession(camera_device_));
+    icapture_session_ = Argus::interface_cast<Argus::ICaptureSession>(capture_session_);
     if (!icapture_session_) {
         throw std::runtime_error("Failed to get ICaptureSession");
     }
@@ -195,52 +221,63 @@ void LibargusEglCapturer::InitCamera() {
     if (!iegl_stream_settings) {
         throw std::runtime_error("Failed to get IEGLOutputStreamSettings");
     }
-    iegl_stream_settings->setEGLDisplay(EGL_NO_DISPLAY);
-    iegl_stream_settings->setPixelFormat(Argus::PIXEL_FMT_YCbCr_420_888);
-    iegl_stream_settings->setResolution(Argus::Size2D<uint32_t>(width_, height_));
-
-    output_stream_ = Argus::UniqueObj<Argus::OutputStream>(
-        icapture_session_->createOutputStream(stream_settings.get()));
-    request_ = Argus::UniqueObj<Argus::Request>(
-        icapture_session_->createRequest(Argus::CAPTURE_INTENT_VIDEO_RECORD));
-    irequest_ = interface_cast<Argus::IRequest>(request_);
-    irequest_->enableOutputStream(output_stream_.get());
-
-    auto isource_settings = interface_cast<Argus::ISourceSettings>(irequest_->getSourceSettings());
-    if (isource_settings) {
-        auto mode = FindBestSensorMode(width_, height_, fps_);
-        isource_settings->setSensorMode(mode);
-        isource_settings->setFrameDurationRange(
-            Argus::Range<uint64_t>(static_cast<uint64_t>(1e9 / fps_)));
+    for (size_t i = 0; i < num_streams_; i++) {
+        iegl_stream_settings->setEGLDisplay(EGL_NO_DISPLAY);
+        iegl_stream_settings->setPixelFormat(Argus::PIXEL_FMT_YCbCr_420_888);
+        iegl_stream_settings->setResolution(stream_handlers_[i]->GetSize());
+        output_streams_[i].reset(icapture_session_->createOutputStream(stream_settings.get()));
+        stream_handlers_[i]->SetOutputStream(output_streams_[i].get());
     }
 
-    iegl_stream_ = interface_cast<Argus::IEGLOutputStream>(output_stream_);
-    frame_consumer_ = Argus::UniqueObj<EGLStream::FrameConsumer>(
-        EGLStream::FrameConsumer::create(output_stream_.get()));
-    iframe_consumer_ = interface_cast<EGLStream::IFrameConsumer>(frame_consumer_);
+    request_ = Argus::UniqueObj<Argus::Request>(icapture_session_->createRequest());
+    auto irequest_ = interface_cast<Argus::IRequest>(request_);
+    for (size_t i = 0; i < num_streams_; i++) {
+        irequest_->enableOutputStream(output_streams_[i].get());
+        auto irequest_stream_settings_ = interface_cast<Argus::IStreamSettings>(
+            irequest_->getStreamSettings(output_streams_[i].get()));
+        if (!irequest_stream_settings_) {
+            throw std::runtime_error("Failed to get IStreamSettings");
+        }
+        irequest_stream_settings_->setPostProcessingEnable(false);
+    }
 
-    if (icapture_session_->repeat(request_.get()) != Argus::STATUS_OK) {
-        throw std::runtime_error("Failed to start repeat");
+    auto isource_settings = interface_cast<Argus::ISourceSettings>(irequest_->getSourceSettings());
+    if (!isource_settings) {
+        throw std::runtime_error("Failed to get ISourceSettings");
     }
+    auto mode = FindBestSensorMode(config_.width, config_.height, fps_);
+    isource_settings->setSensorMode(mode);
+    isource_settings->setFrameDurationRange(
+        Argus::Range<uint64_t>(static_cast<uint64_t>(1e9 / fps_)));
 }
 
 int LibargusEglCapturer::fps() const { return fps_; }
 
-int LibargusEglCapturer::width() const { return width_; }
+int LibargusEglCapturer::width(int stream_idx) const {
+    if (stream_idx <= 0 || stream_idx >= num_streams_) {
+        return stream_handlers_[0]->width();
+    }
+    return stream_handlers_[stream_idx]->width();
+}
 
-int LibargusEglCapturer::height() const { return height_; }
+int LibargusEglCapturer::height(int stream_idx) const {
+    if (stream_idx <= 0 || stream_idx >= num_streams_) {
+        return stream_handlers_[0]->height();
+    }
+    return stream_handlers_[stream_idx]->height();
+}
 
 bool LibargusEglCapturer::is_dma_capture() const { return false; }
 
 uint32_t LibargusEglCapturer::format() const { return format_; }
 
 Args LibargusEglCapturer::config() const { return config_; }
 
-void LibargusEglCapturer::CaptureImage() {
+void StreamHandler::CaptureImage() {
     Argus::Status status;
     Argus::UniqueObj<EGLStream::Frame> frame(
-        iframe_consumer_->acquireFrame(kAcquireFrameTimeoutNs, &status));
-    if (status != Argus::STATUS_OK) {
+        i_consumer_->acquireFrame(kAcquireFrameTimeoutNs, &status));
+    if (status != Argus::STATUS_OK || !frame) {
         return;
     }
 
@@ -253,15 +290,15 @@ void LibargusEglCapturer::CaptureImage() {
         return;
     }
     if (dma_fd_ == -1) {
-        dma_fd_ = native_buffer->createNvBuffer(iegl_stream_->getResolution(),
-                                                NVBUF_COLOR_FORMAT_YUV420, NVBUF_LAYOUT_PITCH);
+        dma_fd_ =
+            native_buffer->createNvBuffer(size_, NVBUF_COLOR_FORMAT_YUV420, NVBUF_LAYOUT_PITCH);
         if (dma_fd_ < 0) {
             return;
         }
     }
 
     if (native_buffer->copyToNvBuffer(dma_fd_) != Argus::STATUS_OK) {
-        DestroyNvBufferFromFd(dma_fd_);
+        DestroyNvBufferFromFd();
         return;
     }
 
@@ -270,7 +307,7 @@ void LibargusEglCapturer::CaptureImage() {
 
     NvBufSurface *nvbuf = nullptr;
     if (NvBufSurfaceFromFd(dma_fd_, reinterpret_cast<void **>(&nvbuf)) != 0) {
-        DestroyNvBufferFromFd(dma_fd_);
+        DestroyNvBufferFromFd();
         return;
     }
 
@@ -296,25 +333,57 @@ void LibargusEglCapturer::CaptureImage() {
     Next(frame_buffer_);
 }
 
-rtc::scoped_refptr<webrtc::I420BufferInterface> LibargusEglCapturer::GetI420Frame() {
-    return frame_buffer_->ToI420();
+rtc::scoped_refptr<webrtc::I420BufferInterface> LibargusEglCapturer::GetI420Frame(int stream_idx) {
+    if (stream_idx <= 0 || stream_idx >= num_streams_) {
+        return stream_handlers_[0]->GetFrameBuffer()->ToI420();
+    }
+    return stream_handlers_[stream_idx]->GetFrameBuffer()->ToI420();
 }
 
-void LibargusEglCapturer::StartCapture() {
-    framesize_ = width_ * height_ + ((width_ + 1) / 2) * ((height_ + 1) / 2) * 2;
-    frame_buffer_ = V4L2FrameBuffer::Create(width_, height_, framesize_, format_);
-
-    if (iegl_stream_->waitUntilConnected() != Argus::STATUS_OK) {
-        ERROR_PRINT("Stream failed to connect.");
+void StreamHandler::StartCapture() {
+    if (!output_stream_) {
+        ERROR_PRINT("Output stream is null");
         return;
     }
 
-    worker_ = std::make_unique<Worker>("LibargusCapture", [this]() {
-        CaptureImage();
+    consumer_ = Argus::UniqueObj<EGLStream::FrameConsumer>(
+        EGLStream::FrameConsumer::create(output_stream_));
+    i_consumer_ = interface_cast<EGLStream::IFrameConsumer>(consumer_);
+
+    INFO_PRINT("stream %d initialized: %dx%d", stream_idx_, width(), height());
+
+    worker_ = std::make_unique<Worker>("argus stream: " + std::to_string(stream_idx_), [this]() {
+        auto iegl_stream_ = interface_cast<Argus::IEGLOutputStream>(output_stream_);
+
+        if (iegl_stream_->waitUntilConnected() != Argus::STATUS_OK) {
+            ERROR_PRINT("Stream failed to connect.");
+            return;
+        }
+        while (running_) {
+            CaptureImage();
+        }
     });
     worker_->Run();
 }
 
+void LibargusEglCapturer::StartCapture() {
+    for (auto &ctx : stream_handlers_) {
+        ctx->StartCapture();
+    }
+
+    if (icapture_session_->repeat(request_.get()) != Argus::STATUS_OK) {
+        throw std::runtime_error("Failed to start repeat");
+    }
+}
+
+Subscription LibargusEglCapturer::Subscribe(Subject<V4L2FrameBufferRef>::Callback callback,
+                                            int stream_idx) {
+    if (stream_idx <= 0 || stream_idx >= num_streams_) {
+        return stream_handlers_[0]->Subscribe(std::move(callback));
+    }
+    return stream_handlers_[stream_idx]->Subscribe(std::move(callback));
+}
+
 Argus::SensorMode *LibargusEglCapturer::FindBestSensorMode(int req_width, int req_height,
                                                            int req_fps) {
     auto icamera_properties = interface_cast<Argus::ICameraProperties>(camera_device_);