New feature: Play Opus stream from camera (#11)

Author: bangdc90

README.md

@@ -35,7 +35,17 @@ The project can then be opened in android studio and built from there.
 
 ## Installation
 - Download and install PixelPilot.apk from https://github.com/OpenIPC/PixelPilot/releases
-
+- Audio feature: Now PixelPilot app had ability to play opus stream from majestic on camera. In order to enable this feature, pls enable on camera side:
++ Audio settings in (/etc/majestic.yaml):
+```
+audio:
+  enabled: true
+  volume: 30
+  srate: 8000
+  codec: opus
+  outputEnabled: false
+  outputVolume: 30
+```
 ## Tested devices based on real user reviews
 
 * Samsung Galaxy A54 (Exynos 1380 processor)

app/src/main/java/com/openipc/pixelpilot/VideoActivity.java

@@ -631,6 +631,7 @@ protected void onStop() {
         unregisterReceivers();
         wfbLinkManager.stopAdapters();
         videoPlayer.stop();
+        videoPlayer.stopAudio();
         super.onStop();
     }
 
@@ -640,6 +641,7 @@ protected void onResume() {
 
         // On resume is called when the app is reopened, a device might have been plugged since the last time it started.
         videoPlayer.start();
+        videoPlayer.startAudio();
 
         wfbLinkManager.setChannel(getChannel(this));
         wfbLinkManager.refreshAdapters();

app/videonative/src/main/cpp/AudioDecoder.cpp

@@ -0,0 +1,104 @@
+//
+// Created by Admin on 28/09/2024.
+//
+
+#include "AudioDecoder.h"
+#include <android/log.h>
+
+#define TAG "pixelpilot"
+
+#define SAMPLE_RATE 8000
+#define CHANNELS 1
+
+AudioDecoder::AudioDecoder() {
+    initAudio();
+}
+
+AudioDecoder::~AudioDecoder() {
+
+    stopAudioProcessing();
+    delete pOpusDecoder;
+    AAudioStream_requestStop(m_stream);
+    AAudioStream_close(m_stream);
+}
+
+void AudioDecoder::enqueueAudio(const uint8_t *data, const std::size_t data_length)
+{
+    {
+        std::lock_guard<std::mutex> lock(m_mtxQueue);
+        m_audioQueue.push(AudioUDPPacket(data, data_length));
+    }
+    m_cvQueue.notify_one();
+}
+
+void AudioDecoder::processAudioQueue() {
+    while (true) {
+        std::unique_lock<std::mutex> lock(m_mtxQueue);
+        m_cvQueue.wait(lock, [this] { return !m_audioQueue.empty() || stopAudioFlag; });
+
+        if (stopAudioFlag) {
+            break;
+        }
+        if (!m_audioQueue.empty()) {
+            AudioUDPPacket audioPkt = m_audioQueue.front();
+            onNewAudioData(audioPkt.data, audioPkt.len);
+            m_audioQueue.pop();
+            lock.unlock();
+        }
+    }
+}
+
+void AudioDecoder::initAudio() {
+    __android_log_print(ANDROID_LOG_DEBUG, TAG, "initAudio");
+    int error;
+    pOpusDecoder = opus_decoder_create(SAMPLE_RATE, CHANNELS, &error);
+    // Create a stream m_builder
+    AAudio_createStreamBuilder(&m_builder);
+
+    // Set the stream format
+    AAudioStreamBuilder_setFormat(m_builder, AAUDIO_FORMAT_PCM_I16);
+    AAudioStreamBuilder_setChannelCount(m_builder, CHANNELS); // Mono
+    AAudioStreamBuilder_setSampleRate(m_builder, SAMPLE_RATE); // 8000 Hz
+
+    AAudioStreamBuilder_setBufferCapacityInFrames(m_builder, BUFFER_CAPACITY_IN_FRAMES);
+
+    // Open the stream
+    AAudioStreamBuilder_openStream(m_builder, &m_stream);
+    // Clean up the m_builder
+    AAudioStreamBuilder_delete(m_builder);
+
+    AAudioStream_requestStart(m_stream);
+
+    isInit = true;
+}
+
+void AudioDecoder::stopAudio()
+{
+    __android_log_print(ANDROID_LOG_DEBUG, TAG, "stopAudio");
+    AAudioStream_requestStop(m_stream);
+    AAudioStream_close(m_stream);
+    isInit = false;
+}
+
+void AudioDecoder::onNewAudioData(const uint8_t *data, const std::size_t data_length) {
+    const int rtp_header_size = 12;
+    const uint8_t* opus_payload = data + rtp_header_size;
+    int opus_payload_size = data_length - rtp_header_size;
+
+    int frame_size = opus_packet_get_samples_per_frame(opus_payload, SAMPLE_RATE);
+    int nb_frames = opus_packet_get_nb_frames(opus_payload, opus_payload_size);
+
+    // Decode the frame
+    int pcm_size = frame_size * nb_frames * CHANNELS;
+    if(pOpusDecoder && m_stream) {
+        opus_int16 pcm[pcm_size];
+        int decoded_samples = opus_decode(pOpusDecoder, opus_payload, opus_payload_size, pcm,
+                                          pcm_size, 0);
+
+        if (decoded_samples < 0) {
+            return;
+        }
+        // Process the decoded PCM data
+        AAudioStream_write(m_stream, pcm, decoded_samples, 0);
+    }
+}

app/videonative/src/main/cpp/AudioDecoder.h

@@ -0,0 +1,66 @@
+//
+// Created by BangDC on 28/09/2024.
+//
+
+#ifndef PIXELPILOT_AUDIODECODER_H
+#define PIXELPILOT_AUDIODECODER_H
+#include "libs/include/opus.h"
+#include <aaudio/AAudio.h>
+#include <memory>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <thread>
+
+typedef struct _AudioUDPPacket{
+    _AudioUDPPacket(const uint8_t* _data, size_t _len)
+    {
+        memcpy(data, _data, _len);
+        len = _len;
+    };
+    uint8_t data[250];
+    size_t len;
+} AudioUDPPacket;
+
+class AudioDecoder {
+public:
+    AudioDecoder();
+    ~AudioDecoder();
+
+    // Audio buffer
+    void initAudio();
+    void enqueueAudio(const uint8_t *data, const std::size_t data_length);
+    void startAudioProcessing() {
+        stopAudioFlag = false;
+        m_audioThread = std::thread(&AudioDecoder::processAudioQueue, this);
+    }
+
+    void stopAudioProcessing() {
+        {
+            std::lock_guard<std::mutex> lock(m_mtxQueue);
+            stopAudioFlag = true;
+        }
+        m_cvQueue.notify_all();
+        if (m_audioThread.joinable()) {
+            m_audioThread.join();
+        }
+    }
+    void processAudioQueue();
+    void stopAudio();
+    bool isInit = false;
+
+private:
+    void onNewAudioData(const uint8_t *data, const std::size_t data_length);
+
+private:
+    const int BUFFER_CAPACITY_IN_FRAMES = (1024 + 256);
+    std::queue<AudioUDPPacket> m_audioQueue;
+    std::mutex m_mtxQueue;
+    std::condition_variable m_cvQueue;
+    bool stopAudioFlag = false;
+    std::thread m_audioThread;
+    AAudioStreamBuilder* m_builder = nullptr;
+    AAudioStream* m_stream = nullptr;
+    OpusDecoder *pOpusDecoder = nullptr;
+};
+#endif //PIXELPILOT_AUDIODECODER_H

app/videonative/src/main/cpp/CMakeLists.txt

@@ -3,9 +3,12 @@ cmake_minimum_required(VERSION 3.6)
 
 project("VideoNative")
 
+include_directories(libs/include)
+
 add_library(${CMAKE_PROJECT_NAME} SHARED
         parser/H26XParser.cpp
         parser/ParseRTP.cpp
+        AudioDecoder.cpp
         UdpReceiver.cpp
         VideoDecoder.cpp
         VideoPlayer.cpp)
@@ -15,6 +18,8 @@ target_link_libraries(${CMAKE_PROJECT_NAME}
         # List libraries link to the target library
         android
         mediandk
+        aaudio
+        ${CMAKE_SOURCE_DIR}/libs/${ANDROID_ABI}/libopus.so
         log)
 
 set_property(TARGET ${CMAKE_PROJECT_NAME} PROPERTY CXX_STANDARD 20)

app/videonative/src/main/cpp/UdpReceiver.cpp

@@ -94,7 +94,7 @@ void UDPReceiver::receiveFromUDPLoop() {
     //wrap into unique pointer to avoid running out of stack
     const auto buff = std::make_unique<std::array<uint8_t, UDP_PACKET_MAX_SIZE>>();
 
-    MLOGE << "Listening on " << INADDR_ANY << ":" << mPort;
+    MLOGD << "Listening on " << INADDR_ANY << ":" << mPort;
 
     sockaddr_in source;
     socklen_t sourceLen = sizeof(sockaddr_in);

app/videonative/src/main/cpp/VideoDecoder.cpp

@@ -23,9 +23,9 @@ VideoDecoder::VideoDecoder(JNIEnv *env) {
 
 void VideoDecoder::setOutputSurface(JNIEnv *env, jobject surface, jint idx) {
     if (surface == nullptr) {
-        MLOGD << "Set output null surface";
+        MLOGD << "Set output null surface idx: " << idx;
         //assert(decoder.window!=nullptr);
-        if (decoder.window[idx] == nullptr || decoder.codec[idx] == nullptr) {
+        if (decoder.window[idx] == nullptr && decoder.codec[idx] == nullptr) {
             //MLOGD<<"Decoder window is already null";
             return;
         }
@@ -35,18 +35,22 @@ void VideoDecoder::setOutputSurface(JNIEnv *env, jobject surface, jint idx) {
             AMediaCodec_stop(decoder.codec[idx]);
             AMediaCodec_delete(decoder.codec[idx]);
             decoder.codec[idx] = nullptr;
+            MLOGD << "Set decoder.codec null idx: " << idx;
             mKeyFrameFinder.reset();
             decoder.configured[idx] = false;
             if (mCheckOutputThread[idx]->joinable()) {
                 mCheckOutputThread[idx]->join();
                 mCheckOutputThread[idx].reset();
             }
         }
-        ANativeWindow_release(decoder.window[idx]);
-        decoder.window[idx] = nullptr;
+        if (decoder.window[idx]) {
+            ANativeWindow_release(decoder.window[idx]);
+            decoder.window[idx] = nullptr;
+            MLOGD << "Set decoder.window null idx: " << idx;
+        }
         resetStatistics();
     } else {
-        MLOGD << "Set output non-null surface";
+        MLOGD << "Set output non-null surface idx :" << idx;
         // Throw warning if the surface is set without clearing it first
         assert(decoder.window[idx] == nullptr);
         decoder.window[idx] = ANativeWindow_fromSurface(env, surface);

app/videonative/src/main/cpp/VideoPlayer.cpp

@@ -93,9 +93,15 @@ void VideoPlayer::processQueue() {
 }
 
 //Not yet parsed bit stream (e.g. raw h264 or rtp data)
-void VideoPlayer::onNewVideoData(const uint8_t *data, const std::size_t data_length) {
-    //MLOGD << "onNewVideoData " << data_length;
-    mParser.parse_rtp_stream(data, data_length);
+void VideoPlayer::onNewRTPData(const uint8_t *data, const std::size_t data_length) {
+    const RTP::RTPPacket rtpPacket(data, data_length);
+    if (rtpPacket.header.payload == RTP_PAYLOAD_TYPE_AUDIO) {
+        audioDecoder.enqueueAudio(data, data_length);
+    }
+    else
+    {
+        mParser.parse_rtp_stream(data, data_length);
+    }
 }
 
 void VideoPlayer::onNewNALU(const NALU &nalu) {
@@ -126,7 +132,7 @@ void VideoPlayer::start(JNIEnv *env, jobject androidContext) {
     mUDPReceiver = std::make_unique<UDPReceiver>(javaVm, VS_PORT, "UdpReceiver", -16,
                                                  [this](const uint8_t *data,
                                                         size_t data_length) {
-                                                     onNewVideoData(data, data_length);
+                                                     onNewRTPData(data, data_length);
                                                  }, WANTED_UDP_RCVBUF_SIZE);
     mUDPReceiver->startReceiving();
 }
@@ -136,6 +142,8 @@ void VideoPlayer::stop(JNIEnv *env, jobject androidContext) {
         mUDPReceiver->stopReceiving();
         mUDPReceiver.reset();
     }
+
+    audioDecoder.stopAudio();
 }
 
 std::string VideoPlayer::getInfoString() const {
@@ -315,3 +323,20 @@ Java_com_openipc_videonative_VideoPlayer_nativeIsRecording(JNIEnv *env, jclass c
                                                            jlong native_instance) {
     return native(native_instance)->isRecording();
 }
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_openipc_videonative_VideoPlayer_nativeStartAudio(JNIEnv *env, jclass clazz,
+                                                          jlong native_instance) {
+    if(!native(native_instance)->audioDecoder.isInit)
+    {
+        native(native_instance)->audioDecoder.initAudio();
+    }
+    native(native_instance)->audioDecoder.stopAudioProcessing();
+    native(native_instance)->audioDecoder.startAudioProcessing();
+}
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_openipc_videonative_VideoPlayer_nativeStopAudio(JNIEnv *env, jclass clazz,
+                                                         jlong native_instance) {
+    native(native_instance)->audioDecoder.stopAudioProcessing();
+}

app/videonative/src/main/cpp/VideoPlayer.h

@@ -4,9 +4,9 @@
 
 #ifndef FPV_VR_VIDEOPLAYERN_H
 #define FPV_VR_VIDEOPLAYERN_H
-
 #include <jni.h>
 #include "VideoDecoder.h"
+#include "AudioDecoder.h"
 #include "UdpReceiver.h"
 #include "parser/H26XParser.h"
 #include "minimp4.h"
@@ -21,7 +21,7 @@ class VideoPlayer {
 public:
     VideoPlayer(JNIEnv *env, jobject context);
 
-    void onNewVideoData(const uint8_t *data, const std::size_t data_length);
+    void onNewRTPData(const uint8_t *data, const std::size_t data_length);
 
     /*
      * Set the surface the decoder can be configured with. When @param surface==nullptr
@@ -103,6 +103,7 @@ class VideoPlayer {
     void processQueue();
 
 public:
+    AudioDecoder audioDecoder;
     VideoDecoder videoDecoder;
     std::unique_ptr<UDPReceiver> mUDPReceiver;
     long nNALUsAtLastCall = 0;