[update] update llm-asr llm-audio llm-kws llm-melotts & add AI_Pyramid_Demo file

Author: LittleMouse

projects/llm_framework/main_asr/SConstruct

@@ -40,7 +40,7 @@ LDFLAGS += [
 
 STATIC_FILES += Glob('mode_*.json')
 
-env['COMPONENTS'].append({'target':'llm_asr-1.8',
+env['COMPONENTS'].append({'target':'llm_asr-1.9',
                           'SRCS':SRCS,
                           'INCLUDE':INCLUDE,
                           'PRIVATE_INCLUDE':PRIVATE_INCLUDE,

projects/llm_framework/main_asr/src/main.cpp

@@ -116,6 +116,7 @@ class llm_task {
     int delay_audio_frame_  = 10;
     float silence_ms_accum_ = 0.0f;
     float silence_timeout   = 1000.0f;
+    std::string pending_text_;
 
     buffer_t *pcmdata;
     std::function<void(void)> pause;
@@ -554,146 +555,98 @@ class llm_task {
 
     void sys_pcm_on_data_onnx(const std::string &raw)
     {
-        if (delay_audio_frame_ == 0) {
-            buffer_write_char(pcmdata, raw.data(), raw.length());
-            buffer_position_set(pcmdata, 0);
-
-            std::vector<float> floatSamples;
-            int16_t audio_val;
-            while (buffer_read_i16(pcmdata, &audio_val, 1)) {
-                float normalizedSample = static_cast<float>(audio_val) / INT16_MAX;
-                floatSamples.push_back(normalizedSample);
-            }
+        if (raw.size() >= sizeof(int16_t)) {
+            const int16_t *pcm16 = reinterpret_cast<const int16_t *>(raw.data());
+            size_t n16           = raw.size() / sizeof(int16_t);
+            PushPreRollPcm(pcm16, n16);
+        }
 
-            buffer_resize(pcmdata, 0);
-            int32_t window_size = vad_config_.silero_vad.window_size;
-            int32_t i           = 0;
-            std::string final_text;
+        static int count = 0;
+        if (count < delay_audio_frame_) {
+            buffer_write_char(pcmdata, raw.data(), raw.length());
+            count++;
+            return;
+        }
 
-            while (i < floatSamples.size()) {
-                if (i + window_size <= floatSamples.size()) {
-                    vad_->AcceptWaveform(floatSamples.data() + i, window_size);
-                } else {
-                    vad_->Flush();
-                }
-                i += window_size;
+        buffer_write_char(pcmdata, raw.data(), raw.length());
+        buffer_position_set(pcmdata, 0);
 
-                while (!vad_->Empty()) {
-                    const auto &segment = vad_->Front();
-                    float duration      = segment.samples.size() / 16000.f;
-                    float start_time    = segment.start / 16000.f;
-                    float end_time      = start_time + duration;
+        std::vector<float> floatSamples;
+        floatSamples.reserve((delay_audio_frame_ + 1) * kFrameSamples);
 
-                    if (duration < 0.1f) {
-                        vad_->Pop();
-                        continue;
-                    }
+        int16_t audio_val;
+        while (buffer_read_i16(pcmdata, &audio_val, 1)) {
+            floatSamples.push_back(static_cast<float>(audio_val) / 32768.0f);
+        }
 
-                    if (!offline_stream_) offline_stream_ = onnx_recognizer_->CreateStream();
+        buffer_resize(pcmdata, 0);
+        count = 0;
 
-                    offline_stream_->AcceptWaveform(onnx_asr_config_.feat_config.sampling_rate, segment.samples.data(),
-                                                    segment.samples.size());
+        vad_->AcceptWaveform(floatSamples.data(), floatSamples.size());
+        bool detected      = vad_->IsSpeechDetected();
+        bool speech_start  = (!prev_vad_detected_ && detected);
+        prev_vad_detected_ = detected;
 
-                    onnx_recognizer_->DecodeStream(offline_stream_.get());
-                    const auto &result = offline_stream_->GetResult();
+        while (!vad_->Empty()) {
+            const auto &segment = vad_->Front();
 
-                    final_text += result.text;
+            if (!offline_stream_) {
+                offline_stream_ = onnx_recognizer_->CreateStream();
+            }
 
-                    vad_->Pop();
-                    offline_stream_.reset();
+            if (speech_start && !pre_roll_pcm_.empty()) {
+                std::vector<float> pre;
+                pre.reserve(pre_roll_pcm_.size());
+                for (int16_t s : pre_roll_pcm_) {
+                    pre.push_back(static_cast<float>(s) / 32768.0f);
                 }
-            }
 
-            if (out_callback_) {
-                out_callback_(final_text, true);
-            }
-        } else {
-            if (raw.size() >= sizeof(int16_t)) {
-                const int16_t *pcm16 = reinterpret_cast<const int16_t *>(raw.data());
-                size_t n16           = raw.size() / sizeof(int16_t);
-                PushPreRollPcm(pcm16, n16);
-            }
+                std::vector<float> merged;
+                merged.reserve(pre.size() + segment.samples.size());
+                merged.insert(merged.end(), pre.begin(), pre.end());
+                merged.insert(merged.end(), segment.samples.begin(), segment.samples.end());
 
-            static int count = 0;
-            if (count < delay_audio_frame_) {
-                buffer_write_char(pcmdata, raw.data(), raw.length());
-                count++;
-                return;
+                offline_stream_->AcceptWaveform(kSampleRate, merged.data(), merged.size());
+                pre_roll_pcm_.clear();
+                speech_start = false;
+            } else {
+                offline_stream_->AcceptWaveform(kSampleRate, segment.samples.data(), segment.samples.size());
             }
 
-            buffer_write_char(pcmdata, raw.data(), raw.length());
-            buffer_position_set(pcmdata, 0);
-
-            std::vector<float> floatSamples;
-            floatSamples.reserve((delay_audio_frame_ + 1) * kFrameSamples);
+            onnx_recognizer_->DecodeStream(offline_stream_.get());
+            const auto &result = offline_stream_->GetResult();
 
-            int16_t audio_val;
-            while (buffer_read_i16(pcmdata, &audio_val, 1)) {
-                floatSamples.push_back(static_cast<float>(audio_val) / 32768.0f);
+            if (!result.text.empty() && out_callback_) {
+                out_callback_(result.text, false);
             }
 
-            buffer_resize(pcmdata, 0);
-            count = 0;
-
-            vad_->AcceptWaveform(floatSamples.data(), floatSamples.size());
-
-            bool detected      = vad_->IsSpeechDetected();
-            bool speech_start  = (!prev_vad_detected_ && detected);
-            prev_vad_detected_ = detected;
-
-            while (!vad_->Empty()) {
-                const auto &segment = vad_->Front();
-
-                if (!offline_stream_) {
-                    offline_stream_ = onnx_recognizer_->CreateStream();
-                }
-
-                if (speech_start && !pre_roll_pcm_.empty()) {
-                    std::vector<float> pre;
-                    pre.reserve(pre_roll_pcm_.size());
-                    for (int16_t s : pre_roll_pcm_) {
-                        pre.push_back(static_cast<float>(s) / 32768.0f);
-                    }
-
-                    std::vector<float> merged;
-                    merged.reserve(pre.size() + segment.samples.size());
-                    merged.insert(merged.end(), pre.begin(), pre.end());
-                    merged.insert(merged.end(), segment.samples.begin(), segment.samples.end());
-
-                    offline_stream_->AcceptWaveform(kSampleRate, merged.data(), merged.size());
-
-                    pre_roll_pcm_.clear();
-                    speech_start = false;
-                } else {
-                    offline_stream_->AcceptWaveform(kSampleRate, segment.samples.data(), segment.samples.size());
-                }
-
-                onnx_recognizer_->DecodeStream(offline_stream_.get());
-
-                const auto &result = offline_stream_->GetResult();
-                if (!result.text.empty() && out_callback_) {
-                    out_callback_(result.text, true);
-                }
+            if (!result.text.empty()) {
+                if (!pending_text_.empty()) pending_text_ += " ";
+                pending_text_ += result.text;
+            }
 
-                vad_->Pop();
+            vad_->Pop();
+            offline_stream_.reset();
+        }
 
-                offline_stream_.reset();
+        {
+            float chunk_ms = (delay_audio_frame_ + 1) * 10.0f;
+            if (detected) {
+                silence_ms_accum_ = 0.0f;
+            } else {
+                silence_ms_accum_ += chunk_ms;
             }
 
-            {
-                float chunk_ms = (delay_audio_frame_ + 1) * 10.0f;
-                if (detected) {
-                    silence_ms_accum_ = 0.0f;
-                } else {
-                    silence_ms_accum_ += chunk_ms;
+            if (silence_ms_accum_ >= silence_timeout) {
+                if (!pending_text_.empty() && out_callback_) {
+                    out_callback_(pending_text_, true);
                 }
+                pending_text_.clear();
 
-                if (silence_ms_accum_ >= silence_timeout) {
-                    if (ensleep_) {
-                        if (pause) pause();
-                    }
-                    silence_ms_accum_ = 0.0f;
+                if (ensleep_) {
+                    if (pause) pause();
                 }
+                silence_ms_accum_ = 0.0f;
             }
         }
     }

projects/llm_framework/main_audio/SConstruct

@@ -32,7 +32,7 @@ if 'CONFIG_AX_620E_MSP_ENABLED' in os.environ:
 STATIC_FILES += [AFile('audio.json'), AFile('audio_kit.json'), AFile('audio_pyramid.json')]
 STATIC_FILES += Glob('mode_*.json')
 
-env['COMPONENTS'].append({'target':'llm_audio-1.8',
+env['COMPONENTS'].append({'target':'llm_audio-1.9',
                           'SRCS':SRCS,
                           'INCLUDE':INCLUDE,
                           'PRIVATE_INCLUDE':PRIVATE_INCLUDE,