Release v0.4.0 — VAD smart command end (parity with Python v0.2.0)

Cargo.lock

@@ -3331,6 +3331,7 @@ dependencies = [
  "toml 0.9.11+spec-1.1.0",
  "unic-langid",
  "vosk",
+ "webrtc-vad",
  "winrt-notification",
 ]
 
@@ -8598,6 +8599,15 @@ dependencies = [
  "rustls-pki-types",
 ]
 
+[[package]]
+name = "webrtc-vad"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39a1e40fd6ca90be95459152a2537f2ba4286ee1b13073f7ebcaa74fc94e3008"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "webview2-com"
 version = "0.38.2"

Cargo.toml

@@ -53,4 +53,5 @@ ndarray = "0.17"
 tokenizers = { version = "0.22", default-features = false }
 regex = "1"
 sys-locale = "0.3"
+webrtc-vad = "0.4"
 

README.md

@@ -137,6 +137,33 @@ println!("{}", reply);
 
 Системный промпт (русский) описывает J.A.R.V.I.S. как британского дворецкого Тони Старка — короткие реплики (1–3 предложения), обращение «сэр», без излишней цензуры. Меняется в `config::LLM_SYSTEM_PROMPT_RU`.
 
+## VAD: умное завершение команды по тишине
+
+После срабатывания wake-word `jarvis-app` слушает команду. Раньше окно закрывалось либо по таймауту Vosk, либо по жёсткому пределу из `config::CMS_WAIT_DELAY` — короткие команды («стоп», «громче») всё равно ждали несколько секунд тишины.
+
+Теперь окно закрывается, как только пользователь замолчал. Реализовано через `webrtc-vad` (чистый Rust, тот же алгоритм, что в Python-версии). Это **паритет с Python v0.2.0**, где использовался `webrtcvad` с теми же параметрами.
+
+Логика (state-машина в `crates/jarvis-core/src/audio_processing/vad/listen_window.rs`):
+
+- Кадр VAD: 30 мс при 16 кГц = 480 сэмплов = 960 байт (mono i16). Кадры с микрофона (по 512 сэмплов от `pv_recorder`) аккумулируются и режутся на VAD-кадры адаптером `WebRtcVad::push_samples`.
+- Каждый VAD-кадр учитывается в счётчиках `speech_ms` / `silence_ms`.
+- Окно закрывается, когда `silence_ms >= VAD_COMMAND_END_SILENCE_MS` И `speech_ms >= VAD_COMMAND_MIN_SPEECH_MS`.
+- До истечения `VAD_COMMAND_MIN_LISTEN_MS` окно не закрывается — пользователю даётся время начать говорить.
+- При достижении `VAD_COMMAND_MAX_LISTEN_MS` срабатывает hard-cap и управление возвращается в режим ожидания wake-word.
+- На событии «закрыть окно» вызывается `stt::finalize_speech()` — Vosk форсированно отдаёт финальный результат, не дожидаясь собственного таймаута.
+
+Параметры (`crates/jarvis-core/src/config.rs`):
+
+| Константа                       | Значение | Назначение                                                  |
+|---------------------------------|----------|-------------------------------------------------------------|
+| `VAD_AGGRESSIVENESS`            | `2`      | Уровень WebRTC VAD: 0 — Quality, 3 — VeryAggressive.        |
+| `VAD_COMMAND_END_SILENCE_MS`    | `1200`   | Сколько тишины подряд считать «команда закончилась».        |
+| `VAD_COMMAND_MIN_SPEECH_MS`     | `500`    | Минимум речи в окне — иначе закрытие игнорируется.          |
+| `VAD_COMMAND_MIN_LISTEN_MS`     | `1000`   | Минимальная длительность окна (страховка от ранних закрытий)|
+| `VAD_COMMAND_MAX_LISTEN_MS`     | `15000`  | Жёсткий потолок окна.                                       |
+
+На короткие команды отзыв стал заметно быстрее (мы выходим на распознавание сразу после паузы пользователя, а не по 5-секундному порогу Vosk).
+
 ## Лицензия
 
 Creative Commons **Attribution-NonCommercial-ShareAlike 4.0 International** (CC BY-NC-SA 4.0).

crates/jarvis-app/src/app.rs

@@ -1,7 +1,7 @@
 use std::sync::mpsc::Receiver;
 use std::time::SystemTime;
 
-use jarvis_core::{audio_buffer::AudioRingBuffer, audio_processing, commands, config, listener, recorder, stt, COMMANDS_LIST, intent, voices, ipc::{self, IpcEvent}, i18n, slots};
+use jarvis_core::{audio_buffer::AudioRingBuffer, audio_processing, audio_processing::vad::listen_window::{ListenWindow, WindowDecision}, audio_processing::vad::webrtc::WebRtcVad, commands, config, listener, recorder, stt, COMMANDS_LIST, intent, voices, ipc::{self, IpcEvent}, i18n, slots};
 use rand::seq::SliceRandom;
 
 use crate::should_stop;
@@ -159,9 +159,10 @@ fn recognize_command(
     let mut silence_frames: u32 = 0;
     let mut start = SystemTime::now();
     let mut first_recognition = prefed_audio;
-    
+
-    // longer silence threshold for commands (user might pause to think)
+    let mut webrtc_vad = WebRtcVad::with_aggressiveness(config::VAD_AGGRESSIVENESS);
-    // 5 seconds
+    let mut window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
+
     let silence_threshold: u32 = ((5.0 * sample_rate as f32) / frame_length as f32) as u32;
     
     loop {
@@ -171,11 +172,40 @@ fn recognize_command(
         
         recorder::read_microphone(frame_buffer);
         let processed = audio_processing::process(frame_buffer);
-        
+
+        let mut vad_finalized: Option<String> = None;
+        let mut vad_hard_cap = false;
+        for is_speech in webrtc_vad.push_samples(frame_buffer) {
+            match window.push(is_speech) {
+                WindowDecision::KeepListening => {}
+                WindowDecision::Close => {
+                    if window.had_speech() {
+                        info!(
+                            "VAD: silence after speech ({} ms speech, {} ms silence), finalizing.",
+                            window.speech_ms(), window.silence_ms()
+                        );
+                        vad_finalized = stt::finalize_speech();
+                        if vad_finalized.is_none() {
+                            return;
+                        }
+                        break;
+                    }
+                }
+                WindowDecision::HardCap => {
+                    info!("VAD: hard cap reached ({} ms), returning to wake word mode.", window.elapsed_ms());
+                    vad_hard_cap = true;
+                    break;
+                }
+            }
+        }
+        if vad_hard_cap {
+            return;
+        }
+
         match vad_state {
             VadState::WaitingForVoice => {
                 audio_buffer.push(frame_buffer);
-                
+
                 if processed.is_voice {
                     // flush buffer to STT
                     for buffered_frame in audio_buffer.drain_all() {
@@ -185,7 +215,7 @@ fn recognize_command(
                     silence_frames = 0;
                 } else {
                     silence_frames += 1;
-                    
+
                     if silence_frames > silence_threshold {
                         info!("Long silence detected, returning to wake word mode.");
                         return;
@@ -194,8 +224,9 @@ fn recognize_command(
             }
             
             VadState::VoiceActive => {
-                // feed to STT
+                // feed to STT (or use VAD-forced finalization)
-                if let Some(mut recognized_voice) = stt::recognize(frame_buffer, false) {
+                let recognized = vad_finalized.take().or_else(|| stt::recognize(frame_buffer, false));
+                if let Some(mut recognized_voice) = recognized {
                     info!("Recognized voice: {}", recognized_voice);
                     
                     ipc::send(IpcEvent::SpeechRecognized {
@@ -227,6 +258,8 @@ fn recognize_command(
                                 silence_frames = 0;
                                 start = SystemTime::now();
                                 audio_buffer.clear();
+                                webrtc_vad.reset();
+                                window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
                                 continue;
                             }
 
@@ -235,11 +268,13 @@ fn recognize_command(
                             voices::play_reply();
                             stt::reset_speech_recognizer();
                             ipc::send(IpcEvent::Listening);
-                            
+
                             vad_state = VadState::WaitingForVoice;
                             silence_frames = 0;
                             start = SystemTime::now();
                             audio_buffer.clear();
+                            webrtc_vad.reset();
+                            window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
                             continue;
                         } else {
                             // wake word + command in one phrase - execute the command part
@@ -259,6 +294,8 @@ fn recognize_command(
                             silence_frames = 0;
                             start = SystemTime::now();
                             audio_buffer.clear();
+                            webrtc_vad.reset();
+                            window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
                             ipc::send(IpcEvent::Listening);
                             continue;
                         }
@@ -294,6 +331,8 @@ fn recognize_command(
                         silence_frames = 0;
                         start = SystemTime::now();
                         audio_buffer.clear();
+                        webrtc_vad.reset();
+                        window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
                         ipc::send(IpcEvent::Listening);
                         continue;
                     } else {

crates/jarvis-core/Cargo.toml

@@ -24,6 +24,7 @@ parking_lot.workspace = true
 toml.workspace = true
 sha2.workspace = true
 nnnoiseless = { workspace = true, optional = true }
+webrtc-vad = { workspace = true, optional = true }
 tokio-tungstenite = { workspace = true, optional = true }
 futures-util = { workspace = true, optional = true }
 fluent.workspace = true
@@ -58,7 +59,9 @@ default = ["jarvis_app"]
 jarvis_app = [
     "vosk", "intent-classifier", "fastembed", "tokio", "nnnoiseless", "tokio-tungstenite", "futures-util",
     "lua", "llm",
-    "ort", "ndarray", "tokenizers", "regex",]
+    "ort", "ndarray", "tokenizers", "regex",
+    "webrtc-vad",
+]
 
 intent = ["intent-classifier", "tokio"]
 lua = ["mlua", "reqwest", "winrt-notification"]

crates/jarvis-core/src/audio_processing/vad.rs

@@ -1,5 +1,8 @@
 mod none;
 mod energy;
+pub mod listen_window;
+#[cfg(feature = "webrtc-vad")]
+pub mod webrtc;
 
 use once_cell::sync::OnceCell;
 use parking_lot::Mutex;

crates/jarvis-core/src/audio_processing/vad/listen_window.rs

@@ -0,0 +1,135 @@
+use crate::config;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum WindowDecision {
+    KeepListening,
+    Close,
+    HardCap,
+}
+
+#[derive(Debug, Clone)]
+pub struct ListenWindow {
+    frame_ms: u32,
+    elapsed_ms: u32,
+    speech_ms: u32,
+    silence_ms: u32,
+    end_silence_ms: u32,
+    min_speech_ms: u32,
+    min_listen_ms: u32,
+    max_listen_ms: u32,
+}
+
+impl ListenWindow {
+    pub fn new(frame_ms: u32) -> Self {
+        Self::with_params(
+            frame_ms,
+            config::VAD_COMMAND_END_SILENCE_MS,
+            config::VAD_COMMAND_MIN_SPEECH_MS,
+            config::VAD_COMMAND_MIN_LISTEN_MS,
+            config::VAD_COMMAND_MAX_LISTEN_MS,
+        )
+    }
+
+    pub fn with_params(
+        frame_ms: u32,
+        end_silence_ms: u32,
+        min_speech_ms: u32,
+        min_listen_ms: u32,
+        max_listen_ms: u32,
+    ) -> Self {
+        Self {
+            frame_ms,
+            elapsed_ms: 0,
+            speech_ms: 0,
+            silence_ms: 0,
+            end_silence_ms,
+            min_speech_ms,
+            min_listen_ms,
+            max_listen_ms,
+        }
+    }
+
+    pub fn push(&mut self, is_speech: bool) -> WindowDecision {
+        self.elapsed_ms = self.elapsed_ms.saturating_add(self.frame_ms);
+
+        if is_speech {
+            self.speech_ms = self.speech_ms.saturating_add(self.frame_ms);
+            self.silence_ms = 0;
+        } else {
+            self.silence_ms = self.silence_ms.saturating_add(self.frame_ms);
+        }
+
+        if self.elapsed_ms >= self.max_listen_ms {
+            return WindowDecision::HardCap;
+        }
+        if self.elapsed_ms < self.min_listen_ms {
+            return WindowDecision::KeepListening;
+        }
+        if self.speech_ms >= self.min_speech_ms && self.silence_ms >= self.end_silence_ms {
+            return WindowDecision::Close;
+        }
+        WindowDecision::KeepListening
+    }
+
+    pub fn elapsed_ms(&self) -> u32 { self.elapsed_ms }
+    pub fn speech_ms(&self) -> u32 { self.speech_ms }
+    pub fn silence_ms(&self) -> u32 { self.silence_ms }
+
+    pub fn had_speech(&self) -> bool { self.speech_ms > 0 }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn run(verdicts: &[bool]) -> (WindowDecision, usize) {
+        let mut w = ListenWindow::with_params(30, 1200, 500, 1000, 15000);
+        for (i, &v) in verdicts.iter().enumerate() {
+            let d = w.push(v);
+            if d != WindowDecision::KeepListening {
+                return (d, i + 1);
+            }
+        }
+        (WindowDecision::KeepListening, verdicts.len())
+    }
+
+    #[test]
+    fn closes_after_speech_then_silence() {
+        let speech: Vec<bool> = std::iter::repeat(true).take(40).collect();
+        let silence: Vec<bool> = std::iter::repeat(false).take(60).collect();
+        let mut v = speech;
+        v.extend(silence);
+
+        let (decision, frame_idx) = run(&v);
+        assert_eq!(decision, WindowDecision::Close);
+        let elapsed_ms = (frame_idx as u32) * 30;
+        let speech_ms = 40 * 30;
+        assert!(elapsed_ms >= speech_ms + 1200, "closed at {} ms, expected >= {}", elapsed_ms, speech_ms + 1200);
+        assert!(elapsed_ms <= speech_ms + 1200 + 30, "closed too late at {} ms", elapsed_ms);
+    }
+
+    #[test]
+    fn does_not_close_before_min_listen_even_with_long_silence() {
+        let v: Vec<bool> = std::iter::repeat(false).take(100).collect();
+        let mut w = ListenWindow::with_params(30, 1200, 500, 1000, 15000);
+        for &x in v.iter().take(33) {
+            assert_eq!(w.push(x), WindowDecision::KeepListening);
+        }
+        assert!(w.elapsed_ms() <= 1000);
+    }
+
+    #[test]
+    fn hard_cap_fires_at_max_listen() {
+        let v: Vec<bool> = std::iter::repeat(true).take(1000).collect();
+        let (decision, frame_idx) = run(&v);
+        assert_eq!(decision, WindowDecision::HardCap);
+        assert_eq!((frame_idx as u32) * 30, 15000);
+    }
+
+    #[test]
+    fn silence_only_does_not_close_keeps_waiting_until_hard_cap() {
+        let v: Vec<bool> = std::iter::repeat(false).take(600).collect();
+        let (decision, _) = run(&v);
+        assert_eq!(decision, WindowDecision::HardCap);
+    }
+}

crates/jarvis-core/src/audio_processing/vad/webrtc.rs

@@ -0,0 +1,49 @@
+use webrtc_vad::{SampleRate, Vad, VadMode};
+
+use crate::config;
+
+pub const FRAME_SAMPLES: usize = 480;
+pub const FRAME_MS: u32 = 30;
+
+pub struct WebRtcVad {
+    inner: Vad,
+    buf: Vec<i16>,
+}
+
+impl WebRtcVad {
+    pub fn new() -> Self {
+        Self::with_aggressiveness(config::VAD_AGGRESSIVENESS)
+    }
+
+    pub fn with_aggressiveness(level: u8) -> Self {
+        let mode = match level {
+            0 => VadMode::Quality,
+            1 => VadMode::LowBitrate,
+            2 => VadMode::Aggressive,
+            _ => VadMode::VeryAggressive,
+        };
+        Self {
+            inner: Vad::new_with_rate_and_mode(SampleRate::Rate16kHz, mode),
+            buf: Vec::with_capacity(FRAME_SAMPLES * 2),
+        }
+    }
+
+    pub fn push_samples(&mut self, samples: &[i16]) -> Vec<bool> {
+        self.buf.extend_from_slice(samples);
+        let mut out = Vec::new();
+        while self.buf.len() >= FRAME_SAMPLES {
+            let frame: Vec<i16> = self.buf.drain(..FRAME_SAMPLES).collect();
+            let is_speech = self.inner.is_voice_segment(&frame).unwrap_or(false);
+            out.push(is_speech);
+        }
+        out
+    }
+
+    pub fn reset(&mut self) {
+        self.buf.clear();
+    }
+}
+
+impl Default for WebRtcVad {
+    fn default() -> Self { Self::new() }
+}

crates/jarvis-core/src/config.rs

@@ -171,6 +171,13 @@ pub const VAD_ENERGY_THRESHOLD: f32 = 100.0;  // RMS threshold for energy-based
 pub const VAD_NNNOISELESS_THRESHOLD: f32 = 0.8;  // probability threshold for nnnoiseless
 pub const VAD_SILENCE_FRAMES: u32 = 15;  // frames of silence before speech end (~480ms)
 
+// post-wake command listening window (паритет с Python v0.2.0, webrtcvad)
+pub const VAD_AGGRESSIVENESS: u8 = 2;
+pub const VAD_COMMAND_END_SILENCE_MS: u32 = 1200;
+pub const VAD_COMMAND_MIN_SPEECH_MS: u32 = 500;
+pub const VAD_COMMAND_MIN_LISTEN_MS: u32 = 1000;
+pub const VAD_COMMAND_MAX_LISTEN_MS: u32 = 15000;
+
 // gain normalizer settings
 pub const GAIN_TARGET_RMS: f32 = 3000.0;  // target RMS level
 pub const GAIN_MIN: f32 = 0.5;  // minimum gain multiplier

crates/jarvis-core/src/stt.rs

@@ -10,6 +10,7 @@ pub use self::vosk::recognize_wake_word;
 pub use self::vosk::recognize_speech;
 pub use self::vosk::reset_speech_recognizer;
 pub use self::vosk::reset_wake_recognizer;
+pub use self::vosk::finalize_speech;
 
 static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new();
 

crates/jarvis-core/src/stt/vosk.rs

@@ -97,6 +97,16 @@ pub fn reset_speech_recognizer() {
     }
 }
 
+pub fn finalize_speech() -> Option<String> {
+    let mut recognizer = SPEECH_RECOGNIZER.get()?.lock();
+    let result = recognizer.final_result()
+        .multiple()
+        .and_then(|m| m.alternatives.first().map(|a| a.text.to_string()))
+        .filter(|s| !s.trim().is_empty());
+    recognizer.reset();
+    result
+}
+
 pub fn reset_wake_recognizer() {
     if let Some(recognizer) = WAKE_RECOGNIZER.get() {
         recognizer.lock().reset();