From 37c8371f3f856ad44f0b90001c6aaf41ba7582ba Mon Sep 17 00:00:00 2001 From: Bossiara13 <236771060+DmitryBykov-ISPO@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:07:35 +0300 Subject: [PATCH] feat(core): vad-driven listening window state machine --- .../jarvis-core/src/audio_processing/vad.rs | 3 + .../src/audio_processing/vad/listen_window.rs | 79 +++++++++++++++++++ .../src/audio_processing/vad/webrtc.rs | 49 ++++++++++++ crates/jarvis-core/src/config.rs | 7 ++ crates/jarvis-core/src/stt.rs | 1 + crates/jarvis-core/src/stt/vosk.rs | 10 +++ 6 files changed, 149 insertions(+) create mode 100644 crates/jarvis-core/src/audio_processing/vad/listen_window.rs create mode 100644 crates/jarvis-core/src/audio_processing/vad/webrtc.rs diff --git a/crates/jarvis-core/src/audio_processing/vad.rs b/crates/jarvis-core/src/audio_processing/vad.rs index a2b57c5..ba86c1a 100644 --- a/crates/jarvis-core/src/audio_processing/vad.rs +++ b/crates/jarvis-core/src/audio_processing/vad.rs @@ -1,5 +1,8 @@ mod none; mod energy; +pub mod listen_window; +#[cfg(feature = "webrtc-vad")] +pub mod webrtc; use once_cell::sync::OnceCell; use parking_lot::Mutex; diff --git a/crates/jarvis-core/src/audio_processing/vad/listen_window.rs b/crates/jarvis-core/src/audio_processing/vad/listen_window.rs new file mode 100644 index 0000000..eb8c82c --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/vad/listen_window.rs @@ -0,0 +1,79 @@ +use crate::config; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WindowDecision { + KeepListening, + Close, + HardCap, +} + +#[derive(Debug, Clone)] +pub struct ListenWindow { + frame_ms: u32, + elapsed_ms: u32, + speech_ms: u32, + silence_ms: u32, + end_silence_ms: u32, + min_speech_ms: u32, + min_listen_ms: u32, + max_listen_ms: u32, +} + +impl ListenWindow { + pub fn new(frame_ms: u32) -> Self { + Self::with_params( + frame_ms, + config::VAD_COMMAND_END_SILENCE_MS, + config::VAD_COMMAND_MIN_SPEECH_MS, + config::VAD_COMMAND_MIN_LISTEN_MS, + config::VAD_COMMAND_MAX_LISTEN_MS, + ) + } + + pub fn with_params( + frame_ms: u32, + end_silence_ms: u32, + min_speech_ms: u32, + min_listen_ms: u32, + max_listen_ms: u32, + ) -> Self { + Self { + frame_ms, + elapsed_ms: 0, + speech_ms: 0, + silence_ms: 0, + end_silence_ms, + min_speech_ms, + min_listen_ms, + max_listen_ms, + } + } + + pub fn push(&mut self, is_speech: bool) -> WindowDecision { + self.elapsed_ms = self.elapsed_ms.saturating_add(self.frame_ms); + + if is_speech { + self.speech_ms = self.speech_ms.saturating_add(self.frame_ms); + self.silence_ms = 0; + } else { + self.silence_ms = self.silence_ms.saturating_add(self.frame_ms); + } + + if self.elapsed_ms >= self.max_listen_ms { + return WindowDecision::HardCap; + } + if self.elapsed_ms < self.min_listen_ms { + return WindowDecision::KeepListening; + } + if self.speech_ms >= self.min_speech_ms && self.silence_ms >= self.end_silence_ms { + return WindowDecision::Close; + } + WindowDecision::KeepListening + } + + pub fn elapsed_ms(&self) -> u32 { self.elapsed_ms } + pub fn speech_ms(&self) -> u32 { self.speech_ms } + pub fn silence_ms(&self) -> u32 { self.silence_ms } + + pub fn had_speech(&self) -> bool { self.speech_ms > 0 } +} diff --git a/crates/jarvis-core/src/audio_processing/vad/webrtc.rs b/crates/jarvis-core/src/audio_processing/vad/webrtc.rs new file mode 100644 index 0000000..28e09f8 --- /dev/null +++ b/crates/jarvis-core/src/audio_processing/vad/webrtc.rs @@ -0,0 +1,49 @@ +use webrtc_vad::{SampleRate, Vad, VadMode}; + +use crate::config; + +pub const FRAME_SAMPLES: usize = 480; +pub const FRAME_MS: u32 = 30; + +pub struct WebRtcVad { + inner: Vad, + buf: Vec, +} + +impl WebRtcVad { + pub fn new() -> Self { + Self::with_aggressiveness(config::VAD_AGGRESSIVENESS) + } + + pub fn with_aggressiveness(level: u8) -> Self { + let mode = match level { + 0 => VadMode::Quality, + 1 => VadMode::LowBitrate, + 2 => VadMode::Aggressive, + _ => VadMode::VeryAggressive, + }; + Self { + inner: Vad::new_with_rate_and_mode(SampleRate::Rate16kHz, mode), + buf: Vec::with_capacity(FRAME_SAMPLES * 2), + } + } + + pub fn push_samples(&mut self, samples: &[i16]) -> Vec { + self.buf.extend_from_slice(samples); + let mut out = Vec::new(); + while self.buf.len() >= FRAME_SAMPLES { + let frame: Vec = self.buf.drain(..FRAME_SAMPLES).collect(); + let is_speech = self.inner.is_voice_segment(&frame).unwrap_or(false); + out.push(is_speech); + } + out + } + + pub fn reset(&mut self) { + self.buf.clear(); + } +} + +impl Default for WebRtcVad { + fn default() -> Self { Self::new() } +} diff --git a/crates/jarvis-core/src/config.rs b/crates/jarvis-core/src/config.rs index 4e525b1..f7b55a4 100644 --- a/crates/jarvis-core/src/config.rs +++ b/crates/jarvis-core/src/config.rs @@ -171,6 +171,13 @@ pub const VAD_ENERGY_THRESHOLD: f32 = 100.0; // RMS threshold for energy-based pub const VAD_NNNOISELESS_THRESHOLD: f32 = 0.8; // probability threshold for nnnoiseless pub const VAD_SILENCE_FRAMES: u32 = 15; // frames of silence before speech end (~480ms) +// post-wake command listening window (паритет с Python v0.2.0, webrtcvad) +pub const VAD_AGGRESSIVENESS: u8 = 2; +pub const VAD_COMMAND_END_SILENCE_MS: u32 = 1200; +pub const VAD_COMMAND_MIN_SPEECH_MS: u32 = 500; +pub const VAD_COMMAND_MIN_LISTEN_MS: u32 = 1000; +pub const VAD_COMMAND_MAX_LISTEN_MS: u32 = 15000; + // gain normalizer settings pub const GAIN_TARGET_RMS: f32 = 3000.0; // target RMS level pub const GAIN_MIN: f32 = 0.5; // minimum gain multiplier diff --git a/crates/jarvis-core/src/stt.rs b/crates/jarvis-core/src/stt.rs index 08429ba..db59dba 100644 --- a/crates/jarvis-core/src/stt.rs +++ b/crates/jarvis-core/src/stt.rs @@ -10,6 +10,7 @@ pub use self::vosk::recognize_wake_word; pub use self::vosk::recognize_speech; pub use self::vosk::reset_speech_recognizer; pub use self::vosk::reset_wake_recognizer; +pub use self::vosk::finalize_speech; static STT_TYPE: OnceCell = OnceCell::new(); diff --git a/crates/jarvis-core/src/stt/vosk.rs b/crates/jarvis-core/src/stt/vosk.rs index a6aa771..2e59206 100644 --- a/crates/jarvis-core/src/stt/vosk.rs +++ b/crates/jarvis-core/src/stt/vosk.rs @@ -97,6 +97,16 @@ pub fn reset_speech_recognizer() { } } +pub fn finalize_speech() -> Option { + let mut recognizer = SPEECH_RECOGNIZER.get()?.lock(); + let result = recognizer.final_result() + .multiple() + .and_then(|m| m.alternatives.first().map(|a| a.text.to_string())) + .filter(|s| !s.trim().is_empty()); + recognizer.reset(); + result +} + pub fn reset_wake_recognizer() { if let Some(recognizer) = WAKE_RECOGNIZER.get() { recognizer.lock().reset();