noise suppression added via nnnoiseless + vad + gain-normalizer + few frontend changes

crates/jarvis-core/Cargo.toml

@@ -23,6 +23,7 @@ rustpotter.workspace = true
 parking_lot.workspace = true
 toml.workspace = true
 sha2.workspace = true
+nnnoiseless = { workspace = true, optional = true }
 
 # pv_recorder = { workspace = true, optional = true }
 vosk = { version = "0.3.1", optional = true }
@@ -33,5 +34,5 @@ tokio = { version = "1", features = ["sync"], optional = true }
 
 [features]
 default = ["jarvis_app"]
-jarvis_app = ["vosk", "intent-classifier", "tokio"]
+jarvis_app = ["vosk", "intent-classifier", "tokio", "nnnoiseless"]
 intent = ["intent-classifier", "tokio"]

crates/jarvis-core/src/audio_processing.rs

@@ -0,0 +1,118 @@
+pub mod noise_suppression;
+pub mod vad;
+pub mod gain_normalizer;
+
+use once_cell::sync::OnceCell;
+use std::sync::Mutex;
+
+use crate::config::structs::{NoiseSuppressionBackend, VadBackend};
+use crate::DB;
+
+static PROCESSOR: OnceCell<Mutex<AudioProcessor>> = OnceCell::new();
+
+#[derive(Debug, Clone)]
+pub struct ProcessedAudio {
+    pub samples: Vec<i16>,
+    pub is_voice: bool,
+    pub vad_confidence: f32,
+}
+
+struct AudioProcessor {
+    ns_backend: NoiseSuppressionBackend,
+    vad_backend: VadBackend,
+    gain_enabled: bool,
+}
+
+impl AudioProcessor {
+    fn new(ns: NoiseSuppressionBackend, vad: VadBackend, gain: bool) -> Self {
+        // init backends
+        noise_suppression::init(ns);
+        vad::init(vad);
+        if gain {
+            gain_normalizer::init();
+        }
+
+        Self {
+            ns_backend: ns,
+            vad_backend: vad,
+            gain_enabled: gain,
+        }
+    }
+
+    fn process(&mut self, input: &[i16]) -> ProcessedAudio {
+        let mut samples = input.to_vec();
+
+        // step 1: gain normalization (before other processing)
+        if self.gain_enabled {
+            samples = gain_normalizer::normalize(&samples);
+        }
+
+        // step 2: noise suppression
+        samples = noise_suppression::process(&samples);
+
+        // step 3: VAD
+        let (is_voice, confidence) = vad::detect(&samples);
+
+        ProcessedAudio {
+            samples,
+            is_voice,
+            vad_confidence: confidence,
+        }
+    }
+
+    fn reset(&mut self) {
+        noise_suppression::reset();
+        vad::reset();
+        gain_normalizer::reset();
+    }
+}
+
+
+
+pub fn init() -> Result<(), String> {
+    if PROCESSOR.get().is_some() {
+        return Ok(());
+    }
+
+    let (ns, vad, gain) = get_settings();
+    info!("Initializing audio processing: NS={:?}, VAD={:?}, Gain={}", ns, vad, gain);
+
+    let processor = AudioProcessor::new(ns, vad, gain);
+    PROCESSOR
+        .set(Mutex::new(processor))
+        .map_err(|_| "Audio processor already initialized")?;
+
+    info!("Audio processing initialized.");
+    Ok(())
+}
+
+pub fn process(input: &[i16]) -> ProcessedAudio {
+    match PROCESSOR.get() {
+        Some(p) => p.lock().unwrap().process(input),
+        None => ProcessedAudio {
+            samples: input.to_vec(),
+            is_voice: true,
+            vad_confidence: 1.0,
+        },
+    }
+}
+
+pub fn reset() {
+    if let Some(p) = PROCESSOR.get() {
+        p.lock().unwrap().reset();
+    }
+}
+
+fn get_settings() -> (NoiseSuppressionBackend, VadBackend, bool) {
+    match DB.get() {
+        Some(db) => {
+            let settings = db.read();
+            (settings.noise_suppression, settings.vad, settings.gain_normalizer)
+        }
+        None => (
+            crate::config::DEFAULT_NOISE_SUPPRESSION,
+            crate::config::DEFAULT_VAD,
+            crate::config::DEFAULT_GAIN_NORMALIZER,
+        ),
+    }
+}

crates/jarvis-core/src/audio_processing/gain_normalizer.rs

@@ -0,0 +1,28 @@
+mod simple;
+
+use once_cell::sync::OnceCell;
+use std::sync::Mutex;
+
+static NORMALIZER: OnceCell<Mutex<simple::GainNormalizer>> = OnceCell::new();
+
+pub fn init() {
+    if NORMALIZER.get().is_some() {
+        return;
+    }
+
+    NORMALIZER.set(Mutex::new(simple::GainNormalizer::new())).ok();
+    info!("Gain normalizer: enabled");
+}
+
+pub fn normalize(input: &[i16]) -> Vec<i16> {
+    match NORMALIZER.get() {
+        Some(n) => n.lock().unwrap().normalize(input),
+        None => input.to_vec(),
+    }
+}
+
+pub fn reset() {
+    if let Some(n) = NORMALIZER.get() {
+        n.lock().unwrap().reset();
+    }
+}

crates/jarvis-core/src/audio_processing/gain_normalizer/simple.rs

@@ -0,0 +1,47 @@
+use crate::config;
+
+pub struct GainNormalizer {
+    current_gain: f32,
+}
+
+impl GainNormalizer {
+    pub fn new() -> Self {
+        Self { current_gain: 1.0 }
+    }
+
+    pub fn normalize(&mut self, input: &[i16]) -> Vec<i16> {
+        let rms = self.calculate_rms(input);
+        
+        if rms < 1.0 {
+            return input.to_vec();
+        }
+
+        let target_gain = config::GAIN_TARGET_RMS / rms;
+        let clamped_gain = target_gain.clamp(config::GAIN_MIN, config::GAIN_MAX);
+
+        self.current_gain = self.current_gain * 0.9 + clamped_gain * 0.1;
+
+        input.iter()
+            .map(|&s| {
+                let amplified = (s as f32) * self.current_gain;
+                amplified.clamp(i16::MIN as f32, i16::MAX as f32) as i16
+            })
+            .collect()
+    }
+
+    pub fn reset(&mut self) {
+        self.current_gain = 1.0;
+    }
+
+    fn calculate_rms(&self, samples: &[i16]) -> f32 {
+        if samples.is_empty() {
+            return 0.0;
+        }
+
+        let sum: f64 = samples.iter()
+            .map(|&s| (s as f64).powi(2))
+            .sum();
+
+        (sum / samples.len() as f64).sqrt() as f32
+    }
+}

crates/jarvis-core/src/audio_processing/noise_suppression.rs

@@ -0,0 +1,66 @@
+mod none;
+
+#[cfg(feature = "nnnoiseless")]
+mod nnnoiseless;
+
+use once_cell::sync::OnceCell;
+use std::sync::Mutex;
+
+use crate::config::structs::NoiseSuppressionBackend;
+
+static BACKEND: OnceCell<NoiseSuppressionBackend> = OnceCell::new();
+
+#[cfg(feature = "nnnoiseless")]
+static NNNOISELESS_STATE: OnceCell<Mutex<nnnoiseless::NnnoiselessNS>> = OnceCell::new();
+
+pub fn init(backend: NoiseSuppressionBackend) {
+    if BACKEND.get().is_some() {
+        return;
+    }
+
+    BACKEND.set(backend).ok();
+
+    match backend {
+        NoiseSuppressionBackend::None => {
+            info!("Noise suppression: disabled");
+        }
+        #[cfg(feature = "nnnoiseless")]
+        NoiseSuppressionBackend::Nnnoiseless => {
+            NNNOISELESS_STATE.set(Mutex::new(nnnoiseless::NnnoiselessNS::new())).ok();
+            info!("Noise suppression: Nnnoiseless");
+        }
+        #[cfg(not(feature = "nnnoiseless"))]
+        NoiseSuppressionBackend::Nnnoiseless => {
+            warn!("Nnnoiseless not compiled in, falling back to None");
+            BACKEND.set(NoiseSuppressionBackend::None).ok();
+        }
+    }
+}
+
+pub fn process(input: &[i16]) -> Vec<i16> {
+    match BACKEND.get() {
+        Some(NoiseSuppressionBackend::None) | None => none::process(input),
+        #[cfg(feature = "nnnoiseless")]
+        Some(NoiseSuppressionBackend::Nnnoiseless) => {
+            if let Some(state) = NNNOISELESS_STATE.get() {
+                state.lock().unwrap().process(input)
+            } else {
+                none::process(input)
+            }
+        }
+        #[cfg(not(feature = "nnnoiseless"))]
+        Some(NoiseSuppressionBackend::Nnnoiseless) => none::process(input),
+    }
+}
+
+pub fn reset() {
+    match BACKEND.get() {
+        #[cfg(feature = "nnnoiseless")]
+        Some(NoiseSuppressionBackend::Nnnoiseless) => {
+            if let Some(state) = NNNOISELESS_STATE.get() {
+                state.lock().unwrap().reset();
+            }
+        }
+        _ => {}
+    }
+}

crates/jarvis-core/src/audio_processing/noise_suppression/nnnoiseless.rs

@@ -0,0 +1,51 @@
+use nnnoiseless::DenoiseState;
+use crate::config;
+
+pub struct NnnoiselessNS {
+    state: Box<DenoiseState<'static>>,
+    buffer: Vec<f32>,
+}
+
+impl NnnoiselessNS {
+    pub fn new() -> Self {
+        Self {
+            state: DenoiseState::new(),
+            buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
+        }
+    }
+
+    pub fn process(&mut self, input: &[i16]) -> Vec<i16> {
+        for &sample in input {
+            self.buffer.push(sample as f32);
+        }
+
+        let mut output: Vec<i16> = Vec::with_capacity(input.len());
+
+        while self.buffer.len() >= config::NNNOISELESS_FRAME_SIZE {
+            let mut input_frame = [0.0f32; 480];
+            let mut output_frame = [0.0f32; 480];
+            
+            input_frame.copy_from_slice(&self.buffer[..config::NNNOISELESS_FRAME_SIZE]);
+            self.buffer.drain(..config::NNNOISELESS_FRAME_SIZE);
+
+            // process: input -> output (denoised)
+            let _ = self.state.process_frame(&mut output_frame, &input_frame);
+
+            for &sample in &output_frame {
+                let clamped = sample.clamp(i16::MIN as f32, i16::MAX as f32);
+                output.push(clamped as i16);
+            }
+        }
+
+        if output.is_empty() {
+            return input.to_vec();
+        }
+
+        output
+    }
+
+    pub fn reset(&mut self) {
+        self.state = DenoiseState::new();
+        self.buffer.clear();
+    }
+}

crates/jarvis-core/src/audio_processing/noise_suppression/none.rs

@@ -0,0 +1,4 @@
+// return unprocessed input
+pub fn process(input: &[i16]) -> Vec<i16> {
+    input.to_vec()
+}

crates/jarvis-core/src/audio_processing/vad.rs

@@ -0,0 +1,72 @@
+mod none;
+mod energy;
+
+#[cfg(feature = "nnnoiseless")]
+mod nnnoiseless;
+
+use once_cell::sync::OnceCell;
+use std::sync::Mutex;
+
+use crate::config::structs::VadBackend;
+
+static BACKEND: OnceCell<VadBackend> = OnceCell::new();
+
+#[cfg(feature = "nnnoiseless")]
+static NNNOISELESS_STATE: OnceCell<Mutex<nnnoiseless::NnnoiselessVAD>> = OnceCell::new();
+
+pub fn init(backend: VadBackend) {
+    if BACKEND.get().is_some() {
+        return;
+    }
+
+    BACKEND.set(backend).ok();
+
+    match backend {
+        VadBackend::None => {
+            info!("VAD: disabled");
+        }
+        VadBackend::Energy => {
+            info!("VAD: Energy-based");
+        }
+        #[cfg(feature = "nnnoiseless")]
+        VadBackend::Nnnoiseless => {
+            NNNOISELESS_STATE.set(Mutex::new(nnnoiseless::NnnoiselessVAD::new())).ok();
+            info!("VAD: Nnnoiseless");
+        }
+        #[cfg(not(feature = "nnnoiseless"))]
+        VadBackend::Nnnoiseless => {
+            warn!("Nnnoiseless not compiled in, falling back to Energy");
+            BACKEND.set(VadBackend::Energy).ok();
+        }
+    }
+}
+
+// Returns (is_voice, confidence)
+pub fn detect(input: &[i16]) -> (bool, f32) {
+    match BACKEND.get() {
+        Some(VadBackend::None) | None => none::detect(input),
+        Some(VadBackend::Energy) => energy::detect(input),
+        #[cfg(feature = "nnnoiseless")]
+        Some(VadBackend::Nnnoiseless) => {
+            if let Some(state) = NNNOISELESS_STATE.get() {
+                state.lock().unwrap().detect(input)
+            } else {
+                energy::detect(input)
+            }
+        }
+        #[cfg(not(feature = "nnnoiseless"))]
+        Some(VadBackend::Nnnoiseless) => energy::detect(input),
+    }
+}
+
+pub fn reset() {
+    match BACKEND.get() {
+        #[cfg(feature = "nnnoiseless")]
+        Some(VadBackend::Nnnoiseless) => {
+            if let Some(state) = NNNOISELESS_STATE.get() {
+                state.lock().unwrap().reset();
+            }
+        }
+        _ => {}
+    }
+}

crates/jarvis-core/src/audio_processing/vad/energy.rs

@@ -0,0 +1,24 @@
+use crate::config;
+
+// Simple energy-based VAD
+pub fn detect(input: &[i16]) -> (bool, f32) {
+    let rms = calculate_rms(input);
+    let is_voice = rms > config::VAD_ENERGY_THRESHOLD;
+    
+    // normalize confidence to 0-1 range (rough approximation)
+    let confidence = (rms / (config::VAD_ENERGY_THRESHOLD * 2.0)).min(1.0);
+    
+    (is_voice, confidence)
+}
+
+fn calculate_rms(samples: &[i16]) -> f32 {
+    if samples.is_empty() {
+        return 0.0;
+    }
+    
+    let sum: f64 = samples.iter()
+        .map(|&s| (s as f64).powi(2))
+        .sum();
+    
+    (sum / samples.len() as f64).sqrt() as f32
+}

crates/jarvis-core/src/audio_processing/vad/nnnoiseless.rs

@@ -0,0 +1,51 @@
+use nnnoiseless::DenoiseState;
+use crate::config;
+
+pub struct NnnoiselessVAD {
+    state: Box<DenoiseState<'static>>,
+    buffer: Vec<f32>,
+}
+
+impl NnnoiselessVAD {
+    pub fn new() -> Self {
+        Self {
+            state: DenoiseState::new(),
+            buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
+        }
+    }
+
+    pub fn detect(&mut self, input: &[i16]) -> (bool, f32) {
+        for &sample in input {
+            self.buffer.push(sample as f32);
+        }
+
+        let mut total_vad = 0.0f32;
+        let mut frame_count = 0u32;
+
+        while self.buffer.len() >= config::NNNOISELESS_FRAME_SIZE {
+            let mut input_frame = [0.0f32; 480];
+            let mut output_frame = [0.0f32; 480];
+            
+            input_frame.copy_from_slice(&self.buffer[..config::NNNOISELESS_FRAME_SIZE]);
+            self.buffer.drain(..config::NNNOISELESS_FRAME_SIZE);
+
+            let vad_prob = self.state.process_frame(&mut output_frame, &input_frame);
+            total_vad += vad_prob;
+            frame_count += 1;
+        }
+
+        if frame_count == 0 {
+            return (true, 0.5);
+        }
+
+        let avg_vad = total_vad / frame_count as f32;
+        let is_voice = avg_vad >= config::VAD_NNNOISELESS_THRESHOLD;
+
+        (is_voice, avg_vad)
+    }
+
+    pub fn reset(&mut self) {
+        self.state = DenoiseState::new();
+        self.buffer.clear();
+    }
+}

crates/jarvis-core/src/audio_processing/vad/none.rs

@@ -0,0 +1,4 @@
+// Always returns voice detected (no vad)
+pub fn detect(_input: &[i16]) -> (bool, f32) {
+    (true, 1.0)
+}

crates/jarvis-core/src/config.rs

@@ -18,6 +18,8 @@ use rustpotter::{
 };
 
 use crate::IntentRecognitionEngine;
+use crate::config::structs::NoiseSuppressionBackend;
+use crate::config::structs::VadBackend;
 use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
 
 #[allow(dead_code)]
@@ -106,7 +108,11 @@ pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
         },
         filters: FiltersConfig {
             gain_normalizer: GainNormalizationConfig {
-                enabled: true,
+                // enabled: true,
+                // gain_ref: None,
+                // min_gain: 0.7,
+                // max_gain: 1.0,
+                enabled: false, // disable, now we have separate gain normalizer implementation
                 gain_ref: None,
                 min_gain: 0.7,
                 max_gain: 1.0,
@@ -146,6 +152,26 @@ pub const VOSK_SPEECH_PARTIAL_WORDS: bool = false;
 // IRE (intents recognition)
 pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.75;
 
+
+// AUDIO PROCESSING DEFAULTS
+pub const DEFAULT_NOISE_SUPPRESSION: NoiseSuppressionBackend = NoiseSuppressionBackend::None;
+pub const DEFAULT_VAD: VadBackend = VadBackend::Energy;
+pub const DEFAULT_GAIN_NORMALIZER: bool = false;
+
+// VAD settings
+pub const VAD_ENERGY_THRESHOLD: f32 = 500.0;  // RMS threshold for energy-based VAD
+pub const VAD_NNNOISELESS_THRESHOLD: f32 = 0.5;  // probability threshold for nnnoiseless
+pub const VAD_SILENCE_FRAMES: u32 = 15;  // frames of silence before speech end (~480ms)
+
+// gain normalizer settings
+pub const GAIN_TARGET_RMS: f32 = 3000.0;  // target RMS level
+pub const GAIN_MIN: f32 = 0.5;  // minimum gain multiplier
+pub const GAIN_MAX: f32 = 3.0;  // maximum gain multiplier
+
+// nnnoiseless frame size (fixed by library)
+pub const NNNOISELESS_FRAME_SIZE: usize = 480;
+
+
 // ETC
 pub const CMD_RATIO_THRESHOLD: f64 = 65f64;
 pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);

crates/jarvis-core/src/config/structs.rs

@@ -1,23 +1,30 @@
 use std::fmt;
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
+#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
 pub enum WakeWordEngine {
     Rustpotter,
     Vosk,
     Porcupine,
 }
 
-#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
+#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
 pub enum IntentRecognitionEngine {
     IntentClassifier,
     Rasa,
 }
 
-impl fmt::Display for WakeWordEngine {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{:?}", self)
-    }
+#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
+pub enum NoiseSuppressionBackend {
+    None,
+    Nnnoiseless,
+}
+
+#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
+pub enum VadBackend {
+    None,
+    Energy,
+    Nnnoiseless,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -25,12 +32,6 @@ pub enum SpeechToTextEngine {
     Vosk,
 }
 
-impl fmt::Display for SpeechToTextEngine {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{:?}", self)
-    }
-}
-
 #[derive(PartialEq, Debug)]
 pub enum RecorderType {
     Cpal,
@@ -44,6 +45,38 @@ pub enum AudioType {
     Kira,
 }
 
+
+
+impl fmt::Display for WakeWordEngine {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl fmt::Display for SpeechToTextEngine {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl fmt::Display for IntentRecognitionEngine {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl fmt::Display for NoiseSuppressionBackend {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl fmt::Display for VadBackend {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
 // pub enum TextToSpeechEngine {}
 
 // pub enum IntentRecognitionEngine {}

crates/jarvis-core/src/db/structs.rs

@@ -4,6 +4,8 @@ use serde::{Deserialize, Serialize};
 use crate::config::structs::SpeechToTextEngine;
 use crate::config::structs::WakeWordEngine;
 use crate::config::structs::IntentRecognitionEngine;
+use crate::config::structs::NoiseSuppressionBackend;
+use crate::config::structs::VadBackend;
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Settings {
@@ -13,9 +15,13 @@ pub struct Settings {
     pub wake_word_engine: WakeWordEngine,
     pub intent_recognition_engine: IntentRecognitionEngine,
     pub speech_to_text_engine: SpeechToTextEngine,
-
     pub vosk_model: String,
 
+    // audio processing
+    pub noise_suppression: NoiseSuppressionBackend,
+    pub vad: VadBackend,
+    pub gain_normalizer: bool,
+
     pub api_keys: ApiKeys,
 }
 
@@ -28,9 +34,13 @@ impl Default for Settings {
             wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
             intent_recognition_engine: config::DEFAULT_INTENT_RECOGNITION_ENGINE,
             speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
-
             vosk_model: String::from(""), // auto detect first available
 
+            // audio processing defaults
+            noise_suppression: config::DEFAULT_NOISE_SUPPRESSION,
+            vad: config::DEFAULT_VAD,
+            gain_normalizer: config::DEFAULT_GAIN_NORMALIZER,
+
             api_keys: ApiKeys {
                 picovoice: String::from(""),
                 openai: String::from(""),

crates/jarvis-core/src/lib.rs

@@ -25,6 +25,9 @@ pub mod intent;
 
 pub mod vosk_models;
 
+#[cfg(feature = "jarvis_app")]
+pub mod audio_processing;
+
 // shared statics
 // pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| std::env::current_dir().unwrap());
 pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| {

crates/jarvis-core/src/recorder/pvrecorder.rs

@@ -114,6 +114,10 @@ pub fn list_audio_devices() -> Vec<String> {
 }
 
 pub fn get_audio_device_name(idx: i32) -> String {
+    if idx == -1 {
+        return String::from("System Default");
+    }
+
     let audio_devices = list_audio_devices();
     let mut first_device: String = String::new();