bossiara13/J.A.R.V.I.S-py
1
1
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Activity Actions

AI models shared registry + Code cleanup + Better async handling + Some fixes, etc

Browse source
This commit is contained in:
Priler 2026-02-18 21:08:48 +05:00
parent b427eacf35
commit 5c3031c977
62 changed files with 1683 additions and 1239 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/jarvis-core/Cargo.toml
View file

@ -30,6 +30,7 @@ fluent.workspace = true
fluent-bundle.workspace = true
unic-langid.workspace = true
chrono.workspace = true
sys-locale.workspace = true
# pv_recorder = { workspace = true, optional = true }
vosk = { version = "0.3.1", optional = true }

3
crates/jarvis-core/src/audio.rs
View file

@ -2,7 +2,6 @@ mod kira;
mod rodio;
use once_cell::sync::OnceCell;
use std::cmp::Ordering;
use std::path::PathBuf;
use crate::config::structs::AudioType;
@ -44,7 +43,7 @@ pub fn init() -> Result<(), ()> {
Ok(_) => {
info!("Successfully initialized Kira audio backend.");
}
Err(msg) => {
Err(_msg) => {
error!("Failed to initialize Kira audio backend.");
return Err(());

4
crates/jarvis-core/src/audio/rodio.rs
View file

@ -30,8 +30,8 @@ pub fn init() -> Result<(), ()> {
// store
// STREAM.set(_stream).unwrap();
STREAM_HANDLE.set(stream_handle);
SINK.set(sink);
let _ = STREAM_HANDLE.set(stream_handle);
let _ = SINK.set(sink);
// success
Ok(())

67
crates/jarvis-core/src/audio_processing.rs
View file

@ -3,9 +3,9 @@ pub mod vad;
pub mod gain_normalizer;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
use parking_lot::Mutex;
use crate::config::structs::{NoiseSuppressionBackend, VadBackend};
use crate::config::structs::NoiseSuppressionBackend;
use crate::DB;
static PROCESSOR: OnceCell<Mutex<AudioProcessor>> = OnceCell::new();
@ -18,43 +18,45 @@ pub struct ProcessedAudio {
}
struct AudioProcessor {
ns_backend: NoiseSuppressionBackend,
vad_backend: VadBackend,
gain_enabled: bool,
has_gain: bool,
has_ns: bool,
}
impl AudioProcessor {
fn new(ns: NoiseSuppressionBackend, vad: VadBackend, gain: bool) -> Self {
// init backends
fn new(ns: NoiseSuppressionBackend, gain: bool) -> Self {
noise_suppression::init(ns);
vad::init(vad);
vad::init();
if gain {
gain_normalizer::init();
}
Self {
ns_backend: ns,
vad_backend: vad,
gain_enabled: gain,
has_gain: gain,
has_ns: !matches!(ns, NoiseSuppressionBackend::None),
}
}
fn process(&mut self, input: &[i16]) -> ProcessedAudio {
let mut samples = input.to_vec();
let gained: Vec<i16>;
let after_gain: &[i16] = if self.has_gain {
gained = gain_normalizer::normalize(input);
&gained
} else {
input
};
// step 1: gain normalization (before other processing)
if self.gain_enabled {
samples = gain_normalizer::normalize(&samples);
}
let suppressed: Vec<i16>;
let after_ns: &[i16] = if self.has_ns {
suppressed = noise_suppression::process(after_gain);
&suppressed
} else {
after_gain
};
// step 2: noise suppression
samples = noise_suppression::process(&samples);
// step 3: VAD
let (is_voice, confidence) = vad::detect(&samples);
let (is_voice, confidence) = vad::detect(after_ns);
ProcessedAudio {
samples,
samples: after_ns.to_vec(),
is_voice,
vad_confidence: confidence,
}
@ -67,20 +69,18 @@ impl AudioProcessor {
}
}
pub fn init() -> Result<(), String> {
if PROCESSOR.get().is_some() {
return Ok(());
}
let (ns, vad, gain) = get_settings();
info!("Initializing audio processing: NS={:?}, VAD={:?}, Gain={}", ns, vad, gain);
let (ns, gain) = get_settings();
info!("Initializing audio processing: NS={:?}, Gain={}", ns, gain);
let processor = AudioProcessor::new(ns, vad, gain);
let processor = AudioProcessor::new(ns, gain);
PROCESSOR
.set(Mutex::new(processor))
.map_err(|_| "Audio processor already initialized")?;
.map_err(|_| "Audio processor already initialized".to_string())?;
info!("Audio processing initialized.");
Ok(())
@ -88,7 +88,7 @@ pub fn init() -> Result<(), String> {
pub fn process(input: &[i16]) -> ProcessedAudio {
match PROCESSOR.get() {
Some(p) => p.lock().unwrap().process(input),
Some(p) => p.lock().process(input),
None => ProcessedAudio {
samples: input.to_vec(),
is_voice: true,
@ -99,20 +99,19 @@ pub fn process(input: &[i16]) -> ProcessedAudio {
pub fn reset() {
if let Some(p) = PROCESSOR.get() {
p.lock().unwrap().reset();
p.lock().reset();
}
}
fn get_settings() -> (NoiseSuppressionBackend, VadBackend, bool) {
fn get_settings() -> (NoiseSuppressionBackend, bool) {
match DB.get() {
Some(db) => {
let settings = db.read();
(settings.noise_suppression, settings.vad, settings.gain_normalizer)
(settings.noise_suppression, settings.gain_normalizer)
}
None => (
crate::config::DEFAULT_NOISE_SUPPRESSION,
crate::config::DEFAULT_VAD,
crate::config::DEFAULT_GAIN_NORMALIZER,
),
}
}
}

6
crates/jarvis-core/src/audio_processing/gain_normalizer.rs
View file

@ -1,7 +1,7 @@
mod simple;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
use parking_lot::Mutex;
static NORMALIZER: OnceCell<Mutex<simple::GainNormalizer>> = OnceCell::new();
@ -16,13 +16,13 @@ pub fn init() {
pub fn normalize(input: &[i16]) -> Vec<i16> {
match NORMALIZER.get() {
Some(n) => n.lock().unwrap().normalize(input),
Some(n) => n.lock().normalize(input),
None => input.to_vec(),
}
}
pub fn reset() {
if let Some(n) = NORMALIZER.get() {
n.lock().unwrap().reset();
n.lock().reset();
}
}

31
crates/jarvis-core/src/audio_processing/noise_suppression.rs
View file

@ -1,23 +1,27 @@
mod none;
#[cfg(feature = "nnnoiseless")]
mod nnnoiseless;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
use parking_lot::Mutex;
use crate::config::structs::NoiseSuppressionBackend;
static BACKEND: OnceCell<NoiseSuppressionBackend> = OnceCell::new();
#[cfg(feature = "nnnoiseless")]
static NNNOISELESS_STATE: OnceCell<Mutex<nnnoiseless::NnnoiselessNS>> = OnceCell::new();
static NNNOISELESS_STATE: OnceCell<Mutex<crate::models::nnnoiseless::NnnoiselessNS>> = OnceCell::new();
pub fn init(backend: NoiseSuppressionBackend) {
if BACKEND.get().is_some() {
return;
}
// fallback if nnnoiseless not compiled in
#[cfg(not(feature = "nnnoiseless"))]
if matches!(backend, NoiseSuppressionBackend::Nnnoiseless) {
warn!("Nnnoiseless not compiled in, falling back to None");
backend = NoiseSuppressionBackend::None;
}
BACKEND.set(backend).ok();
match backend {
@ -26,30 +30,25 @@ pub fn init(backend: NoiseSuppressionBackend) {
}
#[cfg(feature = "nnnoiseless")]
NoiseSuppressionBackend::Nnnoiseless => {
NNNOISELESS_STATE.set(Mutex::new(nnnoiseless::NnnoiselessNS::new())).ok();
NNNOISELESS_STATE.set(Mutex::new(crate::models::nnnoiseless::NnnoiselessNS::new())).ok();
info!("Noise suppression: Nnnoiseless");
}
#[cfg(not(feature = "nnnoiseless"))]
NoiseSuppressionBackend::Nnnoiseless => {
warn!("Nnnoiseless not compiled in, falling back to None");
BACKEND.set(NoiseSuppressionBackend::None).ok();
}
_ => {}
}
}
pub fn process(input: &[i16]) -> Vec<i16> {
match BACKEND.get() {
Some(NoiseSuppressionBackend::None) | None => none::process(input),
#[cfg(feature = "nnnoiseless")]
Some(NoiseSuppressionBackend::Nnnoiseless) => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().unwrap().process(input)
state.lock().process(input)
} else {
none::process(input)
}
}
#[cfg(not(feature = "nnnoiseless"))]
Some(NoiseSuppressionBackend::Nnnoiseless) => none::process(input),
_ => none::process(input),
}
}
@ -58,9 +57,9 @@ pub fn reset() {
#[cfg(feature = "nnnoiseless")]
Some(NoiseSuppressionBackend::Nnnoiseless) => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().unwrap().reset();
state.lock().reset();
}
}
_ => {}
}
}
}

53
crates/jarvis-core/src/audio_processing/noise_suppression/nnnoiseless.rs
View file

@ -1,53 +0,0 @@
use nnnoiseless::DenoiseState;
use crate::config;
pub struct NnnoiselessNS {
state: Box<DenoiseState<'static>>,
buffer: Vec<f32>,
}
impl NnnoiselessNS {
pub fn new() -> Self {
Self {
state: DenoiseState::new(),
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
}
}
pub fn process(&mut self, input: &[i16]) -> Vec<i16> {
for &sample in input {
self.buffer.push(sample as f32);
}
let mut output: Vec<i16> = Vec::with_capacity(input.len());
while self.buffer.len() >= config::NNNOISELESS_FRAME_SIZE {
let mut input_frame = [0.0f32; 480];
let mut output_frame = [0.0f32; 480];
input_frame.copy_from_slice(&self.buffer[..config::NNNOISELESS_FRAME_SIZE]);
self.buffer.drain(..config::NNNOISELESS_FRAME_SIZE);
// process: input -> output (denoised)
let _ = self.state.process_frame(&mut output_frame, &input_frame);
for &sample in &output_frame {
let clamped = sample.clamp(i16::MIN as f32, i16::MAX as f32);
output.push(clamped as i16);
}
}
if output.is_empty() {
return input.to_vec();
}
output
}
pub fn reset(&mut self) {
// self.state = DenoiseState::new();
// self.buffer.clear();
self.buffer.clear();
}
}

60
crates/jarvis-core/src/audio_processing/vad.rs
View file

@ -1,72 +1,72 @@
mod none;
mod energy;
#[cfg(feature = "nnnoiseless")]
mod nnnoiseless;
use once_cell::sync::OnceCell;
use std::sync::Mutex;
use parking_lot::Mutex;
use crate::config::structs::VadBackend;
use crate::DB;
static BACKEND: OnceCell<VadBackend> = OnceCell::new();
static BACKEND: OnceCell<String> = OnceCell::new();
#[cfg(feature = "nnnoiseless")]
static NNNOISELESS_STATE: OnceCell<Mutex<nnnoiseless::NnnoiselessVAD>> = OnceCell::new();
static NNNOISELESS_STATE: OnceCell<Mutex<crate::models::nnnoiseless::NnnoiselessVAD>> = OnceCell::new();
pub fn init(backend: VadBackend) {
pub fn init() {
if BACKEND.get().is_some() {
return;
}
BACKEND.set(backend).ok();
let backend = DB.get()
.map(|db| db.read().vad_backend.clone())
.unwrap_or_else(|| "energy".to_string());
match backend {
VadBackend::None => {
BACKEND.set(backend.clone()).ok();
match backend.as_str() {
"none" => {
info!("VAD: disabled");
}
VadBackend::Energy => {
"energy" => {
info!("VAD: Energy-based");
}
#[cfg(feature = "nnnoiseless")]
VadBackend::Nnnoiseless => {
NNNOISELESS_STATE.set(Mutex::new(nnnoiseless::NnnoiselessVAD::new())).ok();
"nnnoiseless" => {
NNNOISELESS_STATE.set(Mutex::new(crate::models::nnnoiseless::NnnoiselessVAD::new())).ok();
info!("VAD: Nnnoiseless");
}
#[cfg(not(feature = "nnnoiseless"))]
VadBackend::Nnnoiseless => {
warn!("Nnnoiseless not compiled in, falling back to Energy");
BACKEND.set(VadBackend::Energy).ok();
other => {
warn!("Unknown VAD backend '{}', falling back to energy", other);
// overwrite with energy
// (BACKEND already set, so energy::detect will be used via fallthrough)
}
}
}
// Returns (is_voice, confidence)
// returns (is_voice, confidence)
pub fn detect(input: &[i16]) -> (bool, f32) {
match BACKEND.get() {
Some(VadBackend::None) | None => none::detect(input),
Some(VadBackend::Energy) => energy::detect(input),
match BACKEND.get().map(|s| s.as_str()) {
Some("none") | None => none::detect(input),
Some("energy") => energy::detect(input),
#[cfg(feature = "nnnoiseless")]
Some(VadBackend::Nnnoiseless) => {
Some("nnnoiseless") => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().unwrap().detect(input)
state.lock().detect(input)
} else {
energy::detect(input)
}
}
#[cfg(not(feature = "nnnoiseless"))]
Some(VadBackend::Nnnoiseless) => energy::detect(input),
_ => energy::detect(input),
}
}
pub fn reset() {
match BACKEND.get() {
match BACKEND.get().map(|s| s.as_str()) {
#[cfg(feature = "nnnoiseless")]
Some(VadBackend::Nnnoiseless) => {
Some("nnnoiseless") => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().unwrap().reset();
state.lock().reset();
}
}
_ => {}
}
}
}

51
crates/jarvis-core/src/audio_processing/vad/nnnoiseless.rs
View file

@ -1,51 +0,0 @@
use nnnoiseless::DenoiseState;
use crate::config;
pub struct NnnoiselessVAD {
state: Box<DenoiseState<'static>>,
buffer: Vec<f32>,
}
impl NnnoiselessVAD {
pub fn new() -> Self {
Self {
state: DenoiseState::new(),
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
}
}
pub fn detect(&mut self, input: &[i16]) -> (bool, f32) {
for &sample in input {
self.buffer.push(sample as f32);
}
let mut total_vad = 0.0f32;
let mut frame_count = 0u32;
while self.buffer.len() >= config::NNNOISELESS_FRAME_SIZE {
let mut input_frame = [0.0f32; 480];
let mut output_frame = [0.0f32; 480];
input_frame.copy_from_slice(&self.buffer[..config::NNNOISELESS_FRAME_SIZE]);
self.buffer.drain(..config::NNNOISELESS_FRAME_SIZE);
let vad_prob = self.state.process_frame(&mut output_frame, &input_frame);
total_vad += vad_prob;
frame_count += 1;
}
if frame_count == 0 {
return (true, 0.5);
}
let avg_vad = total_vad / frame_count as f32;
let is_voice = avg_vad >= config::VAD_NNNOISELESS_THRESHOLD;
(is_voice, avg_vad)
}
pub fn reset(&mut self) {
self.state = DenoiseState::new();
self.buffer.clear();
}
}

15
crates/jarvis-core/src/commands.rs
View file

@ -247,6 +247,21 @@ pub fn execute_command(cmd_path: &PathBuf, cmd_config: &JCommand, phrase: Option
}
}
// look up a command by its ID
pub fn get_command_by_id<'a>(
commands: &'a [JCommandsList],
id: &str,
) -> Option<(&'a PathBuf, &'a JCommand)> {
for cmd_list in commands {
for cmd in &cmd_list.commands {
if cmd.id == id {
return Some((&cmd_list.path, cmd));
}
}
}
None
}
pub fn list_paths(commands: &[JCommandsList]) -> Vec<&Path> {
commands.iter().map(|x| x.path.as_path()).collect()
}

12
crates/jarvis-core/src/config.rs
View file

@ -17,10 +17,7 @@ use rustpotter::{
RustpotterConfig, ScoreMode,
};
use crate::IntentRecognitionEngine;
use crate::SlotExtractionEngine;
use crate::config::structs::NoiseSuppressionBackend;
use crate::config::structs::VadBackend;
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
#[allow(dead_code)]
@ -68,9 +65,13 @@ pub fn init_dirs() -> Result<(), String> {
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Vosk;
pub const DEFAULT_INTENT_RECOGNITION_ENGINE: IntentRecognitionEngine = IntentRecognitionEngine::IntentClassifier;
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
// backend defaults (string IDs)
pub const DEFAULT_INTENT_BACKEND: &str = "intent-classifier";
pub const DEFAULT_SLOTS_BACKEND: &str = "none";
pub const DEFAULT_VAD_BACKEND: &str = "energy";
pub const DEFAULT_VOICE: &str = "jarvis-remaster";
pub const SOUND_PATH: &str = "resources/sound"; // extended from SOUND_DIR (resources/sound)
pub const VOICES_PATH: &str = "voices"; // extended from SOUND_PATH (resources/sound)
@ -157,15 +158,12 @@ pub const VOSK_SPEECH_PARTIAL_WORDS: bool = false;
// IRE (intents recognition)
pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.75;
// SLOTS EXTRACTION
pub const DEFAULT_SLOT_EXTRACTION_ENGINE: SlotExtractionEngine = SlotExtractionEngine::None;
// embedding classifier
pub const EMBEDDING_MIN_CONFIDENCE: f64 = 0.70;
// AUDIO PROCESSING DEFAULTS
pub const DEFAULT_NOISE_SUPPRESSION: NoiseSuppressionBackend = NoiseSuppressionBackend::None;
pub const DEFAULT_VAD: VadBackend = VadBackend::Energy;
pub const DEFAULT_GAIN_NORMALIZER: bool = false;
// VAD settings

42
crates/jarvis-core/src/config/structs.rs
View file

@ -8,25 +8,12 @@ pub enum WakeWordEngine {
Porcupine,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum IntentRecognitionEngine {
IntentClassifier,
EmbeddingClassifier,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum NoiseSuppressionBackend {
None,
Nnnoiseless,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum VadBackend {
None,
Energy,
Nnnoiseless,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum SpeechToTextEngine {
Vosk,
@ -45,13 +32,6 @@ pub enum AudioType {
Kira,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum SlotExtractionEngine {
None,
GLiNER,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
@ -64,30 +44,8 @@ impl fmt::Display for SpeechToTextEngine {
}
}
impl fmt::Display for IntentRecognitionEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for NoiseSuppressionBackend {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for VadBackend {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for SlotExtractionEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
// pub enum TextToSpeechEngine {}
// pub enum IntentRecognitionEngine {}

27
crates/jarvis-core/src/db.rs
View file

@ -1,12 +1,14 @@
pub mod structs;
pub mod manager;
use crate::{config, APP_CONFIG_DIR};
use log::info;
use std::fs::File;
use std::io::{BufReader, Read};
use std::io::BufReader;
use std::path::PathBuf;
use serde_json;
pub use manager::SettingsManager;
fn get_db_file_path() -> PathBuf {
PathBuf::from(format!(
@ -17,7 +19,6 @@ fn get_db_file_path() -> PathBuf {
}
pub fn init_settings() -> structs::Settings {
let mut db = None;
let db_file_path = get_db_file_path();
info!(
@ -26,23 +27,23 @@ pub fn init_settings() -> structs::Settings {
);
if db_file_path.exists() {
// try load existing settings
if let Ok(mut db_file) = File::open(db_file_path) {
if let Ok(db_file) = File::open(&db_file_path) {
let reader = BufReader::new(db_file);
if let Ok(parsed_json) = serde_json::from_reader(reader) {
if let Ok(settings) = serde_json::from_reader(reader) {
info!("Settings loaded.");
db = Some(parsed_json);
return settings;
}
}
}
if db.is_none() {
// create default settings db file
warn!("No settings file found or there was an error parsing it. Creating default struct.");
db = Some(structs::Settings::default());
}
warn!("No settings file found or there was an error parsing it. Creating default struct.");
structs::Settings::default()
}
db.unwrap()
/// init settings and return a SettingsManager ready to use
pub fn init() -> SettingsManager {
let settings = init_settings();
SettingsManager::new(settings)
}
pub fn save_settings(settings: &structs::Settings) -> Result<(), std::io::Error> {

87
crates/jarvis-core/src/db/manager.rs Normal file
View file

@ -0,0 +1,87 @@
use std::sync::Arc;
use parking_lot::RwLock;
use super::structs::Settings;
use super::save_settings;
// centralized settings manager.
// wraps Arc<RwLock<Settings>> and handles locking + auto-save
// can be used anywhere, ex. from GUI, tray, IPC, CLI, etc.
#[derive(Clone)]
pub struct SettingsManager {
inner: Arc<RwLock<Settings>>,
}
impl SettingsManager {
pub fn new(settings: Settings) -> Self {
Self {
inner: Arc::new(RwLock::new(settings)),
}
}
// wrap an existing Arc<RwLock<Settings>>
pub fn from_arc(arc: Arc<RwLock<Settings>>) -> Self {
Self { inner: arc }
}
// read a setting by key
pub fn read(&self, key: &str) -> Option<String> {
self.inner.read().get(key)
}
// write a setting by key, auto-saves to disk
pub fn write(&self, key: &str, val: &str) -> Result<(), String> {
let snapshot = {
let mut settings = self.inner.write();
settings.set(key, val)?;
settings.clone()
};
save_settings(&snapshot)
.map_err(|e| format!("failed to save settings: {}", e))?;
Ok(())
}
// write multiple settings at once, single save
pub fn write_many(&self, pairs: &[(&str, &str)]) -> Result<(), String> {
let snapshot = {
let mut settings = self.inner.write();
for (key, val) in pairs {
settings.set(key, val)?;
}
settings.clone()
};
save_settings(&snapshot)
.map_err(|e| format!("failed to save settings: {}", e))?;
Ok(())
}
// direct read access to the full Settings struct (for init code that
// needs to read multiple fields at once without key-based access)
pub fn lock(&self) -> parking_lot::RwLockReadGuard<'_, Settings> {
self.inner.read()
}
// direct write access (for bulk operations not covered by set())
pub fn lock_mut(&self) -> parking_lot::RwLockWriteGuard<'_, Settings> {
self.inner.write()
}
// get the underlying Arc
pub fn arc(&self) -> &Arc<RwLock<Settings>> {
&self.inner
}
// dump all settings as key-value pairs (for debugging)
pub fn dump(&self) -> Vec<(String, String)> {
let settings = self.inner.read();
Settings::keys().iter()
.filter_map(|&key| {
settings.get(key).map(|val| (key.to_string(), val))
})
.collect()
}
}

141
crates/jarvis-core/src/db/structs.rs
View file

@ -3,10 +3,7 @@ use serde::{Deserialize, Serialize};
use crate::config::structs::SpeechToTextEngine;
use crate::config::structs::WakeWordEngine;
use crate::config::structs::IntentRecognitionEngine;
use crate::config::structs::NoiseSuppressionBackend;
use crate::config::structs::VadBackend;
use crate::config::structs::SlotExtractionEngine;
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Settings {
@ -14,9 +11,15 @@ pub struct Settings {
pub voice: String,
pub wake_word_engine: WakeWordEngine,
pub intent_recognition_engine: IntentRecognitionEngine,
pub slot_extraction_engine: SlotExtractionEngine,
// backend selections (string IDs matching model or code backend IDs)
#[serde(default = "default_intent_backend")]
pub intent_backend: String,
#[serde(default = "default_slots_backend")]
pub slots_backend: String,
#[serde(default = "default_vad_backend")]
pub vad_backend: String,
pub gliner_model: String,
pub speech_to_text_engine: SpeechToTextEngine,
@ -24,14 +27,127 @@ pub struct Settings {
// audio processing
pub noise_suppression: NoiseSuppressionBackend,
pub vad: VadBackend,
pub gain_normalizer: bool,
#[serde(default = "default_language")]
pub language: String,
pub api_keys: ApiKeys,
}
fn default_intent_backend() -> String { config::DEFAULT_INTENT_BACKEND.to_string() }
fn default_slots_backend() -> String { config::DEFAULT_SLOTS_BACKEND.to_string() }
fn default_vad_backend() -> String { config::DEFAULT_VAD_BACKEND.to_string() }
fn default_language() -> String { crate::i18n::detect_system_language().to_string() }
// ### KEY-VALUE ACCESS
impl Settings {
/// read a setting by key. returns None for unknown keys.
pub fn get(&self, key: &str) -> Option<String> {
match key {
"selected_microphone" => Some(self.microphone.to_string()),
"assistant_voice" => Some(self.voice.clone()),
"selected_wake_word_engine" => Some(format!("{:?}", self.wake_word_engine)),
"intent_backend" => Some(self.intent_backend.clone()),
"slots_backend" => Some(self.slots_backend.clone()),
"vad_backend" => Some(self.vad_backend.clone()),
"selected_gliner_model" => Some(self.gliner_model.clone()),
"selected_vosk_model" => Some(self.vosk_model.clone()),
"speech_to_text_engine" => Some(format!("{:?}", self.speech_to_text_engine)),
"noise_suppression" => Some(format!("{:?}", self.noise_suppression)),
"gain_normalizer" => Some(self.gain_normalizer.to_string()),
"language" => Some(self.language.clone()),
"api_key__picovoice" => Some(self.api_keys.picovoice.clone()),
"api_key__openai" => Some(self.api_keys.openai.clone()),
_ => None,
}
}
/// write a setting by key. returns Err for unknown keys or invalid values.
pub fn set(&mut self, key: &str, val: &str) -> Result<(), String> {
match key {
"selected_microphone" => {
self.microphone = val.parse::<i32>()
.map_err(|_| format!("invalid integer: '{}'", val))?;
}
"assistant_voice" => {
self.voice = val.to_string();
}
"selected_wake_word_engine" => {
self.wake_word_engine = match val.to_lowercase().as_str() {
"rustpotter" => WakeWordEngine::Rustpotter,
"vosk" => WakeWordEngine::Vosk,
"porcupine" => WakeWordEngine::Porcupine,
_ => return Err(format!("unknown wake word engine: '{}'", val)),
};
}
"intent_backend" => {
self.intent_backend = val.to_string();
}
"slots_backend" => {
self.slots_backend = val.to_string();
}
"vad_backend" => {
self.vad_backend = val.to_string();
}
"selected_gliner_model" => {
self.gliner_model = val.to_string();
}
"selected_vosk_model" => {
self.vosk_model = val.to_string();
}
"noise_suppression" => {
self.noise_suppression = match val.to_lowercase().as_str() {
"none" => NoiseSuppressionBackend::None,
"nnnoiseless" => NoiseSuppressionBackend::Nnnoiseless,
_ => return Err(format!("unknown noise suppression backend: '{}'", val)),
};
}
"gain_normalizer" => {
self.gain_normalizer = match val.to_lowercase().as_str() {
"true" => true,
"false" => false,
_ => return Err(format!("expected 'true' or 'false', got: '{}'", val)),
};
}
"language" => {
self.language = val.to_string();
}
"api_key__picovoice" => {
self.api_keys.picovoice = val.to_string();
}
"api_key__openai" => {
self.api_keys.openai = val.to_string();
}
_ => return Err(format!("unknown setting: '{}'", key)),
}
Ok(())
}
/// all valid setting keys (for enumeration, debugging, etc.)
pub fn keys() -> &'static [&'static str] {
&[
"selected_microphone",
"assistant_voice",
"selected_wake_word_engine",
"intent_backend",
"slots_backend",
"vad_backend",
"selected_gliner_model",
"selected_vosk_model",
"speech_to_text_engine",
"noise_suppression",
"gain_normalizer",
"language",
"api_key__picovoice",
"api_key__openai",
]
}
}
// ### DEFAULT
impl Default for Settings {
fn default() -> Settings {
Settings {
@ -39,18 +155,19 @@ impl Default for Settings {
voice: String::from(""),
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
intent_recognition_engine: config::DEFAULT_INTENT_RECOGNITION_ENGINE,
slot_extraction_engine: SlotExtractionEngine::None,
intent_backend: config::DEFAULT_INTENT_BACKEND.to_string(),
slots_backend: config::DEFAULT_SLOTS_BACKEND.to_string(),
vad_backend: config::DEFAULT_VAD_BACKEND.to_string(),
gliner_model: String::new(),
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
vosk_model: String::from(""), // auto detect first available
vosk_model: String::from(""),
// audio processing defaults
noise_suppression: config::DEFAULT_NOISE_SUPPRESSION,
vad: config::DEFAULT_VAD,
gain_normalizer: config::DEFAULT_GAIN_NORMALIZER,
language: String::from("ru"),
language: crate::i18n::detect_system_language().to_string(),
api_keys: ApiKeys {
picovoice: String::from(""),

30
crates/jarvis-core/src/i18n.rs
View file

@ -11,7 +11,33 @@ const LOCALE_EN: &str = include_str!("i18n/locales/en.ftl");
const LOCALE_UA: &str = include_str!("i18n/locales/ua.ftl");
pub const SUPPORTED_LANGUAGES: &[&str] = &["ru", "en", "ua"];
pub const DEFAULT_LANGUAGE: &str = "ru";
pub const DEFAULT_LANGUAGE: &str = "en";
// detect the OS language and map it to a supported language.
// falls back to DEFAULT_LANGUAGE if not supported.
pub fn detect_system_language() -> &'static str {
if let Some(locale) = sys_locale::get_locale() {
// locale can be "en-US", "ru-RU", "uk-UA", etc.
let lang_code = locale.split(&['-', '_'][..]).next().unwrap_or("");
// map OS locale codes to our supported languages
let mapped = match lang_code {
"uk" => "ua", // ISO 639-1 "uk" (ukrainian) -> our "ua"
other => other,
};
if SUPPORTED_LANGUAGES.contains(&mapped) {
info!("Detected system language: {} (from locale '{}')", mapped, locale);
return SUPPORTED_LANGUAGES.iter()
.find(|&&l| l == mapped)
.unwrap();
}
info!("System locale '{}' not supported, using default '{}'", locale, DEFAULT_LANGUAGE);
}
DEFAULT_LANGUAGE
}
// use concurrent bundle (thread-safe)
type Bundle = ConcurrentFluentBundle<FluentResource>;
@ -126,7 +152,7 @@ pub fn get_all_translations() -> HashMap<String, String> {
get_translations_for(&lang)
}
/// Get all translations for a specific language
// Get all translations for a specific language
pub fn get_translations_for(lang: &str) -> HashMap<String, String> {
let mut result = HashMap::new();

6
crates/jarvis-core/src/i18n/locales/en.ftl
View file

@ -7,6 +7,12 @@ tray-restart = Restart
tray-settings = Settings
tray-exit = Exit
tray-tooltip = JARVIS - Voice Assistant
tray-language = Language
tray-voice = Voice
tray-wake-word = Wake Word Engine
tray-noise-suppression = Noise Suppression
tray-vad = Voice Activity Detection
tray-gain-normalizer = Gain Normalizer
# ### HEADER
header-commands = COMMANDS

30
crates/jarvis-core/src/i18n/locales/ru.ftl
View file

@ -1,33 +1,39 @@
# ### APP INFO
# APP INFO
app-name = JARVIS
app-description = Голосовой ассистент
# ### TRAY MENU
# TRAY MENU
tray-restart = Перезапустить
tray-settings = Настройки
tray-exit = Выход
tray-tooltip = JARVIS - Голосовой ассистент
tray-language = Язык
tray-voice = Голос
tray-wake-word = Движок wake-word
tray-noise-suppression = Шумоподавление
tray-vad = Детекция голоса (VAD)
tray-gain-normalizer = Нормализация громкости
# ### HEADER
# HEADER
header-commands = КОМАНДЫ
header-settings = НАСТРОЙКИ
# ### SEARCH
# SEARCH
search-placeholder = Введите команду вручную или произнесите «Джарвис» ...
# ### MAIN PAGE
# MAIN PAGE
assistant-not-running = АССИСТЕНТ НЕ ЗАПУЩЕН
assistant-offline-hint = Настроить его можно не запуская.
btn-start = ЗАПУСТИТЬ
btn-starting = ЗАПУСК...
# ### STATUS
# STATUS
status-disconnected = Отключен
status-standby = Ожидание
status-listening = Слушаю...
status-processing = Обработка...
# ### STATS
# STATS
stats-microphone = МИКРОФОН
stats-neural-networks = НЕЙРОСЕТИ
stats-resources = РЕСУРСЫ
@ -35,13 +41,13 @@ stats-system-default = Системный
stats-not-selected = Не выбран
stats-loading = Загрузка...
# ### FOOTER
# FOOTER
footer-author = Автор проекта
footer-telegram = Наш телеграм канал
footer-github = Github репозиторий проекта
footer-support = Поддержать проект на
# ### SETTINGS
# SETTINGS
settings-title = Настройки
settings-general = Основные
settings-devices = Устройства
@ -102,7 +108,7 @@ settings-models-hint = Поместите модели Vosk в папку resour
settings-openai-key = Ключ OpenAI
settings-openai-not-supported = В данный момент ChatGPT не поддерживается. Он будет добавлен в ближайших обновлениях.
# ### COMMANDS PAGE
# COMMANDS PAGE
commands-title = Команды
commands-search = Поиск команд...
commands-count = { $count } команд
@ -111,12 +117,12 @@ commands-wip-desc = Тут будет список команд + полноце
commands-wip-follow = Следите за обновлениями в
commands-wip-channel = нашем телеграм канале
# ### ERRORS
# ERRORS
error-generic = Произошла ошибка
error-connection = Ошибка подключения
error-not-found = Не найдено
# ### NOTIFICATIONS
# NOTIFICATIONS
notification-saved = Настройки сохранены!
notification-error = Ошибка
notification-assistant-started = Ассистент запущен

6
crates/jarvis-core/src/i18n/locales/ua.ftl
View file

@ -7,6 +7,12 @@ tray-restart = Перезапустити
tray-settings = Налаштування
tray-exit = Вихід
tray-tooltip = JARVIS - Голосовий асистент
tray-language = Мова
tray-voice = Голос
tray-wake-word = Рушій детекції
tray-noise-suppression = Шумозаглушення
tray-vad = Детекцiя голосу (VAD)
tray-gain-normalizer = Нормалізація гучності
# ### HEADER
header-commands = КОМАНДИ

68
crates/jarvis-core/src/intent.rs
View file

@ -3,47 +3,47 @@ mod embeddingclassifier;
use std::path::PathBuf;
use crate::{JCommandsList, commands::JCommand, config};
use crate::{commands::{self, JCommandsList, JCommand}, config, models};
use once_cell::sync::OnceCell;
use crate::config::structs::IntentRecognitionEngine;
use crate::DB;
static IRE_TYPE: OnceCell<IntentRecognitionEngine> = OnceCell::new();
static BACKEND: OnceCell<String> = OnceCell::new();
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
if IRE_TYPE.get().is_some() {
if BACKEND.get().is_some() {
return Ok(());
} // already initialized
}
// set default ire type
// IRE_TYPE.set(config::DEFAULT_INTENT_RECOGNITION_ENGINE).unwrap();
let backend = DB.get().unwrap().read().intent_backend.clone();
// store current ire type
IRE_TYPE
.set(DB.get().unwrap().read().intent_recognition_engine)
.unwrap();
BACKEND.set(backend.clone()).map_err(|_| "Backend already set")?;
// load given recorder
match IRE_TYPE.get().unwrap() {
IntentRecognitionEngine::IntentClassifier => {
info!("Initializing IntentClassifier IRE backend.");
match backend.as_str() {
"none" => {
info!("Intent recognition disabled");
}
"intent-classifier" => {
info!("Initializing IntentClassifier backend.");
intentclassifier::init(&commands).await?;
info!("IntentClassifier IRE backend initialized.");
},
IntentRecognitionEngine::EmbeddingClassifier => {
info!("Initializing EmbeddingClassifier IRE backend.");
embeddingclassifier::init(&commands)?;
info!("EmbeddingClassifier IRE backend initialized.");
},
info!("IntentClassifier backend initialized.");
}
// any other value is treated as a model ID for embedding classification
model_id => {
info!("Initializing EmbeddingClassifier with model '{}'.", model_id);
let model = models::embedding::load(models::registry(), model_id)?;
embeddingclassifier::init_with_model(model, &commands)?;
info!("EmbeddingClassifier backend initialized.");
}
}
Ok(())
}
pub async fn classify(text: &str) -> Option<(String, f64)> {
match IRE_TYPE.get()? {
IntentRecognitionEngine::IntentClassifier => {
match BACKEND.get()?.as_str() {
"none" => None,
"intent-classifier" => {
match intentclassifier::classify(text).await {
Ok(prediction) => {
let confidence = prediction.confidence.value();
@ -59,7 +59,7 @@ pub async fn classify(text: &str) -> Option<(String, f64)> {
}
}
}
IntentRecognitionEngine::EmbeddingClassifier => {
_ => {
match embeddingclassifier::classify(text) {
Ok((intent_id, confidence)) => {
if confidence >= config::EMBEDDING_MIN_CONFIDENCE {
@ -77,13 +77,13 @@ pub async fn classify(text: &str) -> Option<(String, f64)> {
}
}
pub fn get_command_by_intent(commands: &'static Vec<JCommandsList>, intent_id: &str) -> Option<(&'static PathBuf, &'static JCommand)> {
match IRE_TYPE.get()? {
IntentRecognitionEngine::IntentClassifier => {
intentclassifier::get_command(commands, intent_id)
}
IntentRecognitionEngine::EmbeddingClassifier => {
embeddingclassifier::get_command(commands, intent_id)
}
// unified command lookup by intent ID - works for all backends
pub fn get_command_by_intent<'a>(
commands: &'a [JCommandsList],
intent_id: &str,
) -> Option<(&'a PathBuf, &'a JCommand)> {
if matches!(BACKEND.get().map(|s| s.as_str()), Some("none")) {
return None;
}
}
commands::get_command_by_id(commands, intent_id)
}

111
crates/jarvis-core/src/intent/embeddingclassifier.rs
View file

@ -1,79 +1,42 @@
use parking_lot::Mutex;
use std::path::PathBuf;
use std::sync::Arc;
use std::fs;
// use fastembed::{TextEmbedding, InitOptions, EmbeddingModel};
use fastembed::{TextEmbedding, UserDefinedEmbeddingModel, TokenizerFiles, InitOptionsUserDefined, Pooling, QuantizationMode, OutputKey};
use once_cell::sync::OnceCell;
use crate::commands::JCommandsList;
use crate::i18n::get_language;
use crate::{APP_CONFIG_DIR, APP_DIR, i18n};
use crate::i18n;
use crate::APP_CONFIG_DIR;
use crate::models::embedding::EmbeddingModel;
static CLASSIFIER: OnceCell<Mutex<EmbeddingClassifier>> = OnceCell::new();
// no outer Mutex needed - state is immutable after init.
// the embedding model has its own internal Mutex.
static CLASSIFIER: OnceCell<EmbeddingClassifierState> = OnceCell::new();
struct IntentVector {
id: String,
vector: Vec<f32>,
}
struct EmbeddingClassifier {
model: TextEmbedding,
struct EmbeddingClassifierState {
model: Arc<EmbeddingModel>,
intents: Vec<IntentVector>,
}
// model is Arc (Send+Sync), intents are read-only after init
unsafe impl Send for EmbeddingClassifierState {}
unsafe impl Sync for EmbeddingClassifierState {}
const CACHE_FILE: &str = "embedding_intents.json";
const HASH_FILE: &str = "embedding_hash.txt";
pub fn init(commands: &[JCommandsList]) -> Result<(), String> {
// init with a model loaded through the registry
pub fn init_with_model(model: Arc<EmbeddingModel>, commands: &[JCommandsList]) -> Result<(), String> {
if CLASSIFIER.get().is_some() {
return Ok(());
}
info!("Initializing embedding model...");
// let mut model = TextEmbedding::try_new(
// InitOptions::new(EmbeddingModel::AllMiniLML6V2).with_show_download_progress(true),
// ).map_err(|e| format!("Failed to load embedding model: {}", e))?;
let model_dir;
match i18n::get_language().as_str() {
"en" => {
// smaller model for English
info!("Loading all-MiniLM-L6-v2 ...");
model_dir = APP_DIR.join("resources").join("models").join("all-MiniLM-L6-v2");
},
_ => {
// bigger model for any other languages (multilingual)
info!("Loading paraphrase-multilingual-MiniLM-L12-v2-onnx-Q ...");
model_dir = APP_DIR.join("resources").join("models").join("paraphrase-multilingual-MiniLM-L12-v2-onnx-Q");
}
}
// info!("{}", model_dir.display());
let user_model = UserDefinedEmbeddingModel {
onnx_file: std::fs::read(model_dir.join("model.onnx"))
.map_err(|e| format!("Failed to read model.onnx: {}", e))?,
tokenizer_files: TokenizerFiles {
tokenizer_file: std::fs::read(model_dir.join("tokenizer.json"))
.map_err(|e| format!("Failed to read tokenizer.json: {}", e))?,
config_file: std::fs::read(model_dir.join("config.json"))
.map_err(|e| format!("Failed to read config.json: {}", e))?,
special_tokens_map_file: std::fs::read(model_dir.join("special_tokens_map.json"))
.map_err(|e| format!("Failed to read special_tokens_map.json: {}", e))?,
tokenizer_config_file: std::fs::read(model_dir.join("tokenizer_config.json"))
.map_err(|e| format!("Failed to read tokenizer_config.json: {}", e))?,
},
pooling: Some(Pooling::Mean),
quantization: QuantizationMode::None,
output_key: Some(OutputKey::ByName("last_hidden_state")),
};
let mut model = TextEmbedding::try_new_from_user_defined(user_model, Default::default())
.map_err(|e| format!("Failed to load embedding model: {}", e))?;
info!("Embedding model loaded");
info!("Initializing embedding classifier...");
let current_hash = crate::commands::commands_hash(commands);
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
@ -90,7 +53,7 @@ pub fn init(commands: &[JCommandsList]) -> Result<(), String> {
let intents = if should_retrain {
info!("Building intent vectors from commands...");
let intents = build_intent_vectors(&mut model, commands)?;
let intents = build_intent_vectors(&model, commands)?;
// cache to disk
if let Ok(json) = serde_json::to_string(&intents_to_cache(&intents)) {
@ -107,14 +70,14 @@ pub fn init(commands: &[JCommandsList]) -> Result<(), String> {
info!("Embedding classifier ready with {} intents", intents.len());
CLASSIFIER.set(Mutex::new(EmbeddingClassifier { model, intents }))
.map_err(|_| "Classifier already set")?;
CLASSIFIER.set(EmbeddingClassifierState { model, intents })
.map_err(|_| "Classifier already set".to_string())?;
Ok(())
}
fn build_intent_vectors(
model: &mut TextEmbedding,
model: &EmbeddingModel,
commands: &[JCommandsList],
) -> Result<Vec<IntentVector>, String> {
let lang = i18n::get_language();
@ -129,7 +92,7 @@ fn build_intent_vectors(
let texts: Vec<&str> = phrases.iter().map(|s| s.as_str()).collect();
let embeddings = model.embed(texts, None)
let embeddings = model.embedding.lock().embed(texts, None)
.map_err(|e| format!("Embedding failed for '{}': {}", cmd.id, e))?;
// average all phrase vectors into one intent vector
@ -166,9 +129,10 @@ fn build_intent_vectors(
}
pub fn classify(text: &str) -> Result<(String, f64), String> {
let mut classifier = CLASSIFIER.get().ok_or("Classifier not initialized")?.lock();
let state = CLASSIFIER.get().ok_or("Classifier not initialized")?;
let embeddings = classifier.model.embed(vec![text], None)
// only the embedding model needs locking, intents are read-only
let embeddings = state.model.embedding.lock().embed(vec![text], None)
.map_err(|e| format!("Failed to embed query: {}", e))?;
let mut query_vec = embeddings.into_iter().next()
@ -182,11 +146,11 @@ pub fn classify(text: &str) -> Result<(String, f64), String> {
}
}
// cosine similarity against all intents (dot product of normalized vectors)
let mut best_id = String::new();
// cosine similarity - track index, clone only the winner
let mut best_idx: usize = 0;
let mut best_score: f64 = -1.0;
for intent in &classifier.intents {
for (i, intent) in state.intents.iter().enumerate() {
let score: f64 = query_vec.iter()
.zip(intent.vector.iter())
.map(|(a, b)| (*a as f64) * (*b as f64))
@ -194,31 +158,16 @@ pub fn classify(text: &str) -> Result<(String, f64), String> {
if score > best_score {
best_score = score;
best_id = intent.id.clone();
best_idx = i;
}
}
let best_id = state.intents[best_idx].id.clone();
debug!("Embedding classify: '{}' -> '{}' ({:.2}%)", text, best_id, best_score * 100.0);
Ok((best_id, best_score))
}
pub fn get_command<'a>(
commands: &'a [JCommandsList],
intent_id: &str,
) -> Option<(&'a PathBuf, &'a crate::commands::JCommand)> {
for cmd_list in commands {
for cmd in &cmd_list.commands {
if cmd.id == intent_id {
return Some((&cmd_list.path, cmd));
}
}
}
None
}
// ### CACHE HELPERS
#[derive(serde::Serialize, serde::Deserialize)]
struct CachedIntent {
id: String,
@ -243,4 +192,4 @@ fn load_cached_intents(path: &PathBuf) -> Result<Vec<IntentVector>, String> {
id: c.id,
vector: c.vector,
}).collect())
}
}

58
crates/jarvis-core/src/intent/intentclassifier.rs
View file

@ -1,29 +1,27 @@
use intent_classifier::{
IntentClassifier, IntentPrediction, IntentError,
IntentPrediction, IntentError,
TrainingExample, TrainingSource, IntentId
};
use tokio::sync::OnceCell;
use std::path::PathBuf;
use std::sync::Arc;
use std::fs;
use crate::commands::{self, JCommand, JCommandsList};
use crate::commands::{self, JCommandsList};
use crate::models;
use crate::models::intent_classifier::IntentClassifierModel;
use crate::{APP_CONFIG_DIR, i18n};
static CLASSIFIER: OnceCell<IntentClassifier> = OnceCell::const_new();
// static COMMANDS_MAP: OnceCell<Vec<JCommandsList>> = OnceCell::const_new();
use once_cell::sync::OnceCell;
static MODEL: OnceCell<Arc<IntentClassifierModel>> = OnceCell::new();
const TRAINING_CACHE_FILE: &str = "intent_training.json";
const COMMANDS_HASH_FILE: &str = "commands_hash.txt";
pub async fn init(commands: &[JCommandsList]) -> Result<(), String> {
// parse commands first
// let commands = commands::parse_commands()?;
let current_hash = commands::commands_hash(&commands); // regen hash for current commands set
let current_hash = commands::commands_hash(&commands);
// init classifier
let classifier = IntentClassifier::new().await
.map_err(|e| format!("Failed to init IntentClassifier: {}", e))?;
let model = models::intent_classifier::load(models::registry(), "intent-classifier").await?;
// check if we can use cached training data
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
@ -39,10 +37,9 @@ pub async fn init(commands: &[JCommandsList]) -> Result<(), String> {
if should_retrain {
info!("Training intent classifier with {} commands...", commands.len());
train_classifier(&classifier, &commands).await?;
train_classifier(&model.classifier, &commands).await?;
// save training data and hash
if let Ok(export) = classifier.export_training_data().await {
if let Ok(export) = model.classifier.export_training_data().await {
let _ = fs::write(&cache_path, export);
let _ = fs::write(&hash_path, &current_hash);
info!("Training data cached.");
@ -50,41 +47,23 @@ pub async fn init(commands: &[JCommandsList]) -> Result<(), String> {
} else {
info!("Loading cached training data...");
if let Ok(data) = fs::read_to_string(&cache_path) {
classifier.import_training_data(&data).await
model.classifier.import_training_data(&data).await
.map_err(|e| format!("Failed to import training data: {}", e))?;
}
}
// store data
CLASSIFIER.set(classifier).map_err(|_| "Classifier already set")?;
// COMMANDS_MAP.set(commands).map_err(|_| "Commands map already set")?;
MODEL.set(model).map_err(|_| "Model already set")?;
Ok(())
}
pub async fn classify(text: &str) -> Result<IntentPrediction, IntentError> {
let classifier = CLASSIFIER.get().expect("IntentClassifier not initialized");
classifier.predict_intent(text).await
let model = MODEL.get().expect("IntentClassifier not initialized");
model.classifier.predict_intent(text).await
}
// get command by intent ID
pub fn get_command(commands: &'static [JCommandsList], intent_id: &str) -> Option<(&'static PathBuf, &'static JCommand)> {
// let commands = COMMANDS_MAP.get()?;
for assistant_cmd in commands {
for cmd in &assistant_cmd.commands {
if cmd.id == intent_id {
return Some((&assistant_cmd.path, cmd));
}
}
}
None
}
// based on: https://github.com/ciresnave/intent-classifier/blob/main/examples/basic_usage.rs
async fn train_classifier(
classifier: &IntentClassifier,
classifier: &intent_classifier::IntentClassifier,
commands: &[JCommandsList]
) -> Result<(), String> {
let lang = i18n::get_language();
@ -94,7 +73,6 @@ async fn train_classifier(
for assistant_cmd in commands {
for cmd in &assistant_cmd.commands {
// use language-specific phrases
let phrases = cmd.get_phrases(&lang);
for phrase in phrases.iter() {
@ -115,4 +93,4 @@ async fn train_classifier(
info!("Added {} training examples for language '{}'", total_examples, lang);
Ok(())
}
}

8
crates/jarvis-core/src/lib.rs
View file

@ -29,8 +29,11 @@ pub mod intent;
#[cfg(feature = "jarvis_app")]
pub mod slots;
pub mod vosk_models;
pub mod gliner_models;
pub mod models;
// re-exported from models/
pub use models::vosk_models;
pub use models::gliner_models;
#[cfg(feature = "jarvis_app")]
pub mod audio_processing;
@ -64,5 +67,6 @@ pub static COMMANDS_LIST: OnceCell<Vec<JCommandsList>> = OnceCell::new();
pub use commands::JCommandsList;
pub use config::structs::*;
pub use db::structs::Settings;
pub use db::SettingsManager;
// use crate::commands::{JComandsList, JCommand};

47
crates/jarvis-core/src/listener.rs
View file

@ -1,64 +1,45 @@
// mod porcupine;
mod rustpotter;
mod vosk;
use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::config::structs::WakeWordEngine;
use crate::{config, stt};
use crate::DB;
// store wake-word engine being used
static WAKE_WORD_ENGINE: OnceCell<WakeWordEngine> = OnceCell::new();
// track listening state
static LISTENING: AtomicBool = AtomicBool::new(false);
pub fn init() -> Result<(), ()> {
pub fn init() -> Result<(), String> {
if WAKE_WORD_ENGINE.get().is_some() {
return Ok(());
} // already initialized
}
// store current engine
WAKE_WORD_ENGINE
.set(DB.get().unwrap().read().wake_word_engine)
.unwrap();
let engine = DB.get().unwrap().read().wake_word_engine;
// load given wake-word engine
match WAKE_WORD_ENGINE.get().unwrap() {
WAKE_WORD_ENGINE.set(engine)
.map_err(|_| "Wake word engine already set".to_string())?;
match engine {
WakeWordEngine::Porcupine => {
// Init Porcupine wake-word engine
info!("Initializing Porcupine wake-word engine.");
// return porcupine::init();
unimplemented!("f*ck picovoice");
Err("Porcupine wake-word engine is not supported".to_string())
}
WakeWordEngine::Rustpotter => {
// Init Rustpotter wake-word engine
info!("Initializing Rustpotter wake-word engine.");
return rustpotter::init();
rustpotter::init()
.map_err(|_| "Failed to init Rustpotter".to_string())
}
WakeWordEngine::Vosk => {
// Init Vosk as wake-word engine (very slow, though)
info!("Initializing Vosk as wake-word engine.");
warn!("Using Vosk as wake-word engine is highly not recommended, because it's very slow for this task.");
return vosk::init();
vosk::init()
.map_err(|_| "Failed to init Vosk wake-word".to_string())
}
}
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
match WAKE_WORD_ENGINE.get().unwrap() {
WakeWordEngine::Porcupine => {
// porcupine::data_callback(frame_buffer)
unimplemented!("f*ck picovoice");
},
match WAKE_WORD_ENGINE.get()? {
WakeWordEngine::Porcupine => None,
WakeWordEngine::Rustpotter => rustpotter::data_callback(frame_buffer),
WakeWordEngine::Vosk => vosk::data_callback(frame_buffer),
}

9
crates/jarvis-core/src/listener/rustpotter.rs
View file

@ -1,14 +1,9 @@
use std::path::Path;
use std::sync::Mutex;
use once_cell::sync::OnceCell;
use rustpotter::{
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig, Rustpotter,
RustpotterConfig, ScoreMode,
};
use rustpotter::Rustpotter;
use crate::config;
use crate::DB;
// store rustpotter instance
static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
@ -40,7 +35,7 @@ pub fn init() -> Result<(), ()> {
}
// store
RUSTPOTTER.set(Mutex::new(rinstance));
let _ = RUSTPOTTER.set(Mutex::new(rinstance));
}
Err(msg) => {
error!("Rustpotter failed to initialize.\nError details: {}", msg);

2
crates/jarvis-core/src/lua/api/http.rs
View file

@ -148,7 +148,7 @@ fn http_request_with_headers(
// Convert Lua table to serde_json::Value
fn table_to_json(lua: &Lua, table: Table) -> mlua::Result<serde_json::Value> {
use serde_json::{Value as JsonValue, Map, Number};
use serde_json::{Value as JsonValue, Map};
// check if it's an array (sequential integer keys starting from 1)
let is_array = table.clone().pairs::<i64, Value>()

6
crates/jarvis-core/src/lua/engine.rs
View file

@ -1,9 +1,7 @@
use mlua::{Lua, Result as LuaResult, Value, StdLib};
use mlua::{Lua, Value, StdLib};
use std::path::PathBuf;
use std::time::{Duration, Instant};
use std::time::Duration;
use std::fs;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, mpsc};
use super::sandbox::SandboxLevel;
use super::error::LuaError;

67
crates/jarvis-core/src/models.rs Normal file
View file

@ -0,0 +1,67 @@
mod registry;
mod catalog;
pub mod structs;
pub mod loaders;
pub mod vosk_models;
pub mod gliner_models;
// re-export loaders
#[cfg(feature = "jarvis_app")]
pub use loaders::embedding;
#[cfg(feature = "jarvis_app")]
pub use loaders::gliner;
#[cfg(feature = "jarvis_app")]
pub use loaders::ort_model;
#[cfg(feature = "jarvis_app")]
pub use loaders::intent_classifier;
#[cfg(feature = "vosk")]
pub use loaders::vosk;
#[cfg(feature = "nnnoiseless")]
pub use loaders::nnnoiseless;
pub use registry::ModelRegistry;
pub use structs::{Task, ModelDef, BackendOption};
use once_cell::sync::OnceCell;
use crate::APP_DIR;
pub const MODELS_PATH: &str = "resources/models";
static REGISTRY: OnceCell<ModelRegistry> = OnceCell::new();
pub fn init() -> Result<(), String> {
if REGISTRY.get().is_some() {
return Ok(());
}
let registry = ModelRegistry::new();
let models_dir = APP_DIR.join(MODELS_PATH);
let models = catalog::scan_models(&models_dir);
info!("Found {} model(s) in {:?}", models.len(), models_dir);
registry.set_catalog(models);
REGISTRY.set(registry)
.map_err(|_| "Models registry already initialized".to_string())?;
Ok(())
}
pub fn registry() -> &'static ModelRegistry {
REGISTRY.get().expect("Models registry not initialized - call models::init() first")
}
pub fn get_options(task: Task) -> Vec<BackendOption> {
registry().with_catalog(|models| catalog::get_options(task, models))
}
pub fn is_valid_backend(task: Task, backend_id: &str) -> bool {
registry().with_catalog(|models| catalog::is_valid_backend(task, backend_id, models))
}

140
crates/jarvis-core/src/models/catalog.rs Normal file
View file

@ -0,0 +1,140 @@
use std::fs;
use std::path::Path;
use super::structs::{Task, ModelDef, BackendOption};
// scan the models directory for folders containing model.toml
pub fn scan_models(models_dir: &Path) -> Vec<ModelDef> {
let mut models = Vec::new();
if !models_dir.exists() {
warn!("Models directory not found: {:?}", models_dir);
return models;
}
let entries = match fs::read_dir(models_dir) {
Ok(e) => e,
Err(e) => {
warn!("Failed to read models dir: {}", e);
return models;
}
};
for entry in entries.flatten() {
let path = entry.path();
if !path.is_dir() {
continue;
}
let toml_path = path.join("model.toml");
if !toml_path.exists() {
continue;
}
match load_model_def(&toml_path, &path) {
Ok(def) => {
info!("Found model: {} ({}) - tasks: {:?}", def.name, def.id, def.tasks);
models.push(def);
}
Err(e) => warn!("Failed to load model from {:?}: {}", path, e),
}
}
models
}
fn load_model_def(toml_path: &Path, model_dir: &Path) -> Result<ModelDef, String> {
let content = fs::read_to_string(toml_path)
.map_err(|e| format!("read error: {}", e))?;
let parsed: ModelToml = toml::from_str(&content)
.map_err(|e| format!("parse error: {}", e))?;
let mut def = parsed.model;
def.path = model_dir.to_path_buf();
Ok(def)
}
#[derive(serde::Deserialize)]
struct ModelToml {
model: ModelDef,
}
// Code backends per task
pub fn code_backends(task: Task) -> Vec<BackendOption> {
match task {
Task::Intent => vec![
BackendOption {
id: "intent-classifier".into(),
name: "Intent Classifier".into(),
model_id: None,
},
],
Task::Slots => vec![],
Task::Vad => vec![
BackendOption {
id: "energy".into(),
name: "Energy-based".into(),
model_id: None,
},
BackendOption {
id: "nnnoiseless".into(),
name: "Nnnoiseless".into(),
model_id: None,
},
],
Task::NoiseSuppression => vec![
BackendOption {
id: "nnnoiseless".into(),
name: "Nnnoiseless".into(),
model_id: None,
},
],
Task::Stt => vec![
BackendOption {
id: "vosk".into(),
name: "Vosk".into(),
model_id: None,
},
],
}
}
// get all available options for a task:
// "none" first, then code backends, then AI models from catalog
pub fn get_options(task: Task, models: &[ModelDef]) -> Vec<BackendOption> {
let mut options = vec![
BackendOption {
id: "none".into(),
name: "Disabled".into(),
model_id: None,
},
];
options.extend(code_backends(task));
for model in models {
if model.tasks.contains(&task) {
options.push(BackendOption {
id: model.id.clone(),
name: model.name.clone(),
model_id: Some(model.id.clone()),
});
}
}
options
}
pub fn is_valid_backend(task: Task, backend_id: &str, models: &[ModelDef]) -> bool {
if backend_id == "none" {
return true;
}
if code_backends(task).iter().any(|b| b.id == backend_id) {
return true;
}
models.iter().any(|m| m.id == backend_id && m.tasks.contains(&task))
}

0
crates/jarvis-core/src/gliner_models.rs → crates/jarvis-core/src/models/gliner_models.rs
View file

47
crates/jarvis-core/src/models/loaders/embedding.rs Normal file
View file

@ -0,0 +1,47 @@
// fastembed embedding model (all-MiniLM-L6-v2, paraphrase-multilingual, etc.)
use std::sync::Arc;
use parking_lot::Mutex;
use fastembed::{TextEmbedding, UserDefinedEmbeddingModel, TokenizerFiles, Pooling, QuantizationMode, OutputKey};
use crate::models::registry::ModelRegistry;
pub struct EmbeddingModel {
pub embedding: Mutex<TextEmbedding>,
}
// fastembed uses ORT internally which is thread-safe
unsafe impl Send for EmbeddingModel {}
unsafe impl Sync for EmbeddingModel {}
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<EmbeddingModel>, String> {
registry.get_or_load::<EmbeddingModel>(model_id, |def| {
let model_dir = &def.path;
info!("Loading embedding model from: {}", model_dir.display());
let user_model = UserDefinedEmbeddingModel {
onnx_file: std::fs::read(model_dir.join("model.onnx"))
.map_err(|e| format!("Failed to read model.onnx: {}", e))?,
tokenizer_files: TokenizerFiles {
tokenizer_file: std::fs::read(model_dir.join("tokenizer.json"))
.map_err(|e| format!("Failed to read tokenizer.json: {}", e))?,
config_file: std::fs::read(model_dir.join("config.json"))
.map_err(|e| format!("Failed to read config.json: {}", e))?,
special_tokens_map_file: std::fs::read(model_dir.join("special_tokens_map.json"))
.map_err(|e| format!("Failed to read special_tokens_map.json: {}", e))?,
tokenizer_config_file: std::fs::read(model_dir.join("tokenizer_config.json"))
.map_err(|e| format!("Failed to read tokenizer_config.json: {}", e))?,
},
pooling: Some(Pooling::Mean),
quantization: QuantizationMode::None,
output_key: Some(OutputKey::ByName("last_hidden_state")),
};
let model = TextEmbedding::try_new_from_user_defined(user_model, Default::default())
.map_err(|e| format!("Failed to load embedding model: {}", e))?;
info!("Embedding model loaded: {}", def.name);
Ok(EmbeddingModel { embedding: Mutex::new(model) })
})
}

51
crates/jarvis-core/src/models/loaders/gliner.rs Normal file
View file

@ -0,0 +1,51 @@
// GLiNER model for named entity recognition / slot extraction
use std::sync::Arc;
use parking_lot::Mutex;
use regex::Regex;
use tokenizers::Tokenizer;
use crate::models::registry::ModelRegistry;
const WORD_REGEX: &str = r"\w+(?:[-_]\w+)*|\S";
pub struct GlinerModel {
pub session: Mutex<ort::session::Session>,
pub tokenizer: Tokenizer,
pub splitter: Regex,
}
unsafe impl Send for GlinerModel {}
unsafe impl Sync for GlinerModel {}
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<GlinerModel>, String> {
registry.get_or_load::<GlinerModel>(model_id, |def| {
let model_dir = &def.path;
// GLiNER models keep onnx files in an "onnx" subfolder
let onnx_dir = model_dir.join("onnx");
let model_path = if onnx_dir.exists() {
onnx_dir.join("model.onnx")
} else {
model_dir.join("model.onnx")
};
let tokenizer_path = model_dir.join("tokenizer.json");
info!("Loading GLiNER model from: {}", model_dir.display());
let session = ort::session::Session::builder()
.map_err(|e| format!("Failed to create ORT session builder: {}", e))?
.commit_from_file(&model_path)
.map_err(|e| format!("Failed to load ONNX model: {}", e))?;
let tokenizer = Tokenizer::from_file(&tokenizer_path)
.map_err(|e| format!("Failed to load tokenizer: {}", e))?;
let splitter = Regex::new(WORD_REGEX)
.map_err(|e| format!("Failed to compile word regex: {}", e))?;
info!("GLiNER model loaded: {}", def.name);
Ok(GlinerModel { session: Mutex::new(session), tokenizer, splitter })
})
}

30
crates/jarvis-core/src/models/loaders/intent_classifier.rs Normal file
View file

@ -0,0 +1,30 @@
// intent-classifier crate wrapper
use std::sync::Arc;
use intent_classifier::IntentClassifier;
use crate::models::registry::ModelRegistry;
pub struct IntentClassifierModel {
pub classifier: IntentClassifier,
}
unsafe impl Send for IntentClassifierModel {}
unsafe impl Sync for IntentClassifierModel {}
// init is async (IntentClassifier::new().await), so we create it
// outside the registry and insert it after
pub async fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<IntentClassifierModel>, String> {
if let Some(existing) = registry.get::<IntentClassifierModel>(model_id) {
info!("IntentClassifier '{}' already loaded, reusing", model_id);
return Ok(existing);
}
info!("Initializing IntentClassifier...");
let classifier = IntentClassifier::new().await
.map_err(|e| format!("Failed to init IntentClassifier: {}", e))?;
info!("IntentClassifier initialized");
Ok(registry.insert(model_id, IntentClassifierModel { classifier }))
}

12
crates/jarvis-core/src/models/loaders/mod.rs Normal file
View file

@ -0,0 +1,12 @@
#[cfg(feature = "jarvis_app")]
pub mod embedding;
#[cfg(feature = "jarvis_app")]
pub mod gliner;
#[cfg(feature = "jarvis_app")]
pub mod ort_model;
#[cfg(feature = "jarvis_app")]
pub mod intent_classifier;
#[cfg(feature = "vosk")]
pub mod vosk;
#[cfg(feature = "nnnoiseless")]
pub mod nnnoiseless;

110
crates/jarvis-core/src/models/loaders/nnnoiseless.rs Normal file
View file

@ -0,0 +1,110 @@
// nnnoiseless - used for both noise suppression and VAD.
// each consumer needs its own DenoiseState (stateful per-stream),
// so this doesn't go through the registry. just centralizes creation.
use nnnoiseless::DenoiseState;
use crate::config;
// noise suppression instance
pub struct NnnoiselessNS {
state: Box<DenoiseState<'static>>,
buffer: Vec<f32>,
}
impl NnnoiselessNS {
pub fn new() -> Self {
Self {
state: DenoiseState::new(),
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
}
}
pub fn process(&mut self, input: &[i16]) -> Vec<i16> {
self.buffer.extend(input.iter().map(|&s| s as f32));
let frame_size = config::NNNOISELESS_FRAME_SIZE;
let full_frames = self.buffer.len() / frame_size;
if full_frames == 0 {
return input.to_vec();
}
let mut output: Vec<i16> = Vec::with_capacity(full_frames * frame_size);
let mut input_frame = [0.0f32; 480];
let mut output_frame = [0.0f32; 480];
let consumed = full_frames * frame_size;
for i in 0..full_frames {
let offset = i * frame_size;
input_frame.copy_from_slice(&self.buffer[offset..offset + frame_size]);
let _ = self.state.process_frame(&mut output_frame, &input_frame);
for &sample in &output_frame {
let clamped = sample.clamp(i16::MIN as f32, i16::MAX as f32);
output.push(clamped as i16);
}
}
// keep leftover samples (single drain at the end)
self.buffer.drain(..consumed);
output
}
pub fn reset(&mut self) {
self.buffer.clear();
}
}
// VAD instance
pub struct NnnoiselessVAD {
state: Box<DenoiseState<'static>>,
buffer: Vec<f32>,
}
impl NnnoiselessVAD {
pub fn new() -> Self {
Self {
state: DenoiseState::new(),
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
}
}
pub fn detect(&mut self, input: &[i16]) -> (bool, f32) {
self.buffer.extend(input.iter().map(|&s| s as f32));
let frame_size = config::NNNOISELESS_FRAME_SIZE;
let full_frames = self.buffer.len() / frame_size;
if full_frames == 0 {
return (true, 0.5);
}
let mut total_vad = 0.0f32;
let mut input_frame = [0.0f32; 480];
let mut output_frame = [0.0f32; 480];
let consumed = full_frames * frame_size;
for i in 0..full_frames {
let offset = i * frame_size;
input_frame.copy_from_slice(&self.buffer[offset..offset + frame_size]);
let vad_prob = self.state.process_frame(&mut output_frame, &input_frame);
total_vad += vad_prob;
}
// single drain
self.buffer.drain(..consumed);
let avg_vad = total_vad / full_frames as f32;
let is_voice = avg_vad >= config::VAD_NNNOISELESS_THRESHOLD;
(is_voice, avg_vad)
}
pub fn reset(&mut self) {
self.state = DenoiseState::new();
self.buffer.clear();
}
}

44
crates/jarvis-core/src/models/loaders/ort_model.rs Normal file
View file

@ -0,0 +1,44 @@
// generic ORT model - session + optional tokenizer.
// for models like BERT (tiny, distil, mini) that can serve
// multiple tasks (intent, NER, text classification, etc.)
use std::sync::Arc;
use parking_lot::Mutex;
use tokenizers::Tokenizer;
use crate::models::registry::ModelRegistry;
pub struct OrtModel {
pub session: Mutex<ort::session::Session>,
pub tokenizer: Option<Tokenizer>,
}
unsafe impl Send for OrtModel {}
unsafe impl Sync for OrtModel {}
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<OrtModel>, String> {
registry.get_or_load::<OrtModel>(model_id, |def| {
let model_dir = &def.path;
let onnx_path = model_dir.join("model.onnx");
info!("Loading ORT model from: {}", model_dir.display());
let session = ort::session::Session::builder()
.map_err(|e| format!("ORT session builder error: {}", e))?
.commit_from_file(&onnx_path)
.map_err(|e| format!("Failed to load ONNX model '{}': {}", onnx_path.display(), e))?;
let tokenizer_path = model_dir.join("tokenizer.json");
let tokenizer = if tokenizer_path.exists() {
Some(
Tokenizer::from_file(&tokenizer_path)
.map_err(|e| format!("Failed to load tokenizer: {}", e))?
)
} else {
None
};
info!("ORT model loaded: {}", def.name);
Ok(OrtModel { session: Mutex::new(session), tokenizer })
})
}

33
crates/jarvis-core/src/models/loaders/vosk.rs Normal file
View file

@ -0,0 +1,33 @@
// vosk speech recognition model
use std::sync::Arc;
use vosk::Model;
use crate::models::registry::ModelRegistry;
pub struct VoskModel {
pub model: Model,
}
unsafe impl Send for VoskModel {}
unsafe impl Sync for VoskModel {}
// load a vosk model by path through the registry.
// vosk models aren't in the catalog (they use their own directory structure),
// so we pass the path directly and use model_id for dedup.
// @ToDo: Consider moving to catalog
pub fn load(registry: &ModelRegistry, model_id: &str, model_path: &str) -> Result<Arc<VoskModel>, String> {
// check if already loaded
if let Some(existing) = registry.get::<VoskModel>(model_id) {
info!("Vosk model '{}' already loaded, reusing", model_id);
return Ok(existing);
}
info!("Loading Vosk model from: {}", model_path);
let model = Model::new(model_path)
.ok_or_else(|| format!("Failed to load Vosk model from: {}", model_path))?;
info!("Vosk model loaded: {}", model_id);
Ok(registry.insert(model_id, VoskModel { model }))
}

108
crates/jarvis-core/src/models/registry.rs Normal file
View file

@ -0,0 +1,108 @@
use std::any::Any;
use std::collections::HashMap;
use std::sync::Arc;
use parking_lot::{Mutex, RwLock};
use super::structs::ModelDef;
// central model registry. loads models once and shares them between components.
// completely type-agnostic
pub struct ModelRegistry {
loaded: Mutex<HashMap<String, Arc<dyn Any + Send + Sync>>>,
catalog: RwLock<Vec<ModelDef>>,
}
impl ModelRegistry {
pub fn new() -> Self {
Self {
loaded: Mutex::new(HashMap::new()),
catalog: RwLock::new(Vec::new()),
}
}
pub fn set_catalog(&self, defs: Vec<ModelDef>) {
*self.catalog.write() = defs;
}
// read access to catalog without cloning the whole vec
pub fn with_catalog<R>(&self, f: impl FnOnce(&[ModelDef]) -> R) -> R {
f(&self.catalog.read())
}
pub fn get_model_def(&self, id: &str) -> Option<ModelDef> {
self.catalog.read().iter().find(|m| m.id == id).cloned()
}
// get a loaded model, downcasted to the expected type
pub fn get<T: 'static + Send + Sync>(&self, id: &str) -> Option<Arc<T>> {
self.loaded.lock()
.get(id)?
.clone()
.downcast::<T>()
.ok()
}
// get or load a model. if two components request the same id,
// the model only loads once.
//
// the lock is released before calling the loader to avoid deadlocks
// if the loader tries to load a dependency through the registry.
pub fn get_or_load<T: 'static + Send + Sync>(
&self,
id: &str,
loader: impl FnOnce(&ModelDef) -> Result<T, String>,
) -> Result<Arc<T>, String> {
// fast path: already loaded
if let Some(existing) = self.get::<T>(id) {
info!("Model '{}' already loaded, reusing", id);
return Ok(existing);
}
// grab model def (releases catalog lock immediately)
let def = self.get_model_def(id)
.ok_or_else(|| format!("Model '{}' not found in catalog", id))?;
// run loader without holding any lock
info!("Loading model '{}' from {:?}...", id, def.path);
let model = loader(&def)?;
let arc = Arc::new(model);
// insert (check again in case another thread loaded it meanwhile)
let mut map = self.loaded.lock();
if let Some(existing) = map.get(id) {
if let Ok(typed) = existing.clone().downcast::<T>() {
info!("Model '{}' was loaded by another thread, reusing", id);
return Ok(typed);
}
}
map.insert(id.to_string(), arc.clone());
info!("Model '{}' loaded and registered", id);
Ok(arc)
}
// insert a model directly (for models not in the catalog,
// or loaded through non-standard means like async init)
pub fn insert<T: 'static + Send + Sync>(&self, id: &str, model: T) -> Arc<T> {
let arc = Arc::new(model);
self.loaded.lock().insert(id.to_string(), arc.clone());
arc
}
pub fn unload(&self, id: &str) -> bool {
let removed = self.loaded.lock().remove(id).is_some();
if removed {
info!("Model '{}' unloaded from registry", id);
}
removed
}
pub fn is_loaded(&self, id: &str) -> bool {
self.loaded.lock().contains_key(id)
}
pub fn loaded_ids(&self) -> Vec<String> {
self.loaded.lock().keys().cloned().collect()
}
}

38
crates/jarvis-core/src/models/structs.rs Normal file
View file

@ -0,0 +1,38 @@
use std::path::PathBuf;
use serde::{Serialize, Deserialize};
// tasks that components can request a backend for
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Task {
Intent,
Slots,
Vad,
NoiseSuppression,
Stt,
}
// metadata about a model, parsed from model.toml on disk
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelDef {
pub id: String,
pub name: String,
pub tasks: Vec<Task>,
#[serde(default)]
pub description: String,
// set at runtime after scanning the folder
#[serde(skip)]
pub path: PathBuf,
}
// a selectable option for a task (shown in UI / stored in settings)
#[derive(Debug, Clone, Serialize)]
pub struct BackendOption {
pub id: String,
pub name: String,
// if Some, this option uses a model from the registry.
// if None, it's a code-only backend (like energy VAD) or disabled.
pub model_id: Option<String>,
}

0
crates/jarvis-core/src/vosk_models.rs → crates/jarvis-core/src/models/vosk_models.rs
View file

2
crates/jarvis-core/src/recorder/pvrecorder.rs
View file

@ -19,7 +19,7 @@ pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
match pv_recorder {
Ok(pv) => {
// store
RECORDER.set(pv);
let _ = RECORDER.set(pv);
// success
true

38
crates/jarvis-core/src/slots.rs
View file

@ -4,37 +4,37 @@ use std::collections::HashMap;
use once_cell::sync::OnceCell;
use crate::commands::{SlotDefinition, SlotValue};
use crate::config::structs::SlotExtractionEngine;
use crate::DB;
use crate::{models, DB};
static SLOT_ENGINE: OnceCell<SlotExtractionEngine> = OnceCell::new();
static BACKEND: OnceCell<String> = OnceCell::new();
pub fn init() -> Result<(), String> {
if SLOT_ENGINE.get().is_some() {
if BACKEND.get().is_some() {
return Ok(());
}
let engine = DB.get()
.map(|db| db.read().slot_extraction_engine)
.unwrap_or(SlotExtractionEngine::None);
let backend = DB.get()
.map(|db| db.read().slots_backend.clone())
.unwrap_or_else(|| "none".to_string());
SLOT_ENGINE.set(engine).map_err(|_| "Slot engine already set")?;
BACKEND.set(backend.clone()).map_err(|_| "Slot backend already set")?;
match engine {
SlotExtractionEngine::None => {
match backend.as_str() {
"none" => {
info!("Slot extraction disabled");
}
SlotExtractionEngine::GLiNER => {
info!("Initializing GLiNER slot extraction backend.");
gliner::init()?;
info!("GLiNER slot extraction backend initialized.");
// any model ID is treated as a GLiNER model for now
model_id => {
info!("Initializing GLiNER slot extraction with model '{}'.", model_id);
let model = models::gliner::load(models::registry(), model_id)?;
gliner::init_with_model(model)?;
info!("GLiNER slot extraction initialized.");
}
}
Ok(())
}
// Extract slot values from text using the configured engine
pub fn extract(
text: &str,
slots: &HashMap<String, SlotDefinition>,
@ -43,9 +43,9 @@ pub fn extract(
return HashMap::new();
}
match SLOT_ENGINE.get().unwrap_or(&SlotExtractionEngine::None) {
SlotExtractionEngine::None => HashMap::new(),
SlotExtractionEngine::GLiNER => {
match BACKEND.get().map(|s| s.as_str()).unwrap_or("none") {
"none" => HashMap::new(),
_ => {
match gliner::extract(text, slots) {
Ok(result) => result,
Err(e) => {
@ -55,4 +55,4 @@ pub fn extract(
}
}
}
}
}

244
crates/jarvis-core/src/slots/gliner.rs
View file

@ -2,123 +2,43 @@
// https://github.com/fbilhaut/gline-rs
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use once_cell::sync::OnceCell;
use parking_lot::Mutex;
use ndarray::Array;
use regex::Regex;
use tokenizers::Tokenizer;
use ort::value::Tensor;
pub mod structs;
use structs::GlinerModelInfo;
use std::fs;
use crate::commands::{SlotDefinition, SlotValue};
use crate::{APP_DIR, i18n};
use crate::models::gliner::GlinerModel;
// MODEL STATE
static MODEL: OnceCell<Arc<GlinerModel>> = OnceCell::new();
struct GlinerModel {
session: ort::session::Session,
tokenizer: Tokenizer,
splitter: Regex,
}
unsafe impl Send for GlinerModel {}
unsafe impl Sync for GlinerModel {}
static MODEL: OnceCell<Mutex<GlinerModel>> = OnceCell::new();
// GLiNER defaults (same as gline-rs Parameters::default())
// GLiNER defaults
const THRESHOLD: f32 = 0.3;
const MAX_WIDTH: usize = 12;
const MAX_LENGTH: usize = 512;
// applied after decoding
const MIN_CONFIDENCE: f32 = 0.4;
// word splitting regex (gline-rs RegexSplitter default)
const WORD_REGEX: &str = r"\w+(?:[-_]\w+)*|\S";
// INIT
pub fn init() -> Result<(), String> {
if MODEL.get().is_some() {
return Ok(());
}
let variant = crate::DB.get()
.map(|db| db.read().gliner_model.clone())
.unwrap_or_default();
let language = i18n::get_language();
let (model_dir, onnx_file) = if variant.is_empty() {
(select_model_dir(), "model.onnx".to_string())
} else {
crate::gliner_models::resolve_model(&variant, &language)
.unwrap_or_else(|| (select_model_dir(), "model.onnx".to_string()))
};
let model_path = model_dir.join("onnx").join(&onnx_file);
let tokenizer_path = model_dir.join("tokenizer.json");
info!("Loading GLiNER model from: {}, variant {}", model_dir.display(), variant);
let session = ort::session::Session::builder()
.map_err(|e| format!("Failed to create ort session builder: {}", e))?
.commit_from_file(&model_path)
.map_err(|e| format!("Failed to load ONNX model: {}", e))?;
let tokenizer = Tokenizer::from_file(&tokenizer_path)
.map_err(|e| format!("Failed to load tokenizer: {}", e))?;
let splitter = Regex::new(WORD_REGEX)
.map_err(|e| format!("Failed to compile word regex: {}", e))?;
MODEL.set(Mutex::new(GlinerModel { session, tokenizer, splitter }))
.map_err(|_| "GLiNER model already initialized".to_string())?;
info!("GLiNER model loaded");
pub fn init_with_model(model: Arc<GlinerModel>) -> Result<(), String> {
MODEL.set(model).map_err(|_| "GLiNER model already initialized".to_string())?;
info!("GLiNER slot extraction ready");
Ok(())
}
fn select_model_dir() -> PathBuf {
let base = APP_DIR.join("resources").join("models");
// word splitting
match i18n::get_language().as_str() {
"en" => {
let path = base.join("gliner_small-v2.1");
if path.exists() { return path; }
}
_ => {}
}
// multilingual (covers RU, UA, EN)
let multi = base.join("gliner_multi-v2.1");
if multi.exists() { return multi; }
// fallback
base.join("gliner_small-v2.1")
}
// WORD SPLITTING
struct WordToken {
struct WordToken<'a> {
start: usize,
end: usize,
text: String,
text: &'a str,
}
fn split_words(splitter: &Regex, text: &str, limit: Option<usize>) -> Vec<WordToken> {
fn split_words<'a>(text: &'a str, model: &GlinerModel, limit: Option<usize>) -> Vec<WordToken<'a>> {
let mut tokens = Vec::new();
for m in splitter.find_iter(text) {
for m in model.splitter.find_iter(text) {
tokens.push(WordToken {
start: m.start(),
end: m.end(),
text: m.as_str().to_string(),
text: m.as_str(),
});
if let Some(lim) = limit {
if tokens.len() >= lim { break; }
@ -127,7 +47,7 @@ fn split_words(splitter: &Regex, text: &str, limit: Option<usize>) -> Vec<WordTo
tokens
}
// PROMPT CONSTRUCTION
// prompt construction
//
// GLiNER prompt format:
// [<<ENT>>, label1_w1, label1_w2, <<ENT>>, label2_w1, ..., <<SEP>>, word1, word2, ..., wordN]
@ -137,20 +57,20 @@ fn build_prompt(entities: &[&str], words: &[WordToken]) -> (Vec<String>, usize)
for entity in entities {
prompt.push("<<ENT>>".to_string());
prompt.push(entity.to_string()); // whole string, no split
prompt.push(entity.to_string());
}
prompt.push("<<SEP>>".to_string());
let entities_len = prompt.len();
for w in words {
prompt.push(w.text.clone());
prompt.push(w.text.to_string());
}
(prompt, entities_len)
}
// ENCODING
// encoding
struct EncodedBatch {
input_ids: ndarray::Array2<i64>,
@ -161,8 +81,7 @@ struct EncodedBatch {
}
fn encode_single(
tokenizer: &Tokenizer,
_text: &str,
model: &GlinerModel,
entities: &[&str],
words: &[WordToken],
) -> Result<EncodedBatch, String> {
@ -174,7 +93,7 @@ fn encode_single(
let mut entity_tokens: usize = 0;
for (pos, word) in prompt.iter().enumerate() {
let encoding = tokenizer.encode(word.as_str(), false)
let encoding = model.tokenizer.encode(word.as_str(), false)
.map_err(|e| format!("Tokenizer encode error: {}", e))?;
let ids = encoding.get_ids().to_vec();
total_tokens += ids.len();
@ -184,13 +103,13 @@ fn encode_single(
word_encodings.push(ids);
}
// text_offset: index where text tokens start (after BOS + entity tokens)
let text_offset = entity_tokens + 1;
// DEBUG
debug!("GLiNER prompt ({} total, ent_len={}, text_offset={}):", prompt.len(), ent_len, text_offset);
for (i, (word, enc)) in prompt.iter().zip(word_encodings.iter()).enumerate() {
debug!(" [{}]{} '{}' -> {:?}", i, if i < ent_len { " ENT" } else { " TXT" }, word, enc);
if log::log_enabled!(log::Level::Debug) {
debug!("GLiNER prompt ({} total, ent_len={}, text_offset={}):", prompt.len(), ent_len, text_offset);
for (i, (word, enc)) in prompt.iter().zip(word_encodings.iter()).enumerate() {
debug!(" [{}]{} '{}' -> {:?}", i, if i < ent_len { " ENT" } else { " TXT" }, word, enc);
}
}
let mut input_ids = Array::zeros((1, total_tokens));
@ -205,18 +124,15 @@ fn encode_single(
attention_masks[[0, idx]] = 1;
idx += 1;
// encode each word - matching gline-rs idx-based logic exactly
for word_enc in word_encodings.iter() {
for (token_idx, &token_id) in word_enc.iter().enumerate() {
input_ids[[0, idx]] = token_id as i64;
attention_masks[[0, idx]] = 1;
// word mask: only for text tokens (past text_offset), first sub-token only
if idx >= text_offset && token_idx == 0 {
word_masks[[0, idx]] = word_id;
}
idx += 1;
}
// increment word_id for any word whose tokens end past text_offset
if idx >= text_offset {
word_id += 1;
}
@ -229,9 +145,11 @@ fn encode_single(
let mut text_lengths = Array::zeros((1, 1));
text_lengths[[0, 0]] = (text_word_count + 1) as i64;
debug!("GLiNER input_ids: {:?}", input_ids.as_slice().unwrap());
debug!("GLiNER word_masks: {:?}", word_masks.as_slice().unwrap());
debug!("GLiNER text_lengths: {}", text_word_count);
if log::log_enabled!(log::Level::Debug) {
debug!("GLiNER input_ids: {:?}", input_ids.as_slice().unwrap());
debug!("GLiNER word_masks: {:?}", word_masks.as_slice().unwrap());
debug!("GLiNER text_lengths: {}", text_word_count);
}
Ok(EncodedBatch {
input_ids,
@ -242,7 +160,7 @@ fn encode_single(
})
}
// SPAN TENSORS
// span tensors
fn make_span_tensors(num_words: usize, max_width: usize) -> (ndarray::Array3<i64>, ndarray::Array2<bool>) {
let num_spans = num_words * max_width;
@ -264,7 +182,7 @@ fn make_span_tensors(num_words: usize, max_width: usize) -> (ndarray::Array3<i64
(span_idx, span_mask)
}
// DECODE + GREEDY SEARCH
// decode + greedy search
fn sigmoid(x: f32) -> f32 {
1.0 / (1.0 + (-x).exp())
@ -323,56 +241,43 @@ fn decode_and_search(
}
spans.sort_unstable_by(|a, b| (a.start, a.end).cmp(&(b.start, b.end)));
greedy_flat(&spans)
greedy_flat(spans)
}
fn greedy_flat(spans: &[Entity]) -> Vec<Entity> {
if spans.is_empty() {
return Vec::new();
// takes ownership, filters in place - no cloning
fn greedy_flat(mut spans: Vec<Entity>) -> Vec<Entity> {
if spans.len() <= 1 {
return spans;
}
let mut result: Vec<Entity> = Vec::new();
let mut keep = vec![false; spans.len()];
let mut prev = 0usize;
let mut next = 1usize;
while next < spans.len() {
let p = &spans[prev];
let n = &spans[next];
for next in 1..spans.len() {
let no_overlap = spans[next].start >= spans[prev].end
|| spans[prev].start >= spans[next].end;
if n.start >= p.end || p.start >= n.end {
result.push(Entity {
text: p.text.clone(),
label: p.label.clone(),
prob: p.prob,
start: p.start,
end: p.end,
});
if no_overlap {
keep[prev] = true;
prev = next;
} else if p.prob < n.prob {
} else if spans[prev].prob < spans[next].prob {
prev = next;
}
next += 1;
}
keep[prev] = true;
let last = &spans[prev];
result.push(Entity {
text: last.text.clone(),
label: last.label.clone(),
prob: last.prob,
start: last.start,
end: last.end,
});
result
let mut idx = 0;
spans.retain(|_| { let k = keep[idx]; idx += 1; k });
spans
}
// PUBLIC API
// public extract API
pub fn extract(
text: &str,
slots: &HashMap<String, SlotDefinition>,
) -> Result<HashMap<String, SlotValue>, String> {
let mut model = MODEL.get().ok_or("GLiNER not initialized")?.lock();
let model = MODEL.get().ok_or("GLiNER not initialized")?;
let mut label_to_slots: HashMap<&str, Vec<&str>> = HashMap::new();
for (slot_name, def) in slots {
@ -392,12 +297,12 @@ pub fn extract(
debug!("GLiNER extract: text='{}', labels={:?}", text, labels);
let words = split_words(&model.splitter, text, Some(MAX_LENGTH));
let words = split_words(text, &model, Some(MAX_LENGTH));
if words.is_empty() {
return Ok(HashMap::new());
}
let encoded = encode_single(&model.tokenizer, text, &labels, &words)?;
let encoded = encode_single(&model, &labels, &words)?;
let (span_idx, span_mask) = make_span_tensors(encoded.num_words, MAX_WIDTH);
@ -408,7 +313,8 @@ pub fn extract(
let t_span_idx = Tensor::from_array(span_idx).map_err(|e| format!("tensor: {}", e))?;
let t_span_mask = Tensor::from_array(span_mask).map_err(|e| format!("tensor: {}", e))?;
let outputs = model.session.run(
let mut session = model.session.lock();
let outputs = session.run(
ort::inputs! {
"input_ids" => t_input_ids,
"attention_mask" => t_attn,
@ -425,27 +331,29 @@ pub fn extract(
let logits_shape: Vec<usize> = shape.iter().map(|&d| d as usize).collect();
debug!("GLiNER logits shape: {:?}, data len: {}", logits_shape, logits_data.len());
let max_logit = logits_data.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
debug!("GLiNER max logit: {:.4}, sigmoid: {:.4}", max_logit, sigmoid(max_logit));
// debug dump - gated so sigmoid/loop don't run in release
if log::log_enabled!(log::Level::Debug) {
debug!("GLiNER logits shape: {:?}, data len: {}", logits_shape, logits_data.len());
let max_logit = logits_data.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
debug!("GLiNER max logit: {:.4}, sigmoid: {:.4}", max_logit, sigmoid(max_logit));
// dump all scores above 5%
let num_words = logits_shape.get(1).copied().unwrap_or(0);
let dim_mw = logits_shape.get(2).copied().unwrap_or(0);
let dim_e = logits_shape.get(3).copied().unwrap_or(0);
for start in 0..num_words {
for width in 0..dim_mw.min(num_words - start) {
for class_idx in 0..dim_e {
let flat_idx = start * dim_mw * dim_e + width * dim_e + class_idx;
if flat_idx < logits_data.len() {
let score = logits_data[flat_idx];
let prob = sigmoid(score);
if prob > 0.05 {
let end = start + width;
let w_start = if start < words.len() { &words[start].text } else { "?" };
let w_end = if end < words.len() { &words[end].text } else { "?" };
debug!(" span[{}..{}] '{}'->'{}' label={} score={:.3} prob={:.3}",
start, end, w_start, w_end, labels.get(class_idx).unwrap_or(&"?"), score, prob);
let num_words = logits_shape.get(1).copied().unwrap_or(0);
let dim_mw = logits_shape.get(2).copied().unwrap_or(0);
let dim_e = logits_shape.get(3).copied().unwrap_or(0);
for start in 0..num_words {
for width in 0..dim_mw.min(num_words - start) {
for class_idx in 0..dim_e {
let flat_idx = start * dim_mw * dim_e + width * dim_e + class_idx;
if flat_idx < logits_data.len() {
let score = logits_data[flat_idx];
let prob = sigmoid(score);
if prob > 0.05 {
let end = start + width;
let w_start = if start < words.len() { words[start].text } else { "?" };
let w_end = if end < words.len() { words[end].text } else { "?" };
debug!(" span[{}..{}] '{}'->'{}' label={} score={:.3} prob={:.3}",
start, end, w_start, w_end, labels.get(class_idx).unwrap_or(&"?"), score, prob);
}
}
}
}
@ -484,4 +392,4 @@ fn parse_slot_value(text: &str) -> SlotValue {
return SlotValue::Number(n);
}
SlotValue::Text(text.to_string())
}
}

7
crates/jarvis-core/src/slots/gliner/structs.rs
View file

@ -1,7 +0,0 @@
#[derive(Debug, Clone)]
pub struct GlinerModelInfo {
pub model_dir: String,
pub file_name: String,
pub display_name: String,
pub value: String,
}

22
crates/jarvis-core/src/stt.rs
View file

@ -5,9 +5,6 @@ use crate::config;
use once_cell::sync::OnceCell;
use crate::config::structs::SpeechToTextEngine;
use crate::vosk_models;
// use vosk_models::{scan_vosk_models, get_model_path, VoskModelInfo};
pub use self::vosk::init_vosk;
pub use self::vosk::recognize_wake_word;
pub use self::vosk::recognize_speech;
@ -16,21 +13,18 @@ pub use self::vosk::reset_wake_recognizer;
static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new();
pub fn init() -> Result<(), ()> {
pub fn init() -> Result<(), String> {
if STT_TYPE.get().is_some() {
return Ok(());
} // already initialized
}
// set default stt type
// @TODO. Make it configurable?
STT_TYPE.set(config::DEFAULT_SPEECH_TO_TEXT_ENGINE).unwrap();
STT_TYPE.set(config::DEFAULT_SPEECH_TO_TEXT_ENGINE)
.map_err(|_| "STT type already set".to_string())?;
// load given recorder
match STT_TYPE.get().unwrap() {
SpeechToTextEngine::Vosk => {
// Init Vosk
info!("Initializing Vosk STT backend.");
vosk::init_vosk();
vosk::init_vosk()?;
info!("STT backend initialized.");
}
}
@ -45,9 +39,3 @@ pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
vosk::recognize_speech(data)
}
}
// pub fn recognize(data: &[i16], partial: bool) -> Option<String> {
// match STT_TYPE.get().unwrap() {
// SpeechToTextEngine::Vosk => vosk::recognize(data, partial),
// }
// }

113
crates/jarvis-core/src/stt/vosk.rs
View file

@ -1,47 +1,50 @@
use once_cell::sync::OnceCell;
use vosk::{DecodingState, Model, Recognizer};
use vosk::{DecodingState, Recognizer};
use std::sync::Arc;
use parking_lot::Mutex;
use std::sync::Mutex;
// use crate::config::VOSK_MODEL_PATH;
use crate::{stt::vosk_models, i18n, config};
use crate::{vosk_models, i18n, config, models};
use crate::models::vosk::VoskModel;
use crate::DB;
static MODEL: OnceCell<Model> = OnceCell::new();
// the model Arc keeps the vosk::Model alive for the recognizers
static VOSK_MODEL: OnceCell<Arc<VoskModel>> = OnceCell::new();
static WAKE_RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
static SPEECH_RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
pub fn init_vosk() -> Result<(), String> {
if MODEL.get().is_some() {
if VOSK_MODEL.get().is_some() {
return Ok(());
} // already initialized
}
let model_path = get_configured_model_path()?;
info!("Loading Vosk model from: {}", model_path.display());
let model_id = format!("vosk:{}", model_path.display());
let model = Model::new(model_path.to_str().unwrap())
.ok_or_else(|| format!("Failed to load Vosk model from: {}", model_path.display()))?;
// load through registry (shared if anything else needs the same model)
let vosk = models::vosk::load(
models::registry(),
&model_id,
model_path.to_str().unwrap(),
)?;
// language-specific wake grammar
let lang = i18n::get_language();
let wake_grammar = config::get_wake_grammar(&lang);
info!("Wake grammar for '{}': {:?}", lang, wake_grammar);
//let mut recognizer = Recognizer::new(&model, 16000.0)
// .ok_or("Failed to create Vosk recognizer")?;
let mut wake_recognizer = Recognizer::new_with_grammar(&model, 16000.0, wake_grammar)
let mut wake_recognizer = Recognizer::new_with_grammar(&vosk.model, 16000.0, wake_grammar)
.ok_or("Failed to create wake word recognizer")?;
wake_recognizer.set_max_alternatives(1); // required for confidence check later on
wake_recognizer.set_max_alternatives(1);
let mut speech_recognizer = Recognizer::new(&model, 16000.0)
let mut speech_recognizer = Recognizer::new(&vosk.model, 16000.0)
.ok_or("Failed to create speech recognizer")?;
speech_recognizer.set_max_alternatives(config::VOSK_SPEECH_RECOGNIZER_MAX_ALTERNATIVES);
speech_recognizer.set_words(config::VOSK_SPEECH_RECOGNIZER_WORDS);
speech_recognizer.set_partial_words(config::VOSK_SPEECH_PARTIAL_WORDS);
MODEL.set(model).map_err(|_| "Model already set")?;
VOSK_MODEL.set(vosk).map_err(|_| "Model already set")?;
WAKE_RECOGNIZER.set(Mutex::new(wake_recognizer)).map_err(|_| "Wake recognizer already set")?;
SPEECH_RECOGNIZER.set(Mutex::new(speech_recognizer)).map_err(|_| "Speech recognizer already set")?;
@ -50,17 +53,15 @@ pub fn init_vosk() -> Result<(), String> {
pub fn recognize_wake_word(data: &[i16]) -> Option<(String, f32)> {
let mut recognizer = WAKE_RECOGNIZER.get()?.lock().unwrap();
let mut recognizer = WAKE_RECOGNIZER.get()?.lock();
match recognizer.accept_waveform(data) {
Ok(DecodingState::Running) => {
// partials don't have confidence, skip them
None
}
Ok(DecodingState::Finalized) => {
let result = recognizer.result();
// compensate confidence issues
if let Some(alternatives) = result.multiple() {
if let Some(best) = alternatives.alternatives.first() {
if !best.text.is_empty() {
@ -77,7 +78,7 @@ pub fn recognize_wake_word(data: &[i16]) -> Option<(String, f32)> {
pub fn recognize_speech(data: &[i16]) -> Option<String> {
let mut recognizer = SPEECH_RECOGNIZER.get()?.lock().unwrap();
let mut recognizer = SPEECH_RECOGNIZER.get()?.lock();
match recognizer.accept_waveform(data) {
Ok(DecodingState::Finalized) => {
@ -92,65 +93,16 @@ pub fn recognize_speech(data: &[i16]) -> Option<String> {
pub fn reset_speech_recognizer() {
if let Some(recognizer) = SPEECH_RECOGNIZER.get() {
recognizer.lock().unwrap().reset();
recognizer.lock().reset();
}
}
pub fn reset_wake_recognizer() {
if let Some(recognizer) = WAKE_RECOGNIZER.get() {
recognizer.lock().unwrap().reset();
recognizer.lock().reset();
}
}
// pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
// let state = RECOGNIZER
// .get()
// .unwrap()
// .lock()
// .unwrap()
// .accept_waveform(data);
// match state {
// Ok(ds) => {
// match ds {
// DecodingState::Running => {
// if include_partial {
// Some(
// RECOGNIZER
// .get()
// .unwrap()
// .lock()
// .unwrap()
// .partial_result()
// .partial
// .into(),
// )
// } else {
// None
// }
// }
// DecodingState::Finalized => {
// // Result will always be multiple because we called set_max_alternatives
// RECOGNIZER
// .get()
// .unwrap()
// .lock()
// .unwrap()
// .result()
// .multiple()
// .and_then(|m| m.alternatives.first().map(|a| a.text.to_string()))
// }
// DecodingState::Failed => None,
// }
// },
// Err(err) => {
// error!("Vosk accept waveform error.\nError details: {}", err);
// None
// }
// }
// }
fn get_configured_model_path() -> Result<std::path::PathBuf, String> {
// try to get from settings
if let Some(db) = DB.get() {
@ -167,11 +119,10 @@ fn get_configured_model_path() -> Result<std::path::PathBuf, String> {
let available = vosk_models::scan_vosk_models();
let language = i18n::get_language();
// try language match first
let lang_code = match language.as_str() {
"ru" => "ru",
"en" => "us", // vosk uses "us" not "en"
"ua" => "uk", // vosk uses "uk" not "ua"
"en" => "us",
"ua" => "uk",
other => other,
};
@ -180,7 +131,6 @@ fn get_configured_model_path() -> Result<std::path::PathBuf, String> {
return Ok(matched.path.clone());
}
// fallback to first available
if let Some(first) = available.first() {
info!("Auto-detected Vosk model (no language match): {}", first.name);
return Ok(first.path.clone());
@ -194,14 +144,3 @@ fn get_configured_model_path() -> Result<std::path::PathBuf, String> {
Err("No Vosk models found".into())
}
// pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
// let mut result = Vec::with_capacity(input_data.len() / 2);
// result.extend(
// input_data
// .chunks_exact(2)
// .map(|chunk| chunk[0] / 2 + chunk[1] / 2),
// );
// result
// }

29
crates/jarvis-core/src/voices.rs
View file

@ -13,9 +13,7 @@ pub use structs::*;
static VOICES: OnceCell<Vec<structs::VoiceConfig>> = OnceCell::new();
static CURRENT_VOICE_ID: OnceCell<RwLock<String>> = OnceCell::new();
pub fn init(default_voice: &str) -> Result<(), String> {
CURRENT_VOICE_ID.get_or_init(|| RwLock::new(default_voice.to_string()));
pub fn init(default_voice: &str, language: &str) -> Result<(), String> {
let voices = scan_voices()?;
if voices.is_empty() {
@ -26,7 +24,30 @@ pub fn init(default_voice: &str) -> Result<(), String> {
voices.len(),
voices.iter().map(|v| &v.voice.id).collect::<Vec<_>>()
);
// resolve which voice to use
let voice_id = if !default_voice.is_empty() && voices.iter().any(|v| v.voice.id == default_voice) {
default_voice.to_string()
} else {
// auto-detect: pick the first voice that supports the active language
let auto = voices.iter()
.find(|v| v.voice.languages.contains(&language.to_string()))
.or_else(|| voices.first());
match auto {
Some(v) => {
if default_voice.is_empty() {
info!("No voice configured, auto-selected '{}' for language '{}'", v.voice.id, language);
} else {
warn!("Voice '{}' not found, auto-selected '{}'", default_voice, v.voice.id);
}
v.voice.id.clone()
}
None => return Err("No compatible voice found".into()),
}
};
CURRENT_VOICE_ID.get_or_init(|| RwLock::new(voice_id));
VOICES.set(voices).map_err(|_| "Voices already initialized")?;
Ok(())