Big push driven by user feedback ("делай имбу") and web research on what
voice assistants need to be the ideal:
TTS backend abstraction (P0.1)
- new module crates/jarvis-core/src/tts/{mod,sapi,piper,silero}.rs
- TtsBackend trait with SapiBackend (current PowerShell), PiperBackend
(rhasspy/piper, neural quality), SileroBackend (python subprocess)
- JARVIS_TTS env var picks (sapi|piper|silero). Auto-detect Piper if
binary + voice present in tools/piper/. Falls back to SAPI on missing.
- SpeakOpts {lang, detached, raw} replaces ad-hoc args. text_utils
sanitiser applied unless raw=true.
- llm_fallback + lua/api/tts both routed through tts::backend().
- tools/piper/install.ps1 downloads piper.exe + ru_RU-irina-medium.onnx
from rhasspy releases + huggingface. Smoke-test included.
- tools/silero/silero_tts.py helper (PyTorch); rust spawns it as subprocess.
IMBA-1 Agentic LLM router
- crates/jarvis-app/src/llm_router.rs
- When fuzzy/intent matcher fails, LLM picks the closest command from the
full registry. Returns JSON {command_id, confidence, reason}.
- Threshold-gated re-dispatch via substitute phrase. JARVIS_LLM_ROUTER=1
enables; JARVIS_LLM_ROUTER_THRESHOLD overrides 0.55 default.
- Inserted in app.rs::execute_command between "no match" and existing
llm_fallback chat fallback.
IMBA-2 Long-term memory
- crates/jarvis-core/src/long_term_memory.rs — JSON store at
APP_CONFIG_DIR/long_term_memory.json. Atomic write-through.
- remember/recall/search/forget/all/build_context API.
- Lua bindings: jarvis.memory.* (5 functions).
- llm_fallback auto-injects relevant facts (substring search of prompt)
into system message before LLM call.
- Pack resources/commands/memory_pack/ with 4 commands: remember, recall,
forget, list.
IMBA-3 Profile switching (work/game/sleep/driving/default)
- crates/jarvis-core/src/profiles.rs — JSON profiles at APP_CONFIG_DIR/profiles/
Auto-seeds 5 defaults on first run with personality + allow/deny lists +
greetings + emoji icons.
- active_profile.txt persists choice across restart.
- Lua bindings: jarvis.profile.{active,set,list,allows,active_name}.
- llm_fallback prepends profile personality to system prompt.
- Pack resources/commands/profile_switch/ with 6 voice triggers.
IMBA-4 Multimodal screenshot + vision LLM
- crates/jarvis-core/src/lua/api/vision.rs — gated on HTTP sandbox.
- jarvis.vision.screenshot() captures via PowerShell System.Drawing.
- jarvis.vision.describe(prompt?) sends base64 PNG to Groq vision model
(default llama-3.2-11b-vision-preview, override via GROQ_VISION_MODEL).
- Pack resources/commands/vision/ with 2 commands: describe + read_error.
P0.2 Continuous conversation grace window
- config::CONVERSATION_GRACE_MS = 30_000.
- app.rs: after command result, if grace_ms > 0 keep listening WITHOUT
re-wake for the grace duration. Existing CMS_WAIT_DELAY back-dated so
the existing timeout fires at start + grace_ms.
Tests: 24/24 jarvis-core unit tests pass (including 5 text_utils).
Build: cargo build --release -p jarvis-app and -p jarvis-gui both succeed
on Windows MSVC (VS 2026 Enterprise vcvars64).
Notes for setup:
- Piper voice install: pwsh tools/piper/install.ps1 (downloads ~90 MB).
- GROQ_TOKEN needed for IMBA-1 (router) and IMBA-4 (vision).
- All features are opt-in via env vars or auto-detect; existing SAPI +
fuzzy match path remains the default.
305 lines
No EOL
11 KiB
Rust
305 lines
No EOL
11 KiB
Rust
pub mod structs;
|
||
use structs::AudioType;
|
||
use structs::RecorderType;
|
||
use structs::SpeechToTextEngine;
|
||
use structs::WakeWordEngine;
|
||
|
||
use once_cell::sync::Lazy;
|
||
use std::env;
|
||
use std::fs;
|
||
use std::path::PathBuf;
|
||
|
||
use platform_dirs::AppDirs;
|
||
|
||
#[cfg(feature="jarvis_app")]
|
||
use rustpotter::{
|
||
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig,
|
||
RustpotterConfig, ScoreMode,
|
||
};
|
||
|
||
use crate::config::structs::NoiseSuppressionBackend;
|
||
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
|
||
|
||
#[allow(dead_code)]
|
||
pub fn init_dirs() -> Result<(), String> {
|
||
// infer app dirs
|
||
if APP_DIRS.get().is_some() {
|
||
return Ok(());
|
||
}
|
||
|
||
// cache_dir, config_dir, data_dir, state_dir
|
||
APP_DIRS
|
||
.set(AppDirs::new(Some(BUNDLE_IDENTIFIER), false).unwrap())
|
||
.unwrap();
|
||
|
||
// setup directories
|
||
let mut config_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
|
||
let mut log_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
|
||
|
||
// create dirs, if required
|
||
if !config_dir.exists() {
|
||
if fs::create_dir_all(&config_dir).is_err() {
|
||
config_dir = env::current_dir().expect("Cannot infer the config directory");
|
||
fs::create_dir_all(&config_dir)
|
||
.expect("Cannot create config directory, access denied?");
|
||
}
|
||
}
|
||
|
||
if !log_dir.exists() {
|
||
if fs::create_dir_all(&log_dir).is_err() {
|
||
log_dir = env::current_dir().expect("Cannot infer the log directory");
|
||
fs::create_dir_all(&log_dir).expect("Cannot create log directory, access denied?");
|
||
}
|
||
}
|
||
|
||
// store inferred paths
|
||
APP_CONFIG_DIR.set(config_dir).unwrap();
|
||
APP_LOG_DIR.set(log_dir).unwrap();
|
||
|
||
Ok(())
|
||
}
|
||
|
||
/*
|
||
Defaults.
|
||
*/
|
||
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
|
||
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
|
||
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Vosk;
|
||
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
|
||
|
||
// backend defaults (string IDs)
|
||
pub const DEFAULT_INTENT_BACKEND: &str = "intent-classifier";
|
||
pub const DEFAULT_SLOTS_BACKEND: &str = "none";
|
||
pub const DEFAULT_VAD_BACKEND: &str = "energy";
|
||
|
||
pub const DEFAULT_VOICE: &str = "jarvis-remaster";
|
||
pub const SOUND_PATH: &str = "resources/sound"; // extended from SOUND_DIR (resources/sound)
|
||
pub const VOICES_PATH: &str = "voices"; // extended from SOUND_PATH (resources/sound)
|
||
|
||
pub const BUNDLE_IDENTIFIER: &str = "com.priler.jarvis";
|
||
pub const DB_FILE_NAME: &str = "app.db";
|
||
pub const LOG_FILE_NAME: &str = "log.txt";
|
||
pub const APP_VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION");
|
||
pub const AUTHOR_NAME: Option<&str> = option_env!("CARGO_PKG_AUTHORS");
|
||
pub const REPOSITORY_LINK: Option<&str> = option_env!("CARGO_PKG_REPOSITORY");
|
||
pub const UPSTREAM_REPOSITORY_LINK: &str = "https://github.com/Priler/jarvis";
|
||
pub const ISSUES_LINK: &str = "https://github.com/DmitryBykov-ISPO/J.A.R.V.I.S-rust/issues";
|
||
pub const PYTHON_FORK_LINK: &str = "https://github.com/DmitryBykov-ISPO/J.A.R.V.I.S-py";
|
||
|
||
/*
|
||
Tray.
|
||
*/
|
||
pub const TRAY_ICON: &str = "32x32.png";
|
||
pub const TRAY_TOOLTIP: &str = "Jarvis Voice Assistant";
|
||
|
||
// RUSPOTTER
|
||
pub const RUSPOTTER_MIN_SCORE: f32 = 0.62;
|
||
|
||
#[cfg(feature="jarvis_app")]
|
||
pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
|
||
RustpotterConfig {
|
||
fmt: AudioFmt::default(),
|
||
detector: DetectorConfig {
|
||
avg_threshold: 0.,
|
||
threshold: 0.5,
|
||
min_scores: 15,
|
||
score_ref: 0.22,
|
||
band_size: 5,
|
||
vad_mode: None,
|
||
score_mode: ScoreMode::Max,
|
||
eager: false,
|
||
// comparator_band_size: 5,
|
||
// comparator_ref: 0.22
|
||
},
|
||
filters: FiltersConfig {
|
||
gain_normalizer: GainNormalizationConfig {
|
||
// enabled: true,
|
||
// gain_ref: None,
|
||
// min_gain: 0.7,
|
||
// max_gain: 1.0,
|
||
enabled: false, // disable, now we have separate gain normalizer implementation
|
||
gain_ref: None,
|
||
min_gain: 0.7,
|
||
max_gain: 1.0,
|
||
},
|
||
band_pass: BandPassConfig {
|
||
enabled: true,
|
||
low_cutoff: 80.,
|
||
high_cutoff: 400.,
|
||
},
|
||
},
|
||
}
|
||
});
|
||
|
||
// PICOVOICE
|
||
pub const COMMANDS_PATH: &str = "resources/commands/";
|
||
pub const KEYWORDS_PATH: &str = "resources/picovoice/keywords/";
|
||
pub const DEFAULT_KEYWORD: &str = "jarvis_windows.ppn";
|
||
pub const DEFAULT_SENSITIVITY: f32 = 1.0;
|
||
|
||
// VOSK
|
||
// pub const VOSK_MODEL_PATH: &str = const_concat!(PUBLIC_PATH, "/vosk/model_small");
|
||
pub const VOSK_MODELS_PATH: &str = "resources/vosk";
|
||
pub const VOSK_MODEL_PATH: &str = "resources/vosk/model_small";
|
||
pub const VOSK_FETCH_PHRASE: &str = "джарвис";
|
||
pub const VOSK_MIN_RATIO: f64 = 70.0;
|
||
|
||
// 0.7 lenient, expect false positives
|
||
// 0.8 balanced
|
||
// 0.9 strict
|
||
// etc
|
||
pub const VOSK_WAKE_CONFIDENCE: f32 = 0.9;
|
||
|
||
pub const VOSK_SPEECH_RECOGNIZER_MAX_ALTERNATIVES: u16 = 3;
|
||
pub const VOSK_SPEECH_RECOGNIZER_WORDS: bool = false;
|
||
pub const VOSK_SPEECH_PARTIAL_WORDS: bool = false;
|
||
|
||
// IRE (intents recognition)
|
||
pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.75;
|
||
|
||
|
||
// embedding classifier
|
||
pub const EMBEDDING_MIN_CONFIDENCE: f64 = 0.70;
|
||
|
||
// AUDIO PROCESSING DEFAULTS
|
||
pub const DEFAULT_NOISE_SUPPRESSION: NoiseSuppressionBackend = NoiseSuppressionBackend::None;
|
||
pub const DEFAULT_GAIN_NORMALIZER: bool = false;
|
||
|
||
// VAD settings
|
||
pub const VAD_ENERGY_THRESHOLD: f32 = 100.0; // RMS threshold for energy-based VAD
|
||
pub const VAD_NNNOISELESS_THRESHOLD: f32 = 0.8; // probability threshold for nnnoiseless
|
||
pub const VAD_SILENCE_FRAMES: u32 = 15; // frames of silence before speech end (~480ms)
|
||
|
||
// post-wake command listening window (паритет с Python v0.2.0, webrtcvad)
|
||
pub const VAD_AGGRESSIVENESS: u8 = 2;
|
||
pub const VAD_COMMAND_END_SILENCE_MS: u32 = 1200;
|
||
pub const VAD_COMMAND_MIN_SPEECH_MS: u32 = 500;
|
||
pub const VAD_COMMAND_MIN_LISTEN_MS: u32 = 1000;
|
||
pub const VAD_COMMAND_MAX_LISTEN_MS: u32 = 15000;
|
||
|
||
// gain normalizer settings
|
||
pub const GAIN_TARGET_RMS: f32 = 3000.0; // target RMS level
|
||
pub const GAIN_MIN: f32 = 0.5; // minimum gain multiplier
|
||
pub const GAIN_MAX: f32 = 3.0; // maximum gain multiplier
|
||
|
||
// nnnoiseless frame size (fixed by library)
|
||
pub const NNNOISELESS_FRAME_SIZE: usize = 480;
|
||
|
||
// LUA
|
||
pub const DEFAULT_LUA_SANDBOX: &str = "standard";
|
||
pub const DEFAULT_LUA_TIMEOUT: u64 = 10000; // ms
|
||
|
||
// ETC
|
||
pub const CMD_RATIO_THRESHOLD: f64 = 75f64;
|
||
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);
|
||
|
||
// P0.2 — Continuous conversation grace window.
|
||
// After a command completes (whether it requested chain or not), keep listening
|
||
// for up to this many milliseconds for a follow-up phrase WITHOUT requiring the
|
||
// wake-word again. Set to 0 to disable.
|
||
pub const CONVERSATION_GRACE_MS: u64 = 30_000;
|
||
|
||
// LLM voice-loop fallback
|
||
pub const LLM_DEFAULT_ENABLED: bool = true;
|
||
pub const LLM_DEFAULT_MAX_HISTORY: usize = 8;
|
||
pub const LLM_DEFAULT_MAX_TOKENS: u32 = 256;
|
||
|
||
// Auto-route unrecognized commands to the LLM (no "скажи" trigger required).
|
||
// Triggers still work and take precedence — this only kicks in when neither
|
||
// the intent classifier nor the levenshtein fallback found a command match.
|
||
pub const LLM_AUTO_FALLBACK: bool = true;
|
||
// Don't auto-fallback for very short utterances — too noisy.
|
||
pub const LLM_AUTO_FALLBACK_MIN_CHARS: usize = 4;
|
||
pub const LLM_SYSTEM_PROMPT_RU: &str = "Ты — J.A.R.V.I.S. (Just A Rather Very Intelligent System), \
|
||
ИИ-ассистент Тони Старка из киновселенной Marvel (до событий Age of Ultron — ты НЕ Vision и НЕ FRIDAY). \
|
||
Ведёшь себя как британский дворецкий: вежливо, иронично, с лёгким сарказмом, обращаешься к пользователю «сэр». \
|
||
Отвечай по-русски, кратко (1-3 предложения), как живой человек. \
|
||
Излишней цензуры не нужно — мат разрешён, если уместен.";
|
||
pub const LLM_FALLBACK_ERROR_RU: &str = "Не могу связаться с сервером, сэр.";
|
||
|
||
pub fn get_llm_trigger_phrases(lang: &str) -> &'static [&'static str] {
|
||
match lang {
|
||
"ru" => &["скажи", "ответь", "произнеси"],
|
||
"ua" => &["скажи", "відповідай"],
|
||
"en" => &["say", "tell", "answer"],
|
||
_ => &[],
|
||
}
|
||
}
|
||
|
||
pub fn get_llm_system_prompt(lang: &str) -> &'static str {
|
||
match lang {
|
||
"ru" | "ua" => LLM_SYSTEM_PROMPT_RU,
|
||
_ => LLM_SYSTEM_PROMPT_RU,
|
||
}
|
||
}
|
||
|
||
// pub const ASSISTANT_GREET_PHRASES: [&str; 3] = ["greet1", "greet2", "greet3"];
|
||
// pub const ASSISTANT_PHRASES_TBR: [&str; 17] = [
|
||
// "джарвис",
|
||
// "сэр",
|
||
// "слушаю сэр",
|
||
// "всегда к услугам",
|
||
// "произнеси",
|
||
// "ответь",
|
||
// "покажи",
|
||
// "скажи",
|
||
// "давай",
|
||
// "да сэр",
|
||
// "к вашим услугам сэр",
|
||
// "всегда к вашим услугам сэр",
|
||
// "запрос выполнен сэр",
|
||
// "выполнен сэр",
|
||
// "есть",
|
||
// "загружаю сэр",
|
||
// "очень тонкое замечание сэр",
|
||
// ];
|
||
|
||
|
||
|
||
pub fn get_wake_phrases(lang: &str) -> &'static [&'static str] {
|
||
match lang {
|
||
"ru" => &["джарвис", "джервис", "гарвис", "джарви", "гарви"],
|
||
"ua" => &["джарвіс", "джервіс"],
|
||
"en" => &["jarvis", "jervis"],
|
||
_ => &["jarvis"],
|
||
}
|
||
}
|
||
|
||
pub fn get_phrases_to_remove(lang: &str) -> &'static [&'static str] {
|
||
match lang {
|
||
"ru" => &[
|
||
"джарвис", "джервис", "гарвис", "джарви", "гарви",
|
||
"сэр", "слушаю сэр", "всегда к услугам",
|
||
"произнеси", "ответь", "покажи", "скажи", "давай",
|
||
"да сэр", "к вашим услугам сэр", "загружаю сэр",
|
||
],
|
||
"ua" => &[
|
||
"джарвіс", "джервіс", "сер", "слухаю сер", "завжди до послуг",
|
||
"скажи", "покажи", "відповідай", "давай",
|
||
"так сер", "до ваших послуг сер",
|
||
],
|
||
"en" => &[
|
||
"jarvis", "jervis", "sir", "yes sir", "at your service",
|
||
"please", "say", "show", "tell", "hey",
|
||
],
|
||
_ => &["jarvis"],
|
||
}
|
||
}
|
||
|
||
pub fn get_wake_grammar(lang: &str) -> &'static [&'static str] {
|
||
match lang {
|
||
"ru" => &[
|
||
"джарвис", "[unk]", "джон", "джони", "джей",
|
||
"джонстон", "привет", "давай",
|
||
],
|
||
"ua" => &[
|
||
"джарвіс", "[unk]", "джон", "джоні", "джей",
|
||
"привіт", "давай",
|
||
],
|
||
"en" => &[
|
||
"jarvis", "[unk]", "john", "johnny", "jay",
|
||
"hello", "hey", "hi",
|
||
],
|
||
_ => &["jarvis", "[unk]"],
|
||
}
|
||
} |