diff --git a/Cargo.lock b/Cargo.lock index 588ec6d..264df03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3281,6 +3281,8 @@ dependencies = [ "parking_lot", "platform-dirs", "rand 0.8.5", + "serde", + "serde_json", "simple-log", "tokio", "tray-icon", diff --git a/crates/jarvis-app/Cargo.toml b/crates/jarvis-app/Cargo.toml index 3a1a89c..76bff45 100644 --- a/crates/jarvis-app/Cargo.toml +++ b/crates/jarvis-app/Cargo.toml @@ -18,6 +18,8 @@ image.workspace = true platform-dirs.workspace = true rand.workspace = true parking_lot.workspace = true +serde.workspace = true +serde_json.workspace = true tokio = { version = "1", features = ["rt-multi-thread"] } diff --git a/crates/jarvis-app/src/app.rs b/crates/jarvis-app/src/app.rs index 2dd2d86..afa4f5d 100644 --- a/crates/jarvis-app/src/app.rs +++ b/crates/jarvis-app/src/app.rs @@ -323,21 +323,45 @@ fn recognize_command( // execute command and check if we should chain let should_chain = execute_command(&recognized_voice, rt); - if should_chain { - // chain: reset and continue listening - info!("Chaining enabled, continuing to listen..."); + let grace_ms = jarvis_core::config::CONVERSATION_GRACE_MS; + if should_chain || grace_ms > 0 { + // Either explicit chain OR P0.2 continuous-conversation grace window. + // Reset listening state and keep going without re-wake. + if should_chain { + info!("Chain requested, continuing to listen..."); + } else { + info!( + "Continuous conversation: {}ms grace window for follow-up.", + grace_ms + ); + } stt::reset_speech_recognizer(); vad_state = VadState::WaitingForVoice; silence_frames = 0; - start = SystemTime::now(); + // For grace-mode, shorten the deadline to grace_ms (instead of CMS_WAIT_DELAY). + // We set `start` such that the existing timeout check (line ~360) fires at + // start + CMS_WAIT_DELAY = now + grace_ms. + if !should_chain { + let cms = jarvis_core::config::CMS_WAIT_DELAY.as_millis() as u64; + if grace_ms < cms { + // back-date `start` so the existing timeout fires at now + grace_ms + let backdate_ms = cms - grace_ms; + start = SystemTime::now() + .checked_sub(std::time::Duration::from_millis(backdate_ms)) + .unwrap_or_else(SystemTime::now); + } else { + start = SystemTime::now(); + } + } else { + start = SystemTime::now(); + } audio_buffer.clear(); webrtc_vad.reset(); window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS); ipc::send(IpcEvent::Listening); continue; } else { - // no chain: return to wake word - info!("No chain, returning to wake word mode."); + info!("Conversation grace disabled, returning to wake word mode."); return; } } @@ -460,6 +484,26 @@ fn execute_command(text: &str, rt: &tokio::runtime::Runtime) -> bool { } else { info!("No command found for: {}", text); + // IMBA-1: Agentic LLM router — try to map unknown phrase to a known command + // by asking the LLM. This is the killer feature competitors don't have. + if crate::llm_router::is_enabled() { + if let Some(routed) = crate::llm_router::try_route(text) { + info!( + "Router → {} ({}%): {} | substitute='{}'", + routed.command_id, + (routed.confidence * 100.0) as u32, + routed.reason, + routed.substitute_phrase + ); + + // Re-dispatch with the canonical phrase for the chosen command. + // Guard against infinite recursion: pass a marker if needed. + if routed.substitute_phrase != text { + return execute_command(&routed.substitute_phrase, rt); + } + } + } + if jarvis_core::config::LLM_AUTO_FALLBACK && crate::llm_fallback::is_enabled() && text.chars().count() >= jarvis_core::config::LLM_AUTO_FALLBACK_MIN_CHARS diff --git a/crates/jarvis-app/src/llm_fallback.rs b/crates/jarvis-app/src/llm_fallback.rs index 7afbe18..7e1f1ab 100644 --- a/crates/jarvis-app/src/llm_fallback.rs +++ b/crates/jarvis-app/src/llm_fallback.rs @@ -5,6 +5,9 @@ use jarvis_core::config; use jarvis_core::i18n; use jarvis_core::ipc::{self, IpcEvent}; use jarvis_core::llm::{ChatMessage, ConversationHistory, LlmClient}; +use jarvis_core::long_term_memory; +use jarvis_core::profiles; +use jarvis_core::tts::{self, SpeakOpts}; use jarvis_core::voices; struct State { @@ -90,7 +93,27 @@ pub fn handle(prompt: &str) { let snapshot: Vec = { let mut h = state.history.lock(); h.push_user(prompt); - h.snapshot() + let mut snap = h.snapshot(); + + // Inject (a) profile personality (b) relevant long-term memory as a fresh + // system message right after the base prompt. Both are optional. + let profile = profiles::active(); + let mut overlay = String::new(); + if !profile.llm_personality.is_empty() { + overlay.push_str(&format!("Активный профиль: {} {}\nХарактер для ответа: {}\n", + profile.icon, profile.name, profile.llm_personality)); + } + let mem_ctx = long_term_memory::build_context(prompt, 5); + if !mem_ctx.is_empty() { + overlay.push_str(&mem_ctx); + } + if !overlay.is_empty() { + // Insert after the base system prompt (index 0 typically), before user msgs. + let insert_at = if !snap.is_empty() && snap[0].role == "system" { 1 } else { 0 }; + snap.insert(insert_at, ChatMessage::system(overlay)); + } + + snap }; match state.client.complete(&snapshot, state.max_tokens) { @@ -100,7 +123,7 @@ pub fn handle(prompt: &str) { state.history.lock().push_assistant(reply.clone()); ipc::send(IpcEvent::LlmReply { text: reply.clone() }); voices::play_ok(); - speak_via_sapi(&reply); + speak_reply(&reply); } Err(e) => { error!("LLM request failed: {}", e); @@ -108,41 +131,14 @@ pub fn handle(prompt: &str) { let err_text = config::LLM_FALLBACK_ERROR_RU.to_string(); ipc::send(IpcEvent::LlmReply { text: err_text.clone() }); voices::play_error(); - speak_via_sapi(&err_text); + speak_reply(&err_text); } } } -#[cfg(target_os = "windows")] -fn speak_via_sapi(text: &str) { +fn speak_reply(text: &str) { if std::env::var("JARVIS_LLM_TTS").as_deref() == Ok("false") { return; } - let cleaned = jarvis_core::text_utils::sanitize_for_speech(text); - let escaped = cleaned.replace('\'', "''"); - let ps = format!( - "Add-Type -AssemblyName System.Speech; \ - $s = New-Object System.Speech.Synthesis.SpeechSynthesizer; \ - foreach ($v in $s.GetInstalledVoices()) {{ \ - if ($v.VoiceInfo.Culture.TwoLetterISOLanguageName -eq 'ru') {{ \ - try {{ $s.SelectVoice($v.VoiceInfo.Name); break }} catch {{ }} \ - }} \ - }} \ - $s.Speak('{}')", - escaped, - ); - match std::process::Command::new("powershell") - .args(["-NoProfile", "-Command", &ps]) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .spawn() - { - Ok(_) => {} - Err(e) => warn!("SAPI spawn failed: {}", e), - } -} - -#[cfg(not(target_os = "windows"))] -fn speak_via_sapi(_text: &str) { - // No-op on non-Windows; LLM reply still arrives via IPC for the GUI to render. + tts::speak(text, &SpeakOpts::lang("ru")); } diff --git a/crates/jarvis-app/src/llm_router.rs b/crates/jarvis-app/src/llm_router.rs new file mode 100644 index 0000000..fd43956 --- /dev/null +++ b/crates/jarvis-app/src/llm_router.rs @@ -0,0 +1,224 @@ +//! IMBA-1: Agentic LLM router. +//! +//! When the fuzzy/intent matcher fails to find a command for the user's phrase, +//! ask the LLM to pick the best matching command from the registry. If confidence +//! exceeds the threshold, we substitute the user's phrase with a canonical phrase +//! from the chosen command and re-dispatch through the normal pipeline. +//! +//! This is what Алиса / Siri / Cortana cannot do: they bounce unknown queries to +//! web search. J.A.R.V.I.S. instead reasons about its OWN capabilities. + +use once_cell::sync::OnceCell; +use serde::Deserialize; + +use jarvis_core::config; +use jarvis_core::i18n; +use jarvis_core::llm::{ChatMessage, LlmClient}; +use jarvis_core::{JCommandsList, COMMANDS_LIST}; + +const SYSTEM_PROMPT_RU: &str = "Ты — диспетчер голосовых команд J.A.R.V.I.S. \ +Тебе даётся фраза пользователя и список доступных команд (id + примеры фраз). \ +Твоя задача: выбрать ОДНУ команду, которая лучше всего подходит, ИЛИ ответить \"none\" если ничего не подходит. \ +Отвечай СТРОГО в формате JSON, без markdown-блоков: \ +{\"command_id\": \"\", \"confidence\": 0.0-1.0, \"reason\": \"<краткое пояснение>\"}. \ +Если фраза неоднозначна, выбирай команду с большим purpose-match. \ +Если фраза похожа на просьбу 'поболтать' или вопрос, требующий длинного ответа — отвечай command_id='none'."; + +const SYSTEM_PROMPT_EN: &str = "You are the voice command dispatcher for J.A.R.V.I.S. \ +You receive the user's phrase and a list of available commands (id + example phrases). \ +Your job: pick ONE command that best matches, OR reply \"none\" if nothing fits. \ +Respond STRICTLY in JSON, no markdown blocks: \ +{\"command_id\": \"\", \"confidence\": 0.0-1.0, \"reason\": \"\"}. \ +If the phrase looks like 'chat' or a question requiring a long answer, return command_id='none'."; + +const DEFAULT_THRESHOLD: f32 = 0.55; +const MAX_TOKENS: u32 = 200; +const TEMPERATURE: f32 = 0.1; +const TOP_P: f32 = 0.9; +const MAX_PHRASES_PER_CMD: usize = 4; + +struct State { + client: LlmClient, + threshold: f32, +} + +static STATE: OnceCell> = OnceCell::new(); + +pub fn init() { + let _ = STATE.set(build_state()); +} + +fn build_state() -> Option { + if !router_enabled() { + info!("LLM router disabled (set JARVIS_LLM_ROUTER=1 to enable)."); + return None; + } + let client = match LlmClient::from_env() { + Ok(c) => c, + Err(e) => { + warn!("LLM router disabled: {}. Set GROQ_TOKEN to enable.", e); + return None; + } + }; + let threshold = std::env::var("JARVIS_LLM_ROUTER_THRESHOLD") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_THRESHOLD); + + info!("LLM router enabled (threshold {:.2}).", threshold); + Some(State { client, threshold }) +} + +fn router_enabled() -> bool { + std::env::var("JARVIS_LLM_ROUTER") + .map(|v| matches!(v.trim().to_lowercase().as_str(), "1" | "true" | "yes" | "on")) + .unwrap_or(true) // default ON when GROQ_TOKEN is present + && config::LLM_DEFAULT_ENABLED +} + +pub fn is_enabled() -> bool { + STATE.get().and_then(|s| s.as_ref()).is_some() +} + +/// Result of routing an unknown phrase. +#[derive(Debug, Clone)] +pub struct RouteResult { + pub command_id: String, + pub substitute_phrase: String, + pub confidence: f32, + pub reason: String, +} + +#[derive(Deserialize)] +struct RouteJson { + #[serde(default)] + command_id: String, + #[serde(default)] + confidence: f32, + #[serde(default)] + reason: String, +} + +/// Try to route `text` to a known command via LLM. Returns `None` if disabled, +/// LLM call fails, no match, or confidence below threshold. +pub fn try_route(text: &str) -> Option { + let state = STATE.get().and_then(|s| s.as_ref())?; + let commands_list = COMMANDS_LIST.get()?; + let lang = i18n::get_language(); + + let prompt = build_routing_prompt(commands_list, text, &lang); + let system = if lang.starts_with("ru") { SYSTEM_PROMPT_RU } else { SYSTEM_PROMPT_EN }; + + let messages = vec![ + ChatMessage::system(system.to_string()), + ChatMessage::user(prompt), + ]; + + let raw = match state.client.complete_with(&messages, MAX_TOKENS, TEMPERATURE, TOP_P) { + Ok(r) => r, + Err(e) => { + warn!("LLM router request failed: {}", e); + return None; + } + }; + + debug!("Router raw reply: {}", raw); + + let parsed: RouteJson = match parse_json(&raw) { + Some(p) => p, + None => { + warn!("Router reply not valid JSON: {}", raw); + return None; + } + }; + + if parsed.command_id.is_empty() || parsed.command_id == "none" { + info!("Router: no match (reason: {})", parsed.reason); + return None; + } + + if parsed.confidence < state.threshold { + info!( + "Router: {} below threshold {:.2} (confidence {:.2}, reason: {})", + parsed.command_id, state.threshold, parsed.confidence, parsed.reason + ); + return None; + } + + let substitute = lookup_first_phrase(commands_list, &parsed.command_id, &lang)?; + + Some(RouteResult { + command_id: parsed.command_id, + substitute_phrase: substitute, + confidence: parsed.confidence, + reason: parsed.reason, + }) +} + +fn build_routing_prompt(commands_list: &[JCommandsList], user_text: &str, lang: &str) -> String { + let mut buf = String::with_capacity(4096); + + if lang.starts_with("ru") { + buf.push_str("Доступные команды:\n"); + } else { + buf.push_str("Available commands:\n"); + } + + for pack in commands_list { + for cmd in &pack.commands { + let phrases = cmd.get_phrases(lang); + if phrases.is_empty() { continue; } + let sample: Vec<&str> = phrases.iter() + .take(MAX_PHRASES_PER_CMD) + .map(|s| s.as_str()) + .collect(); + buf.push_str(&format!("- id={} ", cmd.id)); + if !cmd.description.is_empty() { + buf.push_str(&format!("({}) ", cmd.description)); + } + buf.push_str(&format!("examples=[{}]\n", sample.join(" | "))); + } + } + + if lang.starts_with("ru") { + buf.push_str(&format!("\nФраза пользователя: \"{}\"\n", user_text)); + buf.push_str("Верни JSON."); + } else { + buf.push_str(&format!("\nUser phrase: \"{}\"\n", user_text)); + buf.push_str("Return JSON."); + } + + buf +} + +fn lookup_first_phrase(commands_list: &[JCommandsList], cmd_id: &str, lang: &str) -> Option { + for pack in commands_list { + for cmd in &pack.commands { + if cmd.id == cmd_id { + let phrases = cmd.get_phrases(lang); + if let Some(first) = phrases.first() { + return Some(first.clone()); + } + } + } + } + None +} + +fn parse_json(raw: &str) -> Option { + // LLM sometimes wraps in ```json ... ``` despite instructions. + let cleaned = raw + .trim() + .trim_start_matches("```json") + .trim_start_matches("```") + .trim_end_matches("```") + .trim(); + + // Find the first '{' and last '}' to be tolerant of prose around the JSON. + let start = cleaned.find('{')?; + let end = cleaned.rfind('}')?; + if end <= start { return None; } + + let slice = &cleaned[start..=end]; + serde_json::from_str(slice).ok() +} diff --git a/crates/jarvis-app/src/main.rs b/crates/jarvis-app/src/main.rs index 3bac3e3..774e7b2 100644 --- a/crates/jarvis-app/src/main.rs +++ b/crates/jarvis-app/src/main.rs @@ -19,6 +19,7 @@ mod log; // include app mod app; mod llm_fallback; +mod llm_router; // include tray // @TODO. macOS currently not supported for tray functionality. @@ -59,6 +60,19 @@ fn main() -> Result<(), String> { eprintln!("[jarvis-app] step: llm_fallback::init"); llm_fallback::init(); + eprintln!("[jarvis-app] step: llm_router::init"); + llm_router::init(); + + eprintln!("[jarvis-app] step: long_term_memory::init"); + if let Err(e) = jarvis_core::long_term_memory::init() { + warn!("Long-term memory init failed: {}", e); + } + + eprintln!("[jarvis-app] step: profiles::init"); + if let Err(e) = jarvis_core::profiles::init() { + warn!("Profiles init failed: {}", e); + } + eprintln!("[jarvis-app] step: recorder::init"); if recorder::init().is_err() { notify_mic_problem(); diff --git a/crates/jarvis-core/src/config.rs b/crates/jarvis-core/src/config.rs index 2a0b228..f47a1fa 100644 --- a/crates/jarvis-core/src/config.rs +++ b/crates/jarvis-core/src/config.rs @@ -193,6 +193,12 @@ pub const DEFAULT_LUA_TIMEOUT: u64 = 10000; // ms pub const CMD_RATIO_THRESHOLD: f64 = 75f64; pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15); +// P0.2 — Continuous conversation grace window. +// After a command completes (whether it requested chain or not), keep listening +// for up to this many milliseconds for a follow-up phrase WITHOUT requiring the +// wake-word again. Set to 0 to disable. +pub const CONVERSATION_GRACE_MS: u64 = 30_000; + // LLM voice-loop fallback pub const LLM_DEFAULT_ENABLED: bool = true; pub const LLM_DEFAULT_MAX_HISTORY: usize = 8; diff --git a/crates/jarvis-core/src/lib.rs b/crates/jarvis-core/src/lib.rs index 2b3f44b..cc420d7 100644 --- a/crates/jarvis-core/src/lib.rs +++ b/crates/jarvis-core/src/lib.rs @@ -50,6 +50,12 @@ pub mod lua; pub mod text_utils; +pub mod tts; + +pub mod long_term_memory; + +pub mod profiles; + #[cfg(feature = "llm")] pub mod llm; diff --git a/crates/jarvis-core/src/long_term_memory.rs b/crates/jarvis-core/src/long_term_memory.rs new file mode 100644 index 0000000..5e9e2c8 --- /dev/null +++ b/crates/jarvis-core/src/long_term_memory.rs @@ -0,0 +1,217 @@ +//! IMBA-2: Long-term memory store. +//! +//! Persistent fact storage for J.A.R.V.I.S.. Lets the user say things like +//! "запомни что у меня собака Бася" / "я люблю чай улун" and have the +//! assistant recall them in later sessions. +//! +//! Storage: JSON file at `/long_term_memory.json`. Atomic +//! write-through (write-then-rename) to survive crashes. Simple substring +//! search for v1; vector search via fastembed is a future upgrade. +//! +//! Lua access: +//! jarvis.memory.remember(key, value) +//! jarvis.memory.recall(key) -- returns value or nil +//! jarvis.memory.search(query, n?) -- returns array of {key, value} matches +//! jarvis.memory.forget(key) +//! jarvis.memory.all() -- returns full table (debug) +//! +//! On-disk record fields: key, value, created_at, last_used_at, use_count. +//! Auto-decay: entries unused for 90 days are returned last in search results +//! (not deleted automatically; explicit `forget` only). + +use once_cell::sync::OnceCell; +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::path::PathBuf; + +use crate::APP_CONFIG_DIR; + +const FILE_NAME: &str = "long_term_memory.json"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRecord { + pub key: String, + pub value: String, + pub created_at: i64, + pub last_used_at: i64, + #[serde(default)] + pub use_count: u64, +} + +#[derive(Default, Debug, Serialize, Deserialize)] +struct Store { + #[serde(default)] + entries: BTreeMap, +} + +struct State { + path: PathBuf, + store: RwLock, +} + +static STATE: OnceCell = OnceCell::new(); + +pub fn init() -> Result<(), String> { + if STATE.get().is_some() { + return Ok(()); + } + + let dir = APP_CONFIG_DIR + .get() + .ok_or_else(|| "APP_CONFIG_DIR not initialised".to_string())?; + let path = dir.join(FILE_NAME); + + let store = load_or_empty(&path); + info!( + "Long-term memory loaded: {} entries from {}", + store.entries.len(), + path.display() + ); + + STATE.set(State { path, store: RwLock::new(store) }) + .map_err(|_| "long_term_memory already initialised".to_string())?; + Ok(()) +} + +fn load_or_empty(path: &PathBuf) -> Store { + if !path.is_file() { + return Store::default(); + } + match std::fs::read_to_string(path) { + Ok(s) => match serde_json::from_str::(&s) { + Ok(store) => store, + Err(e) => { + warn!("Corrupt memory file {}: {} — starting empty", path.display(), e); + Store::default() + } + }, + Err(e) => { + warn!("Cannot read {}: {} — starting empty", path.display(), e); + Store::default() + } + } +} + +fn persist(state: &State) { + let store = state.store.read(); + let tmp = state.path.with_extension("json.tmp"); + let json = match serde_json::to_string_pretty(&*store) { + Ok(s) => s, + Err(e) => { + warn!("Memory serialize failed: {}", e); + return; + } + }; + if let Err(e) = std::fs::write(&tmp, json) { + warn!("Memory write failed to {}: {}", tmp.display(), e); + return; + } + if let Err(e) = std::fs::rename(&tmp, &state.path) { + warn!("Memory rename failed: {}", e); + } +} + +fn now_secs() -> i64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +fn normalize_key(k: &str) -> String { + k.trim().to_lowercase() +} + +/// Store / overwrite a fact. +pub fn remember(key: &str, value: &str) -> Result<(), String> { + let state = STATE.get().ok_or_else(|| "memory not init".to_string())?; + let nk = normalize_key(key); + if nk.is_empty() { + return Err("empty key".into()); + } + let now = now_secs(); + { + let mut store = state.store.write(); + let entry = store.entries.entry(nk.clone()).or_insert(MemoryRecord { + key: nk.clone(), + value: value.to_string(), + created_at: now, + last_used_at: now, + use_count: 0, + }); + entry.value = value.to_string(); + entry.last_used_at = now; + } + persist(state); + info!("Memory: remembered '{}' = '{}'", nk, value); + Ok(()) +} + +/// Retrieve a fact by exact key (case-insensitive). +pub fn recall(key: &str) -> Option { + let state = STATE.get()?; + let nk = normalize_key(key); + let now = now_secs(); + let value = { + let mut store = state.store.write(); + let entry = store.entries.get_mut(&nk)?; + entry.last_used_at = now; + entry.use_count += 1; + entry.value.clone() + }; + persist(state); + Some(value) +} + +/// Substring search (case-insensitive). Returns up to `limit` records ranked by +/// recency. Matches against both key and value. +pub fn search(query: &str, limit: usize) -> Vec { + let Some(state) = STATE.get() else { return Vec::new(); }; + let nq = normalize_key(query); + if nq.is_empty() { + return Vec::new(); + } + let store = state.store.read(); + let mut hits: Vec = store.entries.values() + .filter(|r| r.key.contains(&nq) || r.value.to_lowercase().contains(&nq)) + .cloned() + .collect(); + hits.sort_by(|a, b| b.last_used_at.cmp(&a.last_used_at)); + hits.truncate(limit.max(1)); + hits +} + +/// Delete a fact. Returns true if it existed. +pub fn forget(key: &str) -> bool { + let Some(state) = STATE.get() else { return false; }; + let nk = normalize_key(key); + let removed = state.store.write().entries.remove(&nk).is_some(); + if removed { + persist(state); + info!("Memory: forgot '{}'", nk); + } + removed +} + +/// Snapshot of all records (for debug / GUI display). +pub fn all() -> Vec { + let Some(state) = STATE.get() else { return Vec::new(); }; + state.store.read().entries.values().cloned().collect() +} + +/// Build a system-prompt snippet containing relevant facts for the LLM, +/// based on a substring search of the user's prompt. Empty string if no matches. +pub fn build_context(prompt: &str, limit: usize) -> String { + let hits = search(prompt, limit); + if hits.is_empty() { + return String::new(); + } + let mut buf = String::with_capacity(256); + buf.push_str("Известные факты о пользователе (используй если уместно):\n"); + for h in &hits { + buf.push_str(&format!("- {} = {}\n", h.key, h.value)); + } + buf +} diff --git a/crates/jarvis-core/src/lua/api.rs b/crates/jarvis-core/src/lua/api.rs index 65bc9cd..ba85045 100644 --- a/crates/jarvis-core/src/lua/api.rs +++ b/crates/jarvis-core/src/lua/api.rs @@ -7,4 +7,7 @@ pub mod state; pub mod system; pub mod tts; pub mod llm; -pub mod text; \ No newline at end of file +pub mod text; +pub mod memory; +pub mod profile; +pub mod vision; \ No newline at end of file diff --git a/crates/jarvis-core/src/lua/api/memory.rs b/crates/jarvis-core/src/lua/api/memory.rs new file mode 100644 index 0000000..1b4480e --- /dev/null +++ b/crates/jarvis-core/src/lua/api/memory.rs @@ -0,0 +1,67 @@ +//! Lua bindings for the long-term memory store. +//! +//! Usage from Lua scripts: +//! +//! jarvis.memory.remember("любимый чай", "улун") +//! local v = jarvis.memory.recall("любимый чай") -- "улун" or nil +//! for _, hit in ipairs(jarvis.memory.search("чай", 5)) do +//! print(hit.key, hit.value) +//! end +//! jarvis.memory.forget("любимый чай") +//! local all = jarvis.memory.all() -- array of {key, value, ...} + +use mlua::{Lua, Table}; + +use crate::long_term_memory; + +pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> { + let mem = lua.create_table()?; + + let remember = lua.create_function(|_, (k, v): (String, String)| { + long_term_memory::remember(&k, &v) + .map_err(|e| mlua::Error::external(e)) + })?; + mem.set("remember", remember)?; + + let recall = lua.create_function(|_, k: String| { + Ok(long_term_memory::recall(&k)) + })?; + mem.set("recall", recall)?; + + let search = lua.create_function(|lua, (q, n): (String, Option)| { + let hits = long_term_memory::search(&q, n.unwrap_or(5)); + let arr = lua.create_table()?; + for (i, h) in hits.iter().enumerate() { + let row = lua.create_table()?; + row.set("key", h.key.clone())?; + row.set("value", h.value.clone())?; + row.set("created_at", h.created_at)?; + row.set("last_used_at", h.last_used_at)?; + row.set("use_count", h.use_count)?; + arr.set(i + 1, row)?; + } + Ok(arr) + })?; + mem.set("search", search)?; + + let forget = lua.create_function(|_, k: String| { + Ok(long_term_memory::forget(&k)) + })?; + mem.set("forget", forget)?; + + let all = lua.create_function(|lua, ()| { + let recs = long_term_memory::all(); + let arr = lua.create_table()?; + for (i, h) in recs.iter().enumerate() { + let row = lua.create_table()?; + row.set("key", h.key.clone())?; + row.set("value", h.value.clone())?; + arr.set(i + 1, row)?; + } + Ok(arr) + })?; + mem.set("all", all)?; + + jarvis.set("memory", mem)?; + Ok(()) +} diff --git a/crates/jarvis-core/src/lua/api/profile.rs b/crates/jarvis-core/src/lua/api/profile.rs new file mode 100644 index 0000000..6334773 --- /dev/null +++ b/crates/jarvis-core/src/lua/api/profile.rs @@ -0,0 +1,63 @@ +//! Lua bindings for profile switching. +//! +//! Usage from Lua scripts: +//! +//! local p = jarvis.profile.active() -- table {name, icon, description, ...} +//! jarvis.profile.set("work") -- switch to work profile +//! local names = jarvis.profile.list() -- {"default", "work", "game", "sleep", "driving"} +//! if jarvis.profile.allows("games.steam") then ... end + +use mlua::{Lua, Table}; + +use crate::profiles; + +pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> { + let prof = lua.create_table()?; + + let active = lua.create_function(|lua, ()| { + let p = profiles::active(); + let t = lua.create_table()?; + t.set("name", p.name)?; + t.set("description", p.description)?; + t.set("llm_personality", p.llm_personality)?; + t.set("greeting", p.greeting)?; + t.set("icon", p.icon)?; + Ok(t) + })?; + prof.set("active", active)?; + + let active_name = lua.create_function(|_, ()| Ok(profiles::active_name()))?; + prof.set("active_name", active_name)?; + + let set_active = lua.create_function(|lua, name: String| { + match profiles::set_active(&name) { + Ok(p) => { + let t = lua.create_table()?; + t.set("name", p.name)?; + t.set("icon", p.icon)?; + t.set("greeting", p.greeting)?; + Ok(t) + } + Err(e) => Err(mlua::Error::external(e)), + } + })?; + prof.set("set", set_active)?; + + let list = lua.create_function(|lua, ()| { + let names = profiles::list(); + let arr = lua.create_table()?; + for (i, n) in names.iter().enumerate() { + arr.set(i + 1, n.clone())?; + } + Ok(arr) + })?; + prof.set("list", list)?; + + let allows = lua.create_function(|_, cmd_id: String| { + Ok(profiles::active().allows_command(&cmd_id)) + })?; + prof.set("allows", allows)?; + + jarvis.set("profile", prof)?; + Ok(()) +} diff --git a/crates/jarvis-core/src/lua/api/tts.rs b/crates/jarvis-core/src/lua/api/tts.rs index 6c33257..78f730f 100644 --- a/crates/jarvis-core/src/lua/api/tts.rs +++ b/crates/jarvis-core/src/lua/api/tts.rs @@ -1,70 +1,35 @@ use mlua::{Lua, Table}; -use std::process::{Command, Stdio}; -use crate::text_utils::sanitize_for_speech; +use crate::tts::{self, SpeakOpts}; pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> { // jarvis.speak(text, opts?) // opts.lang: ISO 2-letter code (ru, en, de, ...). default "ru" // opts.async: if true, fire-and-forget; if false, block until done. default true // opts.raw: if true, skip text sanitization (default false) + // + // Uses the active TTS backend (see `crate::tts::backend`). Configured via JARVIS_TTS env var. let speak_fn = lua.create_function(|_, (text, opts): (String, Option)| { - let mut iso = "ru".to_string(); - let mut detached = true; - let mut raw = false; + let mut speak_opts = SpeakOpts::default(); if let Some(t) = opts { - if let Ok(v) = t.get::("lang") { iso = v; } - if let Ok(v) = t.get::("async") { detached = v; } - if let Ok(v) = t.get::("raw") { raw = v; } + if let Ok(v) = t.get::("lang") { speak_opts.lang = v; } + if let Ok(v) = t.get::("async") { speak_opts.detached = v; } + if let Ok(v) = t.get::("raw") { speak_opts.raw = v; } } - let prepared = if raw { text } else { sanitize_for_speech(&text) }; - speak_via_sapi(&prepared, &iso, detached); + tts::speak(&text, &speak_opts); Ok(()) })?; jarvis.set("speak", speak_fn)?; + // jarvis.tts.backend() — returns name of active backend ("sapi", "piper", "silero") + let tts_table = lua.create_table()?; + let backend_fn = lua.create_function(|_, ()| { + Ok(tts::backend().name().to_string()) + })?; + tts_table.set("backend", backend_fn)?; + jarvis.set("tts", tts_table)?; + Ok(()) } - -#[cfg(target_os = "windows")] -fn speak_via_sapi(text: &str, iso: &str, detached: bool) { - if text.is_empty() { - return; - } - let escaped = text.replace('\'', "''").replace('\r', " ").replace('\n', " "); - let safe_iso = if iso.chars().all(|c| c.is_ascii_alphabetic()) && iso.len() == 2 { - iso.to_lowercase() - } else { - "ru".to_string() - }; - let ps = format!( - "Add-Type -AssemblyName System.Speech; \ - $s = New-Object System.Speech.Synthesis.SpeechSynthesizer; \ - foreach ($v in $s.GetInstalledVoices()) {{ \ - if ($v.VoiceInfo.Culture.TwoLetterISOLanguageName -eq '{}') {{ \ - try {{ $s.SelectVoice($v.VoiceInfo.Name); break }} catch {{}} \ - }} \ - }} \ - $s.Speak('{}')", - safe_iso, escaped - ); - - let mut cmd = Command::new("powershell"); - cmd.args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &ps]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - - if detached { - let _ = cmd.spawn(); - } else { - let _ = cmd.status(); - } -} - -#[cfg(not(target_os = "windows"))] -fn speak_via_sapi(text: &str, _iso: &str, _detached: bool) { - log::info!("[Lua tts] would speak: {}", text); -} - diff --git a/crates/jarvis-core/src/lua/api/vision.rs b/crates/jarvis-core/src/lua/api/vision.rs new file mode 100644 index 0000000..ad89e61 --- /dev/null +++ b/crates/jarvis-core/src/lua/api/vision.rs @@ -0,0 +1,189 @@ +//! IMBA-4: Multimodal — screenshot + vision LLM. +//! +//! Lua bindings: +//! jarvis.vision.screenshot(path?) -> path on disk (default temp .png) +//! jarvis.vision.describe(prompt?) -> string description (vision LLM call) +//! +//! Windows-only screenshot via PowerShell System.Drawing. Posts to a vision-capable +//! Groq model (`llama-3.2-11b-vision-preview` by default; override via `GROQ_VISION_MODEL`). +//! +//! Requires HTTP sandbox level (registered behind `allows_http()` in engine.rs). + +use mlua::{Lua, Table}; + +pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> { + let vision = lua.create_table()?; + + let screenshot = lua.create_function(|_, path: Option| { + match take_screenshot(path.as_deref()) { + Ok(p) => Ok(Some(p)), + Err(e) => { + log::warn!("vision.screenshot failed: {}", e); + Ok(None) + } + } + })?; + vision.set("screenshot", screenshot)?; + + let describe = lua.create_function(|_, prompt: Option| { + let user_prompt = prompt.unwrap_or_else(|| { + "Опиши кратко что на этом экране. Если есть ошибки — отметь их.".to_string() + }); + match describe_screen(&user_prompt) { + Ok(text) => Ok(Some(text)), + Err(e) => { + log::warn!("vision.describe failed: {}", e); + Ok(None) + } + } + })?; + vision.set("describe", describe)?; + + jarvis.set("vision", vision)?; + Ok(()) +} + +#[cfg(target_os = "windows")] +fn take_screenshot(out_path: Option<&str>) -> Result { + let target = match out_path { + Some(p) => std::path::PathBuf::from(p), + None => { + let f = tempfile::Builder::new() + .prefix("jarvis-screen-") + .suffix(".png") + .tempfile() + .map_err(|e| format!("tempfile: {}", e))?; + let p = f.path().to_path_buf(); + drop(f); + p + } + }; + + let escaped = target.display().to_string().replace('\'', "''"); + let ps = format!( + r#" +Add-Type -AssemblyName System.Windows.Forms; +Add-Type -AssemblyName System.Drawing; +$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; +$bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height; +$g = [System.Drawing.Graphics]::FromImage($bmp); +$g.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size); +$bmp.Save('{}', [System.Drawing.Imaging.ImageFormat]::Png); +$g.Dispose(); $bmp.Dispose(); +"#, + escaped + ); + + let status = std::process::Command::new("powershell") + .args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &ps]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map_err(|e| format!("powershell spawn: {}", e))?; + + if !status.success() { + return Err(format!("screenshot powershell exited {}", status)); + } + if !target.is_file() { + return Err("screenshot file not created".to_string()); + } + Ok(target.display().to_string()) +} + +#[cfg(not(target_os = "windows"))] +fn take_screenshot(_out_path: Option<&str>) -> Result { + Err("screenshot not implemented on this platform".into()) +} + +#[cfg(target_os = "windows")] +fn describe_screen(prompt: &str) -> Result { + let path = take_screenshot(None)?; + let b64 = read_as_base64(&path)?; + let _ = std::fs::remove_file(&path); + call_vision_llm(prompt, &b64) +} + +#[cfg(not(target_os = "windows"))] +fn describe_screen(_prompt: &str) -> Result { + Err("vision not implemented on this platform".into()) +} + +#[cfg(target_os = "windows")] +fn read_as_base64(path: &str) -> Result { + // Avoid a new base64 crate dep — use PowerShell [Convert]::ToBase64String. + let escaped = path.replace('\'', "''"); + let ps = format!( + "[Convert]::ToBase64String([IO.File]::ReadAllBytes('{}'))", + escaped + ); + let out = std::process::Command::new("powershell") + .args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &ps]) + .output() + .map_err(|e| format!("base64 powershell: {}", e))?; + if !out.status.success() { + return Err(format!( + "base64 powershell exited {}: {}", + out.status, + String::from_utf8_lossy(&out.stderr) + )); + } + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + if s.is_empty() { + return Err("base64 output empty".into()); + } + Ok(s) +} + +#[cfg(target_os = "windows")] +fn call_vision_llm(prompt: &str, image_b64: &str) -> Result { + use serde_json::json; + + let token = std::env::var("GROQ_TOKEN") + .map_err(|_| "GROQ_TOKEN not set".to_string())?; + let base = std::env::var("GROQ_BASE_URL") + .unwrap_or_else(|_| "https://api.groq.com/openai/v1".to_string()); + let model = std::env::var("GROQ_VISION_MODEL") + .unwrap_or_else(|_| "llama-3.2-11b-vision-preview".to_string()); + + let body = json!({ + "model": model, + "max_tokens": 400, + "temperature": 0.2, + "messages": [{ + "role": "user", + "content": [ + { "type": "text", "text": prompt }, + { "type": "image_url", + "image_url": { + "url": format!("data:image/png;base64,{}", image_b64) + } + } + ] + }] + }); + + let url = format!("{}/chat/completions", base.trim_end_matches('/')); + let client = reqwest::blocking::Client::new(); + let resp = client + .post(&url) + .bearer_auth(&token) + .json(&body) + .send() + .map_err(|e| format!("HTTP: {}", e))?; + + let status = resp.status(); + let text = resp.text().map_err(|e| format!("read body: {}", e))?; + if !status.is_success() { + return Err(format!("vision API {}: {}", status.as_u16(), text)); + } + + let json: serde_json::Value = + serde_json::from_str(&text).map_err(|e| format!("parse JSON: {}", e))?; + json.get("choices") + .and_then(|c| c.get(0)) + .and_then(|c| c.get("message")) + .and_then(|m| m.get("content")) + .and_then(|c| c.as_str()) + .map(|s| s.trim().to_string()) + .ok_or_else(|| format!("no content in response: {}", text)) +} diff --git a/crates/jarvis-core/src/lua/engine.rs b/crates/jarvis-core/src/lua/engine.rs index f7380c3..c34fd15 100644 --- a/crates/jarvis-core/src/lua/engine.rs +++ b/crates/jarvis-core/src/lua/engine.rs @@ -78,11 +78,14 @@ impl LuaEngine { api::context::register(&self.lua, &jarvis, context)?; api::tts::register(&self.lua, &jarvis)?; api::text::register(&self.lua, &jarvis)?; + api::memory::register(&self.lua, &jarvis)?; + api::profile::register(&self.lua, &jarvis)?; // sandbox-controlled APIs if self.sandbox.allows_http() { api::http::register(&self.lua, &jarvis)?; api::llm::register(&self.lua, &jarvis)?; + api::vision::register(&self.lua, &jarvis)?; } if self.sandbox.allows_state() { diff --git a/crates/jarvis-core/src/profiles.rs b/crates/jarvis-core/src/profiles.rs new file mode 100644 index 0000000..d87280e --- /dev/null +++ b/crates/jarvis-core/src/profiles.rs @@ -0,0 +1,229 @@ +//! IMBA-3: Profile switching. +//! +//! Lets the user say "режим работа" / "режим игра" / "режим сон" / "режим машина" +//! to swap an active profile. Each profile lives at `/profiles/.json` +//! and contains overlays for behavior: +//! - llm_personality: optional system-prompt addendum +//! - allowed_command_prefixes: optional whitelist (substring match on cmd id) +//! - disabled_command_prefixes: optional blacklist +//! - greeting: voice line played when activated +//! - icon: optional ASCII / emoji indicator for GUI +//! +//! Built-in default profiles seeded on first run: `work`, `game`, `sleep`, `driving`. +//! +//! Switching is voice-driven via the dedicated `profile_switch` lua pack (added separately), +//! or programmatically via `set_active(name)`. The active profile name is persisted in +//! `/active_profile.txt` so it survives restart. + +use once_cell::sync::OnceCell; +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +use crate::APP_CONFIG_DIR; + +const ACTIVE_FILE: &str = "active_profile.txt"; +const PROFILES_DIR: &str = "profiles"; +const DEFAULT_PROFILE: &str = "default"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Profile { + pub name: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub llm_personality: String, + #[serde(default)] + pub allowed_command_prefixes: Vec, + #[serde(default)] + pub disabled_command_prefixes: Vec, + #[serde(default)] + pub greeting: String, + #[serde(default)] + pub icon: String, +} + +impl Profile { + pub fn allows_command(&self, cmd_id: &str) -> bool { + // disabled wins over allowed + if self.disabled_command_prefixes.iter().any(|p| cmd_id.starts_with(p.as_str())) { + return false; + } + if self.allowed_command_prefixes.is_empty() { + return true; + } + self.allowed_command_prefixes.iter().any(|p| cmd_id.starts_with(p.as_str())) + } +} + +struct State { + dir: PathBuf, + active_file: PathBuf, + active: RwLock, +} + +static STATE: OnceCell = OnceCell::new(); + +pub fn init() -> Result<(), String> { + if STATE.get().is_some() { return Ok(()); } + + let config_dir = APP_CONFIG_DIR + .get() + .ok_or_else(|| "APP_CONFIG_DIR not initialised".to_string())?; + let dir = config_dir.join(PROFILES_DIR); + let active_file = config_dir.join(ACTIVE_FILE); + + if !dir.is_dir() { + std::fs::create_dir_all(&dir) + .map_err(|e| format!("create profiles dir: {}", e))?; + } + + seed_defaults(&dir); + + let active_name = std::fs::read_to_string(&active_file) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| DEFAULT_PROFILE.to_string()); + + let active = load_profile(&dir, &active_name) + .unwrap_or_else(|| { + warn!("Active profile '{}' missing — falling back to default", active_name); + default_profile() + }); + + info!("Active profile: {} (icon: {})", active.name, if active.icon.is_empty() { "—" } else { &active.icon }); + + STATE.set(State { dir, active_file, active: RwLock::new(active) }) + .map_err(|_| "profiles already initialised".to_string())?; + Ok(()) +} + +fn seed_defaults(dir: &PathBuf) { + let defaults = vec![ + Profile { + name: DEFAULT_PROFILE.to_string(), + description: "Обычный режим, все команды доступны.".into(), + llm_personality: String::new(), + allowed_command_prefixes: vec![], + disabled_command_prefixes: vec![], + greeting: String::new(), + icon: "★".into(), + }, + Profile { + name: "work".to_string(), + description: "Рабочий режим — фокус, отключены развлечения.".into(), + llm_personality: "Отвечай кратко и по делу. Минимум шуток. Помогай быть продуктивным.".into(), + allowed_command_prefixes: vec![], + disabled_command_prefixes: vec!["games.".into(), "fun.".into(), "music.".into()], + greeting: "Режим работы. Сосредоточимся.".into(), + icon: "💼".into(), + }, + Profile { + name: "game".to_string(), + description: "Игровой режим — голосовые макросы и игры.".into(), + llm_personality: "Будь дружелюбен. Краткие ответы. Поддерживай геймерскую атмосферу.".into(), + allowed_command_prefixes: vec![], + disabled_command_prefixes: vec!["reminders.".into(), "calendar.".into()], + greeting: "Игровой режим активирован.".into(), + icon: "🎮".into(), + }, + Profile { + name: "sleep".to_string(), + description: "Спокойный режим — тихий голос, никаких уведомлений.".into(), + llm_personality: "Говори спокойно, короткими фразами. Ничего не предлагай.".into(), + allowed_command_prefixes: vec!["time.".into(), "weather.".into(), "lights.".into()], + disabled_command_prefixes: vec![], + greeting: "Спокойной ночи.".into(), + icon: "🌙".into(), + }, + Profile { + name: "driving".to_string(), + description: "Режим за рулём — только безопасные команды.".into(), + llm_personality: "Краткие безопасные ответы. Никаких длинных текстов.".into(), + allowed_command_prefixes: vec![ + "music.".into(), "navigation.".into(), "calls.".into(), + "time.".into(), "weather.".into(), "reminders.".into(), + ], + disabled_command_prefixes: vec!["windows.".into(), "screen.".into()], + greeting: "Режим за рулём. Веди безопасно.".into(), + icon: "🚗".into(), + }, + ]; + + for p in defaults { + let path = dir.join(format!("{}.json", p.name)); + if path.is_file() { continue; } + if let Ok(json) = serde_json::to_string_pretty(&p) { + if let Err(e) = std::fs::write(&path, json) { + warn!("seed profile {}: {}", path.display(), e); + } + } + } +} + +fn default_profile() -> Profile { + Profile { + name: DEFAULT_PROFILE.to_string(), + description: "fallback".into(), + llm_personality: String::new(), + allowed_command_prefixes: vec![], + disabled_command_prefixes: vec![], + greeting: String::new(), + icon: "★".into(), + } +} + +fn load_profile(dir: &PathBuf, name: &str) -> Option { + let path = dir.join(format!("{}.json", name)); + let raw = std::fs::read_to_string(&path).ok()?; + serde_json::from_str(&raw).ok() +} + +/// Set the active profile. Persists to disk. Returns the new profile or error. +pub fn set_active(name: &str) -> Result { + let state = STATE.get().ok_or_else(|| "profiles not init".to_string())?; + let profile = load_profile(&state.dir, name) + .ok_or_else(|| format!("profile '{}' not found", name))?; + { + let mut active = state.active.write(); + *active = profile.clone(); + } + if let Err(e) = std::fs::write(&state.active_file, &profile.name) { + warn!("persist active profile: {}", e); + } + info!("Profile switched: {} {}", profile.icon, profile.name); + Ok(profile) +} + +/// Get a clone of the active profile. +pub fn active() -> Profile { + STATE.get() + .map(|s| s.active.read().clone()) + .unwrap_or_else(default_profile) +} + +/// Quick accessor for the active profile name. +pub fn active_name() -> String { + STATE.get() + .map(|s| s.active.read().name.clone()) + .unwrap_or_else(|| DEFAULT_PROFILE.to_string()) +} + +/// List available profile names (alphabetical). +pub fn list() -> Vec { + let Some(state) = STATE.get() else { return Vec::new(); }; + let mut names = Vec::new(); + if let Ok(rd) = std::fs::read_dir(&state.dir) { + for entry in rd.flatten() { + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) == Some("json") { + if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) { + names.push(stem.to_string()); + } + } + } + } + names.sort(); + names +} diff --git a/crates/jarvis-core/src/tts/mod.rs b/crates/jarvis-core/src/tts/mod.rs new file mode 100644 index 0000000..84ee73c --- /dev/null +++ b/crates/jarvis-core/src/tts/mod.rs @@ -0,0 +1,126 @@ +//! TTS backend abstraction. +//! +//! All voice synthesis in J.A.R.V.I.S. goes through `tts::backend().speak()`. +//! The backend is chosen at first call based on the `JARVIS_TTS` env var: +//! +//! - `sapi` — Windows SAPI via PowerShell (default, always available on Windows). +//! - `piper` — rhasspy/piper neural TTS (recommended). Needs `piper.exe` + voice .onnx +//! under `/tools/piper/` (or paths via `JARVIS_TTS_PIPER_BIN` +//! and `JARVIS_TTS_PIPER_VOICE`). Falls back to SAPI if binaries are missing. +//! - `silero` — Silero TTS via python subprocess (requires Python + torch installed). +//! Falls back to SAPI if helper script is missing. +//! +//! If `JARVIS_TTS` is unset, the dispatcher auto-detects Piper, otherwise SAPI. + +use once_cell::sync::OnceCell; +use std::sync::Arc; + +use crate::text_utils::sanitize_for_speech; + +mod sapi; +mod piper; +mod silero; + +pub use sapi::SapiBackend; +pub use piper::PiperBackend; +pub use silero::SileroBackend; + +/// Options for a single `speak()` call. +#[derive(Debug, Clone)] +pub struct SpeakOpts { + /// ISO-639-1 language code: "ru", "en", "de", ... Default: "ru". + pub lang: String, + /// Fire-and-forget if true; block until done if false. Default: true. + pub detached: bool, + /// Skip `text_utils::sanitize_for_speech` if true. Default: false. + pub raw: bool, +} + +impl Default for SpeakOpts { + fn default() -> Self { + Self { + lang: "ru".to_string(), + detached: true, + raw: false, + } + } +} + +impl SpeakOpts { + pub fn lang(lang: impl Into) -> Self { + Self { lang: lang.into(), ..Default::default() } + } +} + +/// All TTS backends implement this. `speak()` must not panic; on failure, log and return. +pub trait TtsBackend: Send + Sync { + fn name(&self) -> &'static str; + fn speak(&self, text: &str, opts: &SpeakOpts); +} + +static BACKEND: OnceCell> = OnceCell::new(); + +/// Get the active TTS backend. Initializes once on first call. +pub fn backend() -> Arc { + BACKEND.get_or_init(init_backend).clone() +} + +/// Speak a text via the active backend. Applies sanitisation unless `opts.raw` is set. +pub fn speak(text: &str, opts: &SpeakOpts) { + if text.trim().is_empty() { + return; + } + let prepared = if opts.raw { + text.to_string() + } else { + sanitize_for_speech(text) + }; + backend().speak(&prepared, opts); +} + +/// Speak with default opts (Russian, detached, sanitised). +pub fn speak_default(text: &str) { + speak(text, &SpeakOpts::default()); +} + +fn init_backend() -> Arc { + let choice = std::env::var("JARVIS_TTS").ok() + .map(|s| s.trim().to_lowercase()) + .unwrap_or_default(); + + match choice.as_str() { + "piper" => match PiperBackend::try_new() { + Ok(b) => { + info!("TTS backend: Piper ({} / {})", b.binary_display(), b.voice_display()); + return Arc::new(b); + } + Err(e) => { + warn!("JARVIS_TTS=piper requested but unavailable ({}). Falling back to SAPI.", e); + } + }, + "silero" => match SileroBackend::try_new() { + Ok(b) => { + info!("TTS backend: Silero ({})", b.helper_display()); + return Arc::new(b); + } + Err(e) => { + warn!("JARVIS_TTS=silero requested but unavailable ({}). Falling back to SAPI.", e); + } + }, + "sapi" | "" => { + // Auto-detect: prefer Piper if installed, else SAPI. + if choice.is_empty() { + if let Ok(b) = PiperBackend::try_new() { + info!("TTS backend: Piper (auto-detected, {} / {})", b.binary_display(), b.voice_display()); + return Arc::new(b); + } + } + } + other => { + warn!("Unknown JARVIS_TTS value '{}' — using SAPI.", other); + } + } + + info!("TTS backend: SAPI (Windows built-in)."); + Arc::new(SapiBackend::new()) +} diff --git a/crates/jarvis-core/src/tts/piper.rs b/crates/jarvis-core/src/tts/piper.rs new file mode 100644 index 0000000..b90e7a7 --- /dev/null +++ b/crates/jarvis-core/src/tts/piper.rs @@ -0,0 +1,185 @@ +//! Piper neural TTS backend (rhasspy/piper). +//! +//! Discovery order for binary: +//! 1. env `JARVIS_TTS_PIPER_BIN` +//! 2. `/tools/piper/piper.exe` +//! 3. `/tools/piper/piper.exe` +//! +//! Discovery order for voice: +//! 1. env `JARVIS_TTS_PIPER_VOICE` (absolute path to .onnx) +//! 2. `/voices/ru_RU-irina-medium.onnx` +//! 3. first `*.onnx` found in `/voices/` + +use std::path::{Path, PathBuf}; + +use super::{SpeakOpts, TtsBackend}; +use crate::APP_DIR; + +pub struct PiperBackend { + binary: PathBuf, + voice: PathBuf, +} + +impl PiperBackend { + pub fn try_new() -> Result { + let binary = find_binary().ok_or_else(|| "piper.exe not found".to_string())?; + let voice_dir = binary + .parent() + .map(|p| p.join("voices")) + .unwrap_or_else(|| PathBuf::from("voices")); + let voice = find_voice(&voice_dir).ok_or_else(|| { + format!("no .onnx voice in {}", voice_dir.display()) + })?; + Ok(Self { binary, voice }) + } + + pub fn binary_display(&self) -> String { + self.binary.display().to_string() + } + + pub fn voice_display(&self) -> String { + self.voice + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| self.voice.display().to_string()) + } +} + +impl TtsBackend for PiperBackend { + fn name(&self) -> &'static str { "piper" } + + fn speak(&self, text: &str, opts: &SpeakOpts) { + if text.trim().is_empty() { + return; + } + let binary = self.binary.clone(); + let voice = self.voice.clone(); + let text = text.to_string(); + let detached = opts.detached; + + let run = move || synth_and_play(&binary, &voice, &text); + + if detached { + std::thread::spawn(run); + } else { + run(); + } + } +} + +fn synth_and_play(binary: &Path, voice: &Path, text: &str) { + let tmp = match tempfile::Builder::new() + .prefix("jarvis-tts-") + .suffix(".wav") + .tempfile() + { + Ok(f) => f, + Err(e) => { + log::warn!("Piper: cannot create temp wav: {}", e); + return; + } + }; + let wav_path = tmp.path().to_path_buf(); + drop(tmp); + + let mut child = match std::process::Command::new(binary) + .arg("--model").arg(voice) + .arg("--output_file").arg(&wav_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + { + Ok(c) => c, + Err(e) => { + log::warn!("Piper spawn failed: {}", e); + return; + } + }; + + if let Some(mut stdin) = child.stdin.take() { + use std::io::Write; + if let Err(e) = stdin.write_all(text.as_bytes()) { + log::warn!("Piper stdin write failed: {}", e); + } + } + + match child.wait() { + Ok(status) if status.success() => { + play_wav(&wav_path); + } + Ok(status) => log::warn!("Piper exited with status {}", status), + Err(e) => log::warn!("Piper wait failed: {}", e), + } + + let _ = std::fs::remove_file(&wav_path); +} + +#[cfg(target_os = "windows")] +fn play_wav(path: &Path) { + let escaped = path.display().to_string().replace('\'', "''"); + let ps = format!( + "$p = New-Object System.Media.SoundPlayer '{}'; $p.PlaySync()", + escaped + ); + let _ = std::process::Command::new("powershell") + .args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &ps]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); +} + +#[cfg(not(target_os = "windows"))] +fn play_wav(path: &Path) { + log::info!("[Piper non-Windows stub] would play: {}", path.display()); +} + +fn find_binary() -> Option { + if let Ok(p) = std::env::var("JARVIS_TTS_PIPER_BIN") { + let candidate = PathBuf::from(p); + if candidate.is_file() { + return Some(candidate); + } + } + + let bin_name = if cfg!(target_os = "windows") { "piper.exe" } else { "piper" }; + + if let Ok(exe) = std::env::current_exe() { + if let Some(dir) = exe.parent() { + let c = dir.join("tools").join("piper").join(bin_name); + if c.is_file() { + return Some(c); + } + } + } + + let c = APP_DIR.join("tools").join("piper").join(bin_name); + if c.is_file() { + return Some(c); + } + + None +} + +fn find_voice(voice_dir: &Path) -> Option { + if let Ok(p) = std::env::var("JARVIS_TTS_PIPER_VOICE") { + let candidate = PathBuf::from(p); + if candidate.is_file() { + return Some(candidate); + } + } + + let preferred = voice_dir.join("ru_RU-irina-medium.onnx"); + if preferred.is_file() { + return Some(preferred); + } + + let entries = std::fs::read_dir(voice_dir).ok()?; + for entry in entries.flatten() { + let p = entry.path(); + if p.extension().and_then(|s| s.to_str()) == Some("onnx") { + return Some(p); + } + } + None +} diff --git a/crates/jarvis-core/src/tts/sapi.rs b/crates/jarvis-core/src/tts/sapi.rs new file mode 100644 index 0000000..51b1159 --- /dev/null +++ b/crates/jarvis-core/src/tts/sapi.rs @@ -0,0 +1,60 @@ +//! Windows SAPI TTS via PowerShell System.Speech. Always available on Win10+. + +use super::{SpeakOpts, TtsBackend}; + +pub struct SapiBackend; + +impl SapiBackend { + pub fn new() -> Self { Self } +} + +impl Default for SapiBackend { + fn default() -> Self { Self::new() } +} + +impl TtsBackend for SapiBackend { + fn name(&self) -> &'static str { "sapi" } + + #[cfg(target_os = "windows")] + fn speak(&self, text: &str, opts: &SpeakOpts) { + let escaped = text.replace('\'', "''").replace('\r', " ").replace('\n', " "); + let safe_iso = sanitize_iso(&opts.lang); + + let ps = format!( + "Add-Type -AssemblyName System.Speech; \ + $s = New-Object System.Speech.Synthesis.SpeechSynthesizer; \ + foreach ($v in $s.GetInstalledVoices()) {{ \ + if ($v.VoiceInfo.Culture.TwoLetterISOLanguageName -eq '{}') {{ \ + try {{ $s.SelectVoice($v.VoiceInfo.Name); break }} catch {{}} \ + }} \ + }} \ + $s.Speak('{}')", + safe_iso, escaped + ); + + let mut cmd = std::process::Command::new("powershell"); + cmd.args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &ps]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + + if opts.detached { + let _ = cmd.spawn(); + } else { + let _ = cmd.status(); + } + } + + #[cfg(not(target_os = "windows"))] + fn speak(&self, text: &str, _opts: &SpeakOpts) { + log::info!("[SAPI stub] would speak: {}", text); + } +} + +#[cfg(target_os = "windows")] +fn sanitize_iso(iso: &str) -> String { + if iso.chars().all(|c| c.is_ascii_alphabetic()) && iso.len() == 2 { + iso.to_lowercase() + } else { + "ru".to_string() + } +} diff --git a/crates/jarvis-core/src/tts/silero.rs b/crates/jarvis-core/src/tts/silero.rs new file mode 100644 index 0000000..5555918 --- /dev/null +++ b/crates/jarvis-core/src/tts/silero.rs @@ -0,0 +1,156 @@ +//! Silero TTS backend (PyTorch / silero-models). +//! +//! Spawns `python ` with text on stdin; the helper synthesises to a +//! temp wav and prints the path on stdout. We then play that wav. +//! +//! Discovery: +//! 1. env `JARVIS_TTS_SILERO_HELPER` → path to .py script +//! 2. `/tools/silero/silero_tts.py` +//! 3. `/tools/silero/silero_tts.py` +//! +//! Python binary: +//! 1. env `JARVIS_TTS_PYTHON` (default "python") +//! +//! Voice selection: +//! - env `JARVIS_TTS_SILERO_VOICE` (default "xenia" — ru-RU female) + +use std::path::{Path, PathBuf}; + +use super::{SpeakOpts, TtsBackend}; +use crate::APP_DIR; + +pub struct SileroBackend { + helper: PathBuf, + python: String, + voice: String, +} + +impl SileroBackend { + pub fn try_new() -> Result { + let helper = find_helper().ok_or_else(|| "silero_tts.py helper not found".to_string())?; + let python = std::env::var("JARVIS_TTS_PYTHON").unwrap_or_else(|_| "python".to_string()); + let voice = std::env::var("JARVIS_TTS_SILERO_VOICE").unwrap_or_else(|_| "xenia".to_string()); + Ok(Self { helper, python, voice }) + } + + pub fn helper_display(&self) -> String { + self.helper.display().to_string() + } +} + +impl TtsBackend for SileroBackend { + fn name(&self) -> &'static str { "silero" } + + fn speak(&self, text: &str, opts: &SpeakOpts) { + let helper = self.helper.clone(); + let python = self.python.clone(); + let voice = self.voice.clone(); + let text = text.to_string(); + let detached = opts.detached; + + let run = move || synth_and_play(&python, &helper, &voice, &text); + + if detached { + std::thread::spawn(run); + } else { + run(); + } + } +} + +fn synth_and_play(python: &str, helper: &Path, voice: &str, text: &str) { + use std::io::Write; + + let mut child = match std::process::Command::new(python) + .arg(helper) + .arg("--voice").arg(voice) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + { + Ok(c) => c, + Err(e) => { + log::warn!("Silero spawn failed: {}", e); + return; + } + }; + + if let Some(mut stdin) = child.stdin.take() { + if let Err(e) = stdin.write_all(text.as_bytes()) { + log::warn!("Silero stdin write failed: {}", e); + } + } + + let output = match child.wait_with_output() { + Ok(o) => o, + Err(e) => { + log::warn!("Silero wait failed: {}", e); + return; + } + }; + + if !output.status.success() { + log::warn!( + "Silero helper exited {} — stderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + return; + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let wav_path = stdout.trim(); + if wav_path.is_empty() { + log::warn!("Silero helper produced no output path"); + return; + } + + let p = PathBuf::from(wav_path); + play_wav(&p); + let _ = std::fs::remove_file(&p); +} + +#[cfg(target_os = "windows")] +fn play_wav(path: &Path) { + let escaped = path.display().to_string().replace('\'', "''"); + let ps = format!( + "$p = New-Object System.Media.SoundPlayer '{}'; $p.PlaySync()", + escaped + ); + let _ = std::process::Command::new("powershell") + .args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", &ps]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); +} + +#[cfg(not(target_os = "windows"))] +fn play_wav(path: &Path) { + log::info!("[Silero non-Windows stub] would play: {}", path.display()); +} + +fn find_helper() -> Option { + if let Ok(p) = std::env::var("JARVIS_TTS_SILERO_HELPER") { + let candidate = PathBuf::from(p); + if candidate.is_file() { + return Some(candidate); + } + } + + if let Ok(exe) = std::env::current_exe() { + if let Some(dir) = exe.parent() { + let c = dir.join("tools").join("silero").join("silero_tts.py"); + if c.is_file() { + return Some(c); + } + } + } + + let c = APP_DIR.join("tools").join("silero").join("silero_tts.py"); + if c.is_file() { + return Some(c); + } + + None +} diff --git a/resources/commands/memory_pack/command.toml b/resources/commands/memory_pack/command.toml new file mode 100644 index 0000000..544e307 --- /dev/null +++ b/resources/commands/memory_pack/command.toml @@ -0,0 +1,59 @@ +# IMBA-2: Long-term memory — remember/recall/forget arbitrary facts. + +[[commands]] +id = "memory.remember" +type = "lua" +script = "remember.lua" +sandbox = "standard" +timeout = 3000 + +[commands.phrases] +ru = [ + "запомни обо мне", "запомни что", + "помни что", "запомни это", + "запомни про меня", +] +en = ["remember that", "remember about me", "memorize"] +ua = ["запам'ятай що", "запам'ятай про мене"] + + +[[commands]] +id = "memory.recall" +type = "lua" +script = "recall.lua" +sandbox = "standard" +timeout = 3000 + +[commands.phrases] +ru = [ + "что ты помнишь о", "что ты знаешь обо мне", + "что помнишь про", "вспомни", +] +en = ["what do you remember about", "recall"] +ua = ["що ти пам'ятаєш про"] + + +[[commands]] +id = "memory.forget" +type = "lua" +script = "forget.lua" +sandbox = "standard" +timeout = 3000 + +[commands.phrases] +ru = ["забудь про", "забудь что", "забудь обо мне"] +en = ["forget about", "forget that"] +ua = ["забудь про"] + + +[[commands]] +id = "memory.list" +type = "lua" +script = "list.lua" +sandbox = "minimal" +timeout = 3000 + +[commands.phrases] +ru = ["что ты помнишь", "что ты знаешь обо мне", "покажи память"] +en = ["what do you remember", "list memory", "show memory"] +ua = ["що ти пам'ятаєш"] diff --git a/resources/commands/memory_pack/forget.lua b/resources/commands/memory_pack/forget.lua new file mode 100644 index 0000000..40fa083 --- /dev/null +++ b/resources/commands/memory_pack/forget.lua @@ -0,0 +1,36 @@ +local phrase = (jarvis.context.phrase or ""):lower() +local key = jarvis.text.strip_trigger(phrase, { + "забудь про", + "забудь что", + "забудь обо мне", + "forget about", + "forget that", + "забудь про", +}) + +key = key:gsub("^[%s,:%.]+", ""):gsub("%s+$", "") + +if key == "" then + jarvis.speak("Что именно забыть?") + jarvis.audio.play_error() + return { chain = false } +end + +local removed = jarvis.memory.forget(key) +if removed then + jarvis.speak("Забыл про " .. key .. ".") + jarvis.audio.play_ok() +else + -- try substring search and remove first hit + local hits = jarvis.memory.search(key, 1) + if #hits > 0 then + jarvis.memory.forget(hits[1].key) + jarvis.speak("Забыл про " .. hits[1].key .. ".") + jarvis.audio.play_ok() + else + jarvis.speak("Я и не помнил про " .. key .. ".") + jarvis.audio.play_not_found() + end +end + +return { chain = false } diff --git a/resources/commands/memory_pack/list.lua b/resources/commands/memory_pack/list.lua new file mode 100644 index 0000000..8633c4a --- /dev/null +++ b/resources/commands/memory_pack/list.lua @@ -0,0 +1,19 @@ +local recs = jarvis.memory.all() +if #recs == 0 then + jarvis.speak("Я пока ничего о вас не запомнил.") + jarvis.audio.play_ok() + return { chain = false } +end + +local count = #recs +local sample = math.min(count, 5) +local line = string.format("Помню %d фактов. Первые: ", count) +for i = 1, sample do + line = line .. recs[i].key + if i < sample then line = line .. ", " end +end +line = line .. "." + +jarvis.speak(line) +jarvis.audio.play_ok() +return { chain = false } diff --git a/resources/commands/memory_pack/recall.lua b/resources/commands/memory_pack/recall.lua new file mode 100644 index 0000000..bd4e1a2 --- /dev/null +++ b/resources/commands/memory_pack/recall.lua @@ -0,0 +1,52 @@ +local phrase = (jarvis.context.phrase or ""):lower() +local query = jarvis.text.strip_trigger(phrase, { + "что ты помнишь о", + "что ты помнишь про", + "что ты знаешь обо мне", + "что помнишь про", + "вспомни", + "what do you remember about", + "recall", + "що ти пам'ятаєш про", +}) + +query = query:gsub("^[%s,:%.]+", ""):gsub("%s+$", "") + +if query == "" then + -- show top-3 most recent + local recs = jarvis.memory.all() + if #recs == 0 then + jarvis.speak("Ничего пока не помню.") + jarvis.audio.play_ok() + return { chain = false } + end + local line = "Я помню: " + for i = 1, math.min(3, #recs) do + line = line .. recs[i].key .. " — " .. recs[i].value .. ". " + end + jarvis.speak(line) + jarvis.audio.play_ok() + return { chain = false } +end + +-- substring search +local hits = jarvis.memory.search(query, 3) +if #hits == 0 then + jarvis.speak("Ничего не нашёл про " .. query .. ".") + jarvis.audio.play_not_found() + return { chain = false } +end + +local line = "" +if #hits == 1 then + line = hits[1].value +else + for i, h in ipairs(hits) do + line = line .. h.key .. ": " .. h.value + if i < #hits then line = line .. ". " end + end +end + +jarvis.speak(line) +jarvis.audio.play_ok() +return { chain = false } diff --git a/resources/commands/memory_pack/remember.lua b/resources/commands/memory_pack/remember.lua new file mode 100644 index 0000000..cca722f --- /dev/null +++ b/resources/commands/memory_pack/remember.lua @@ -0,0 +1,55 @@ +local phrase = (jarvis.context.phrase or ""):lower() +local body = jarvis.text.strip_trigger(phrase, { + "запомни обо мне что", + "запомни обо мне", + "запомни что", + "помни что", + "запомни это", + "запомни про меня", + "запомни", + "remember that", + "remember about me", + "memorize", + "запам'ятай що", + "запам'ятай про мене", +}) + +body = body:gsub("^[%s,:%.]+", ""):gsub("%s+$", "") + +if body == "" then + jarvis.speak("Что именно запомнить?") + jarvis.audio.play_error() + return { chain = false } +end + +-- Heuristic: derive a key. Look for "что X = Y" / "что у меня X" / etc. +-- For now: first 3-5 meaningful words become the key. +local key = body +local _, after_eq = body:find("=", 1, true) +if after_eq then + key = body:sub(1, after_eq - 1):gsub("%s+$", "") + body = body:sub(after_eq + 1):gsub("^%s+", "") +end + +-- truncate key to first 6 words +local words = {} +for w in key:gmatch("%S+") do + table.insert(words, w) + if #words >= 6 then break end +end +key = table.concat(words, " ") + +local ok, err = pcall(function() + jarvis.memory.remember(key, body) +end) + +if not ok then + jarvis.log("warn", "memory.remember failed: " .. tostring(err)) + jarvis.speak("Не удалось запомнить.") + jarvis.audio.play_error() + return { chain = false } +end + +jarvis.speak("Запомнил.") +jarvis.audio.play_ok() +return { chain = false } diff --git a/resources/commands/profile_switch/command.toml b/resources/commands/profile_switch/command.toml new file mode 100644 index 0000000..cfc7ad3 --- /dev/null +++ b/resources/commands/profile_switch/command.toml @@ -0,0 +1,78 @@ +# IMBA-3: Profile switching — work / game / sleep / driving / default. + +[[commands]] +id = "profile.work" +type = "lua" +script = "switch.lua" +sandbox = "standard" +timeout = 3000 + +[commands.phrases] +ru = ["режим работа", "включи режим работы", "рабочий режим", "переключись в работу"] +en = ["work mode", "switch to work mode", "enable work mode"] +ua = ["робочий режим", "режим роботи"] + + +[[commands]] +id = "profile.game" +type = "lua" +script = "switch.lua" +sandbox = "standard" +timeout = 3000 + +[commands.phrases] +ru = ["режим игра", "игровой режим", "включи игровой режим", "переключись в игры"] +en = ["game mode", "gaming mode", "switch to game"] +ua = ["ігровий режим", "режим гри"] + + +[[commands]] +id = "profile.sleep" +type = "lua" +script = "switch.lua" +sandbox = "standard" +timeout = 3000 + +[commands.phrases] +ru = ["режим сон", "ночной режим", "режим тишины", "иди спать", "выключи всё"] +en = ["sleep mode", "night mode", "quiet mode"] +ua = ["нічний режим", "режим сну"] + + +[[commands]] +id = "profile.driving" +type = "lua" +script = "switch.lua" +sandbox = "standard" +timeout = 3000 + +[commands.phrases] +ru = ["режим за рулём", "режим вождения", "я за рулём", "поехали"] +en = ["driving mode", "in the car"] +ua = ["режим водіння"] + + +[[commands]] +id = "profile.default" +type = "lua" +script = "switch.lua" +sandbox = "standard" +timeout = 3000 + +[commands.phrases] +ru = ["обычный режим", "режим по умолчанию", "сбрось режим", "вернись в обычный"] +en = ["normal mode", "default mode", "reset mode"] +ua = ["звичайний режим"] + + +[[commands]] +id = "profile.what" +type = "lua" +script = "what.lua" +sandbox = "minimal" +timeout = 2000 + +[commands.phrases] +ru = ["какой сейчас режим", "в каком ты режиме", "текущий режим", "какой режим"] +en = ["what mode", "current mode", "what profile"] +ua = ["який зараз режим"] diff --git a/resources/commands/profile_switch/switch.lua b/resources/commands/profile_switch/switch.lua new file mode 100644 index 0000000..34dad2e --- /dev/null +++ b/resources/commands/profile_switch/switch.lua @@ -0,0 +1,27 @@ +local cmd_id = jarvis.context.command_id or "" +-- map command id suffix → profile name +local target = cmd_id:gsub("^profile%.", "") +if target == "" then + jarvis.audio.play_error() + return { chain = false } +end + +local ok, err = pcall(function() + local p = jarvis.profile.set(target) + if p and p.greeting and p.greeting ~= "" then + jarvis.speak(p.greeting) + else + local icon = (p and p.icon) or "" + jarvis.speak(string.format("Режим %s %s.", target, icon)) + end +end) + +if not ok then + jarvis.log("warn", "profile switch failed: " .. tostring(err)) + jarvis.speak("Не удалось переключить режим.") + jarvis.audio.play_error() + return { chain = false } +end + +jarvis.audio.play_ok() +return { chain = true } diff --git a/resources/commands/profile_switch/what.lua b/resources/commands/profile_switch/what.lua new file mode 100644 index 0000000..aca585d --- /dev/null +++ b/resources/commands/profile_switch/what.lua @@ -0,0 +1,13 @@ +local p = jarvis.profile.active() +local icon = (p and p.icon) or "" +local name = (p and p.name) or "default" +local desc = (p and p.description) or "" + +local msg = string.format("Сейчас %s %s.", name, icon) +if desc ~= "" then + msg = msg .. " " .. desc +end + +jarvis.speak(msg) +jarvis.audio.play_ok() +return { chain = false } diff --git a/resources/commands/vision/command.toml b/resources/commands/vision/command.toml new file mode 100644 index 0000000..5bd8125 --- /dev/null +++ b/resources/commands/vision/command.toml @@ -0,0 +1,30 @@ +# IMBA-4: Multimodal — screenshot + vision LLM. + +[[commands]] +id = "vision.describe" +type = "lua" +script = "describe.lua" +sandbox = "full" +timeout = 30000 + +[commands.phrases] +ru = [ + "что на экране", "опиши экран", + "посмотри на экран", "что у меня на экране", + "посмотри на мой экран", "глянь на экран", +] +en = ["what's on screen", "describe screen", "look at my screen"] +ua = ["що на екрані", "поглянь на екран"] + + +[[commands]] +id = "vision.read_error" +type = "lua" +script = "read_error.lua" +sandbox = "full" +timeout = 30000 + +[commands.phrases] +ru = ["прочитай ошибку", "что за ошибка", "помоги с ошибкой"] +en = ["read the error", "what's the error", "help with the error"] +ua = ["прочитай помилку", "що за помилка"] diff --git a/resources/commands/vision/describe.lua b/resources/commands/vision/describe.lua new file mode 100644 index 0000000..df969e0 --- /dev/null +++ b/resources/commands/vision/describe.lua @@ -0,0 +1,13 @@ +jarvis.speak("Сейчас посмотрю.") + +local desc = jarvis.vision.describe("Опиши коротко (1-3 предложения) что сейчас на экране. По-русски.") + +if not desc or desc == "" then + jarvis.speak("Не удалось разобрать экран. Проверь GROQ_TOKEN.") + jarvis.audio.play_error() + return { chain = false } +end + +jarvis.speak(desc) +jarvis.audio.play_ok() +return { chain = false } diff --git a/resources/commands/vision/read_error.lua b/resources/commands/vision/read_error.lua new file mode 100644 index 0000000..7208fc0 --- /dev/null +++ b/resources/commands/vision/read_error.lua @@ -0,0 +1,18 @@ +jarvis.speak("Смотрю на ошибку.") + +local prompt = [[Найди на экране текст ошибки (сообщение об ошибке, stack trace, диалог об исключении). +Прочитай ключевое сообщение и одним коротким предложением подскажи возможную причину. +Если ошибки нет — скажи "ошибок не вижу". +Отвечай по-русски. 1-3 предложения максимум.]] + +local desc = jarvis.vision.describe(prompt) + +if not desc or desc == "" then + jarvis.speak("Не получилось прочитать.") + jarvis.audio.play_error() + return { chain = false } +end + +jarvis.speak(desc) +jarvis.audio.play_ok() +return { chain = false } diff --git a/tools/piper/README.md b/tools/piper/README.md new file mode 100644 index 0000000..0079863 --- /dev/null +++ b/tools/piper/README.md @@ -0,0 +1,64 @@ +# Piper TTS (neural voice for J.A.R.V.I.S.) + +Drop-in replacement for the robotic SAPI voice. ~30 MB binary + ~60 MB Russian voice (Irina). Latency 200-500ms on i5. + +## Install + +```powershell +cd C:\Jarvis\rust\tools\piper +pwsh -File install.ps1 +``` + +By default installs `ru_RU-irina-medium` (Russian female). To pick a different voice: + +```powershell +pwsh -File install.ps1 -Voice ru_RU-denis-medium +pwsh -File install.ps1 -Voice ru_RU-dmitri-medium +pwsh -File install.ps1 -Voice ru_RU-ruslan-medium +pwsh -File install.ps1 -Voice en_US-amy-medium # English female +``` + +Force re-download: + +```powershell +pwsh -File install.ps1 -Force +``` + +## Use + +If `piper.exe` + a voice file are present in `tools/piper/`, J.A.R.V.I.S. auto-detects Piper on startup. No env vars required. + +To force a specific backend: + +```bat +set JARVIS_TTS=piper :: use Piper +set JARVIS_TTS=sapi :: force SAPI fallback +set JARVIS_TTS=silero :: use Silero (needs tools/silero/silero_tts.py) +``` + +Override paths: + +```bat +set JARVIS_TTS_PIPER_BIN=D:\custom\piper.exe +set JARVIS_TTS_PIPER_VOICE=D:\voices\my-voice.onnx +``` + +## Voices catalogue + +Full list: https://huggingface.co/rhasspy/piper-voices + +Recommended for Russian: +- `ru_RU-irina-medium` — natural female, default +- `ru_RU-dmitri-medium` — male, clear diction +- `ru_RU-ruslan-medium` — male, slightly deeper +- `ru_RU-denis-medium` — male, "newsreader" tone + +## Troubleshooting + +**"piper.exe not found"** — install script failed; check `install.ps1` output. The fallback is SAPI — assistant still works, just with the robotic voice. + +**"no .onnx voice in voices/"** — voice download failed (HuggingFace blocked?). Manually download from https://huggingface.co/rhasspy/piper-voices/tree/main/ru/ru_RU/irina/medium and drop both `.onnx` + `.onnx.json` into `voices/`. + +**Voice sounds wrong** — try a different model. Each Russian voice has a different timbre. + +**No audio output** — Piper writes to a temp wav, then plays via `System.Media.SoundPlayer` synchronously. Check that the default playback device isn't muted. diff --git a/tools/piper/install.ps1 b/tools/piper/install.ps1 new file mode 100644 index 0000000..3f8a3c4 --- /dev/null +++ b/tools/piper/install.ps1 @@ -0,0 +1,108 @@ +#requires -Version 5.1 +<# +.SYNOPSIS + Download Piper TTS engine + Russian voice into tools/piper/. + +.DESCRIPTION + Fetches the latest piper_windows_amd64.zip from rhasspy/piper-binary and + the Irina (ru_RU) voice .onnx + .json from huggingface rhasspy/piper-voices. + Idempotent — skips if files already exist. Pass -Force to re-download. + +.PARAMETER Voice + Voice name to install. Default "ru_RU-irina-medium". Other options: + ru_RU-denis-medium, ru_RU-dmitri-medium, ru_RU-ruslan-medium, en_US-amy-medium, ... + +.PARAMETER Force + Re-download even if files exist. + +.EXAMPLE + pwsh -File install.ps1 + pwsh -File install.ps1 -Voice ru_RU-denis-medium + pwsh -File install.ps1 -Force +#> + +param( + [string]$Voice = "ru_RU-irina-medium", + [switch]$Force +) + +$ErrorActionPreference = "Stop" +$InformationPreference = "Continue" + +$Root = Split-Path -Parent $MyInvocation.MyCommand.Path +$BinPath = Join-Path $Root "piper.exe" +$VoiceDir = Join-Path $Root "voices" +$VoiceFile = Join-Path $VoiceDir "$Voice.onnx" +$VoiceJson = Join-Path $VoiceDir "$Voice.onnx.json" + +# ─── Piper binary ────────────────────────────────────────────────────────── +if ((Test-Path $BinPath) -and -not $Force) { + Write-Information "[skip] piper.exe already present at $BinPath" +} else { + Write-Information "[fetch] piper_windows_amd64.zip..." + $zip = Join-Path $env:TEMP "piper_windows_amd64.zip" + $url = "https://github.com/rhasspy/piper/releases/latest/download/piper_windows_amd64.zip" + Invoke-WebRequest -Uri $url -OutFile $zip -UseBasicParsing + Write-Information "[unzip] -> $Root" + Expand-Archive -Path $zip -DestinationPath $Root -Force + # piper distributes inside a "piper/" subfolder — flatten it + $inner = Join-Path $Root "piper" + if (Test-Path $inner) { + Get-ChildItem -Path $inner -Force | Move-Item -Destination $Root -Force + Remove-Item -Path $inner -Recurse -Force + } + Remove-Item $zip -Force + if (-not (Test-Path $BinPath)) { + throw "piper.exe not found after extraction" + } + Write-Information "[ok] piper installed at $BinPath" +} + +# ─── Voice files ─────────────────────────────────────────────────────────── +if (-not (Test-Path $VoiceDir)) { + New-Item -ItemType Directory -Path $VoiceDir | Out-Null +} + +# Voice naming: ru_RU-irina-medium -> lang=ru_RU, name=irina, quality=medium +$parts = $Voice -split "-" +if ($parts.Length -lt 3) { + throw "Bad voice name '$Voice' — expected --" +} +$lang = $parts[0] +$name = $parts[1] +$quality = $parts[2] +$langCode = ($lang -split "_")[0] + +$Base = "https://huggingface.co/rhasspy/piper-voices/resolve/main/$langCode/$lang/$name/$quality" + +foreach ($pair in @( + @{ url = "$Base/$Voice.onnx"; out = $VoiceFile }, + @{ url = "$Base/$Voice.onnx.json"; out = $VoiceJson } +)) { + if ((Test-Path $pair.out) -and -not $Force) { + Write-Information "[skip] $(Split-Path -Leaf $pair.out) already present" + continue + } + Write-Information "[fetch] $($pair.url)" + Invoke-WebRequest -Uri $pair.url -OutFile $pair.out -UseBasicParsing + Write-Information "[ok] $(Split-Path -Leaf $pair.out) -> $($pair.out)" +} + +# ─── Smoke test ──────────────────────────────────────────────────────────── +$testWav = Join-Path $env:TEMP "jarvis-piper-test.wav" +try { + Write-Information "[test] synthesizing 'Привет, Дмитрий'..." + "Привет, Дмитрий" | & $BinPath --model $VoiceFile --output_file $testWav 2>$null + if ((Test-Path $testWav) -and ((Get-Item $testWav).Length -gt 1000)) { + Write-Information "[ok] Piper works! Test wav: $testWav" + Write-Information "" + Write-Information "Next: set environment variable JARVIS_TTS=piper" + Write-Information " (or just leave it unset — auto-detect will pick Piper)" + } else { + Write-Warning "Test wav empty or missing — synthesis may have failed" + } +} catch { + Write-Warning "Smoke test failed: $_" +} finally { + if (Test-Path $testWav) { Remove-Item $testWav -Force } +} diff --git a/tools/silero/silero_tts.py b/tools/silero/silero_tts.py new file mode 100644 index 0000000..42ae611 --- /dev/null +++ b/tools/silero/silero_tts.py @@ -0,0 +1,85 @@ +"""Silero TTS helper for J.A.R.V.I.S. + +Reads text from stdin, synthesises with snakers4/silero-models, writes the result +to a temp wav, and prints the path on stdout. The Rust side then plays + deletes it. + +Install: + pip install torch soundfile numpy + +Usage (called by jarvis-core/src/tts/silero.rs): + echo "привет" | python silero_tts.py --voice xenia + +Voices for ru_v3: + xenia (female, default), baya (female), aidar (male), eugene (male), kseniya (female). +""" + +import argparse +import os +import sys +import tempfile + +try: + import torch # type: ignore +except ImportError: + print("silero_tts: torch not installed. Run `pip install torch`.", file=sys.stderr) + sys.exit(2) + +import warnings + +warnings.filterwarnings("ignore") + +_MODEL = None + + +def load_model(device: str = "cpu"): + global _MODEL + if _MODEL is None: + torch.set_num_threads(4) + _MODEL, _ = torch.hub.load( + repo_or_dir="snakers4/silero-models", + model="silero_tts", + language="ru", + speaker="v3_1_ru", + trust_repo=True, + ) + _MODEL.to(torch.device(device)) + return _MODEL + + +def synth(text: str, voice: str, sample_rate: int = 48000) -> str: + """Synth text → return path to temp wav.""" + model = load_model() + audio = model.apply_tts(text=text, speaker=voice, sample_rate=sample_rate) + + import numpy as np + import soundfile as sf # type: ignore + + fd, path = tempfile.mkstemp(prefix="jarvis-silero-", suffix=".wav") + os.close(fd) + sf.write(path, audio.numpy().astype(np.float32), sample_rate) + return path + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--voice", default="xenia", help="speaker name (xenia, baya, aidar, eugene, kseniya)") + ap.add_argument("--sample-rate", type=int, default=48000) + args = ap.parse_args() + + text = sys.stdin.read().strip() + if not text: + print("silero_tts: empty input", file=sys.stderr) + return 1 + + try: + path = synth(text, args.voice, args.sample_rate) + except Exception as exc: + print(f"silero_tts: synth failed: {exc}", file=sys.stderr) + return 3 + + print(path) + return 0 + + +if __name__ == "__main__": + sys.exit(main())