J.A.R.V.I.S-rust/crates/jarvis-core/src/lua/engine.rs

175 lines
5.8 KiB
Rust
Raw Normal View History

use mlua::{Lua, Value, StdLib};
use std::path::PathBuf;
use std::time::Duration;
use std::fs;
use super::sandbox::SandboxLevel;
use super::error::LuaError;
use super::{CommandContext, CommandResult};
use super::api;
pub struct LuaEngine {
lua: Lua,
sandbox: SandboxLevel,
}
impl LuaEngine {
pub fn new(sandbox: SandboxLevel) -> Result<Self, LuaError> {
// select which standard libraries to load based on sandbox access level
let std_libs = match sandbox {
SandboxLevel::Minimal => {
StdLib::TABLE | StdLib::STRING | StdLib::MATH
}
SandboxLevel::Standard => {
StdLib::TABLE | StdLib::STRING | StdLib::MATH | StdLib::UTF8
}
SandboxLevel::Full => {
StdLib::TABLE | StdLib::STRING | StdLib::MATH | StdLib::UTF8 | StdLib::OS
}
};
let lua = Lua::new_with(std_libs, mlua::LuaOptions::default())
.map_err(|e| LuaError::InitError(e.to_string()))?;
// remove dangerous globals regardless of sandbox
{
let globals = lua.globals();
// always remove these
let _ = globals.set("loadfile", Value::Nil);
let _ = globals.set("dofile", Value::Nil);
let _ = globals.set("load", Value::Nil);
let _ = globals.set("loadstring", Value::Nil);
// remove io unless full sandbox
if !matches!(sandbox, SandboxLevel::Full) {
let _ = globals.set("io", Value::Nil);
}
// remove os.execute, os.exit, os.setlocale even in full mode
// for SECURITY REASONS!!!
if matches!(sandbox, SandboxLevel::Full) {
if let Ok(os) = globals.get::<mlua::Table>("os") {
let _ = os.set("execute", Value::Nil);
let _ = os.set("exit", Value::Nil);
let _ = os.set("remove", Value::Nil);
let _ = os.set("rename", Value::Nil);
let _ = os.set("setlocale", Value::Nil);
}
}
}
Ok(Self { lua, sandbox })
}
// Register all jarvis APIs
fn register_api(&self, context: &CommandContext) -> Result<(), LuaError> {
let globals = self.lua.globals();
// main jarvis table
let jarvis = self.lua.create_table()
.map_err(|e| LuaError::InitError(e.to_string()))?;
// always register core APIs
api::core::register(&self.lua, &jarvis)?;
api::audio::register(&self.lua, &jarvis)?;
api::context::register(&self.lua, &jarvis, context)?;
arch + features: Lua jarvis.speak() / jarvis.llm() API, /commands GUI page, games / mouse / random_choice packs Architecture — DRY at the Lua API surface: lua/api/tts.rs (new): jarvis.speak(text, opts?). opts.lang = ISO two-letter code (default "ru"); auto-picks first installed voice matching the culture. opts.async = true|false (default true). The 9 packs that previously inlined a 60-character PS-SAPI string for every spoken reply can now reduce to one line. Refactored fun/, dice/, clipboard_read/ as proof — fun/ask.lua went from 75 lines of HTTP+SAPI boilerplate to ~28 lines of intent code. lua/api/llm.rs (new): jarvis.llm(messages, opts?). messages is a list of {role, content} tables, opts.max_tokens / .temperature / .top_p. Returns the assistant string or nil. Gated by sandbox allows_http() (so Standard+ packs only). Internally uses the existing jarvis_core::llm::LlmClient::complete_with, no duplicated HTTP plumbing. llm/client.rs: split complete() into complete_with(messages, max_tokens, temperature, top_p) for the new caller; old complete() is now a thin wrapper at temp=0.7 top_p=1.0 for backward compat. GUI — /commands page rewrite: Replaces the "Раздел в разработке" placeholder. Calls Tauri command get_commands_list (already existed in tauri_commands/commands.rs, was wired but unused). Renders a search-filterable card grid: - cmd id (monospace) - type badge with colour by kind (lua=blue, ahk=red, cli=grey, voice=green, terminate=red, stop_chaining=violet) - sandbox badge - description line (if non-empty) - all phrases for currentLanguage, fallback to en, fallback to first available; each phrase in a small inline pill Toolbar: text input with magnifier icon + reload button (↻). Counter line shows visible/total + "Скажи Джарвис + любую фразу" hint. Counts down as you type the filter. New packs (3, total 33 → 36): games/ — launch_game / list_games. Reads or creates %USERPROFILE%\Documents\jarvis-games.json (the sample preloads dota, cs2, elden ring, witcher 3, factorio + minecraft launcher path + fortnite epic URI). Trigger-strip + fuzzy name/alias match, then launches via steam://rungameid/N (Steam), epic:// (Epic Games), or start "" "path" (anything else). list_games speaks the configured library. mouse/ — left/right/middle/double click + scroll up/down. Single dispatch.lua + _mouse_helper.ps1. PS P/Invokes user32!mouse_event with the right flag pair (LEFTDOWN+LEFTUP / etc.), 30ms gap, doubles do two LEFT cycles with a 50ms gap, wheel uses ±360 ticks. random_choice/ — "выбери из X или Y или Z". Strips the trigger, splits on " или " / " or " / ", " / " либо " / " чи ", math.random picks one, speaks "Я выбираю N". cargo test commands::tests still 3/3. 36 packs verified via jarvis-gui startup log.
2026-05-15 12:58:16 +03:00
api::tts::register(&self.lua, &jarvis)?;
arch + features: jarvis.text API + brightness/window_switch/spelling/network/summarize (41 packs) Architecture — close the trigger-strip duplication: lua/api/text.rs (new): + jarvis.text.strip_trigger(phrase, triggers) — case-insensitive "longest trigger wins" stripping, returns trimmed remainder. The 11 packs that previously inlined a `for _, t in ipairs(triggers) do string.find ... s == 1 then sub` loop can drop to one call. Demonstrated in spelling/, window_switch/, summarize/. + jarvis.text.contains_any(phrase, needles) — boolean check for any of N substrings, case-insensitive. Useful for keyword routing. 5 new portable Lua packs (sandbox=full). 36 → 41 total. cargo test commands::tests still 3/3. brightness/ — brightness_up / down / max / min. WMI WmiMonitorBrightness (read CurrentBrightness) + WmiMonitorBrightnessMethods.WmiSetBrightness (write). ±20% step for up/down; max=100, min=10. Desktop monitors that don'\''t expose the WMI namespace get a friendly "не поддерживается" toast instead of a silent failure. window_switch/ — switch_to_window. Strip trigger ("переключись на" / "switch to" / etc.), apply the same alias map as process_kill (хром→chrome, телега→telegram, ...), then PowerShell finds processes by name OR window-title substring sorted by StartTime desc and uses (New-Object -ComObject WScript.Shell).AppActivate($pid) on the top match. Speaks the window title that got focus. spelling/ — spell_out. Strips "произнеси по буквам" / "spell out" / etc., iterates `word:gmatch(".")`, joins letters with ". " so SAPI gives each one a natural micro-pause. Uses jarvis.text.strip_trigger + jarvis.speak — 24 lines total. network/ — open_wifi_settings (ms-settings:network-wifi) + open_bluetooth_settings (ms-settings:bluetooth) + my_ip. my_ip pulls the first non-loopback, non-APIPA IPv4 with Get-NetIPAddress and fetches WAN IP from api.ipify.org (5s timeout); speaks "Локальный X. Внешний Y". summarize/ — summarize_selection. Synthesises Ctrl+C through user32! keybd_event (so the focused window copies its selection), waits 220 ms for the clipboard to populate, reads via jarvis.system.clipboard.get(), bails if < 20 chars, truncates at 8000, sends to Groq @ temp 0.3 with a "3-5 sentences, keep names" prompt, drops the summary back into the clipboard and speaks it. Lets you select ANY text in ANY app (browser, PDF, IDE) and say "суммируй" to get a spoken TL;DR. The new packs explicitly use the new jarvis.text + jarvis.llm + jarvis.speak API surface — no inline PS-SAPI boilerplate, no inline Groq plumbing. Code is now intent-first.
2026-05-15 13:11:18 +03:00
api::text::register(&self.lua, &jarvis)?;
feat: TTS backend abstraction + 4 'imba' features + Lua packs Big push driven by user feedback ("делай имбу") and web research on what voice assistants need to be the ideal: TTS backend abstraction (P0.1) - new module crates/jarvis-core/src/tts/{mod,sapi,piper,silero}.rs - TtsBackend trait with SapiBackend (current PowerShell), PiperBackend (rhasspy/piper, neural quality), SileroBackend (python subprocess) - JARVIS_TTS env var picks (sapi|piper|silero). Auto-detect Piper if binary + voice present in tools/piper/. Falls back to SAPI on missing. - SpeakOpts {lang, detached, raw} replaces ad-hoc args. text_utils sanitiser applied unless raw=true. - llm_fallback + lua/api/tts both routed through tts::backend(). - tools/piper/install.ps1 downloads piper.exe + ru_RU-irina-medium.onnx from rhasspy releases + huggingface. Smoke-test included. - tools/silero/silero_tts.py helper (PyTorch); rust spawns it as subprocess. IMBA-1 Agentic LLM router - crates/jarvis-app/src/llm_router.rs - When fuzzy/intent matcher fails, LLM picks the closest command from the full registry. Returns JSON {command_id, confidence, reason}. - Threshold-gated re-dispatch via substitute phrase. JARVIS_LLM_ROUTER=1 enables; JARVIS_LLM_ROUTER_THRESHOLD overrides 0.55 default. - Inserted in app.rs::execute_command between "no match" and existing llm_fallback chat fallback. IMBA-2 Long-term memory - crates/jarvis-core/src/long_term_memory.rs — JSON store at APP_CONFIG_DIR/long_term_memory.json. Atomic write-through. - remember/recall/search/forget/all/build_context API. - Lua bindings: jarvis.memory.* (5 functions). - llm_fallback auto-injects relevant facts (substring search of prompt) into system message before LLM call. - Pack resources/commands/memory_pack/ with 4 commands: remember, recall, forget, list. IMBA-3 Profile switching (work/game/sleep/driving/default) - crates/jarvis-core/src/profiles.rs — JSON profiles at APP_CONFIG_DIR/profiles/ Auto-seeds 5 defaults on first run with personality + allow/deny lists + greetings + emoji icons. - active_profile.txt persists choice across restart. - Lua bindings: jarvis.profile.{active,set,list,allows,active_name}. - llm_fallback prepends profile personality to system prompt. - Pack resources/commands/profile_switch/ with 6 voice triggers. IMBA-4 Multimodal screenshot + vision LLM - crates/jarvis-core/src/lua/api/vision.rs — gated on HTTP sandbox. - jarvis.vision.screenshot() captures via PowerShell System.Drawing. - jarvis.vision.describe(prompt?) sends base64 PNG to Groq vision model (default llama-3.2-11b-vision-preview, override via GROQ_VISION_MODEL). - Pack resources/commands/vision/ with 2 commands: describe + read_error. P0.2 Continuous conversation grace window - config::CONVERSATION_GRACE_MS = 30_000. - app.rs: after command result, if grace_ms > 0 keep listening WITHOUT re-wake for the grace duration. Existing CMS_WAIT_DELAY back-dated so the existing timeout fires at start + grace_ms. Tests: 24/24 jarvis-core unit tests pass (including 5 text_utils). Build: cargo build --release -p jarvis-app and -p jarvis-gui both succeed on Windows MSVC (VS 2026 Enterprise vcvars64). Notes for setup: - Piper voice install: pwsh tools/piper/install.ps1 (downloads ~90 MB). - GROQ_TOKEN needed for IMBA-1 (router) and IMBA-4 (vision). - All features are opt-in via env vars or auto-detect; existing SAPI + fuzzy match path remains the default.
2026-05-15 15:32:44 +03:00
api::memory::register(&self.lua, &jarvis)?;
api::profile::register(&self.lua, &jarvis)?;
arch + features: Lua jarvis.speak() / jarvis.llm() API, /commands GUI page, games / mouse / random_choice packs Architecture — DRY at the Lua API surface: lua/api/tts.rs (new): jarvis.speak(text, opts?). opts.lang = ISO two-letter code (default "ru"); auto-picks first installed voice matching the culture. opts.async = true|false (default true). The 9 packs that previously inlined a 60-character PS-SAPI string for every spoken reply can now reduce to one line. Refactored fun/, dice/, clipboard_read/ as proof — fun/ask.lua went from 75 lines of HTTP+SAPI boilerplate to ~28 lines of intent code. lua/api/llm.rs (new): jarvis.llm(messages, opts?). messages is a list of {role, content} tables, opts.max_tokens / .temperature / .top_p. Returns the assistant string or nil. Gated by sandbox allows_http() (so Standard+ packs only). Internally uses the existing jarvis_core::llm::LlmClient::complete_with, no duplicated HTTP plumbing. llm/client.rs: split complete() into complete_with(messages, max_tokens, temperature, top_p) for the new caller; old complete() is now a thin wrapper at temp=0.7 top_p=1.0 for backward compat. GUI — /commands page rewrite: Replaces the "Раздел в разработке" placeholder. Calls Tauri command get_commands_list (already existed in tauri_commands/commands.rs, was wired but unused). Renders a search-filterable card grid: - cmd id (monospace) - type badge with colour by kind (lua=blue, ahk=red, cli=grey, voice=green, terminate=red, stop_chaining=violet) - sandbox badge - description line (if non-empty) - all phrases for currentLanguage, fallback to en, fallback to first available; each phrase in a small inline pill Toolbar: text input with magnifier icon + reload button (↻). Counter line shows visible/total + "Скажи Джарвис + любую фразу" hint. Counts down as you type the filter. New packs (3, total 33 → 36): games/ — launch_game / list_games. Reads or creates %USERPROFILE%\Documents\jarvis-games.json (the sample preloads dota, cs2, elden ring, witcher 3, factorio + minecraft launcher path + fortnite epic URI). Trigger-strip + fuzzy name/alias match, then launches via steam://rungameid/N (Steam), epic:// (Epic Games), or start "" "path" (anything else). list_games speaks the configured library. mouse/ — left/right/middle/double click + scroll up/down. Single dispatch.lua + _mouse_helper.ps1. PS P/Invokes user32!mouse_event with the right flag pair (LEFTDOWN+LEFTUP / etc.), 30ms gap, doubles do two LEFT cycles with a 50ms gap, wheel uses ±360 ticks. random_choice/ — "выбери из X или Y или Z". Strips the trigger, splits on " или " / " or " / ", " / " либо " / " чи ", math.random picks one, speaks "Я выбираю N". cargo test commands::tests still 3/3. 36 packs verified via jarvis-gui startup log.
2026-05-15 12:58:16 +03:00
// sandbox-controlled APIs
if self.sandbox.allows_http() {
api::http::register(&self.lua, &jarvis)?;
arch + features: Lua jarvis.speak() / jarvis.llm() API, /commands GUI page, games / mouse / random_choice packs Architecture — DRY at the Lua API surface: lua/api/tts.rs (new): jarvis.speak(text, opts?). opts.lang = ISO two-letter code (default "ru"); auto-picks first installed voice matching the culture. opts.async = true|false (default true). The 9 packs that previously inlined a 60-character PS-SAPI string for every spoken reply can now reduce to one line. Refactored fun/, dice/, clipboard_read/ as proof — fun/ask.lua went from 75 lines of HTTP+SAPI boilerplate to ~28 lines of intent code. lua/api/llm.rs (new): jarvis.llm(messages, opts?). messages is a list of {role, content} tables, opts.max_tokens / .temperature / .top_p. Returns the assistant string or nil. Gated by sandbox allows_http() (so Standard+ packs only). Internally uses the existing jarvis_core::llm::LlmClient::complete_with, no duplicated HTTP plumbing. llm/client.rs: split complete() into complete_with(messages, max_tokens, temperature, top_p) for the new caller; old complete() is now a thin wrapper at temp=0.7 top_p=1.0 for backward compat. GUI — /commands page rewrite: Replaces the "Раздел в разработке" placeholder. Calls Tauri command get_commands_list (already existed in tauri_commands/commands.rs, was wired but unused). Renders a search-filterable card grid: - cmd id (monospace) - type badge with colour by kind (lua=blue, ahk=red, cli=grey, voice=green, terminate=red, stop_chaining=violet) - sandbox badge - description line (if non-empty) - all phrases for currentLanguage, fallback to en, fallback to first available; each phrase in a small inline pill Toolbar: text input with magnifier icon + reload button (↻). Counter line shows visible/total + "Скажи Джарвис + любую фразу" hint. Counts down as you type the filter. New packs (3, total 33 → 36): games/ — launch_game / list_games. Reads or creates %USERPROFILE%\Documents\jarvis-games.json (the sample preloads dota, cs2, elden ring, witcher 3, factorio + minecraft launcher path + fortnite epic URI). Trigger-strip + fuzzy name/alias match, then launches via steam://rungameid/N (Steam), epic:// (Epic Games), or start "" "path" (anything else). list_games speaks the configured library. mouse/ — left/right/middle/double click + scroll up/down. Single dispatch.lua + _mouse_helper.ps1. PS P/Invokes user32!mouse_event with the right flag pair (LEFTDOWN+LEFTUP / etc.), 30ms gap, doubles do two LEFT cycles with a 50ms gap, wheel uses ±360 ticks. random_choice/ — "выбери из X или Y или Z". Strips the trigger, splits on " или " / " or " / ", " / " либо " / " чи ", math.random picks one, speaks "Я выбираю N". cargo test commands::tests still 3/3. 36 packs verified via jarvis-gui startup log.
2026-05-15 12:58:16 +03:00
api::llm::register(&self.lua, &jarvis)?;
feat: TTS backend abstraction + 4 'imba' features + Lua packs Big push driven by user feedback ("делай имбу") and web research on what voice assistants need to be the ideal: TTS backend abstraction (P0.1) - new module crates/jarvis-core/src/tts/{mod,sapi,piper,silero}.rs - TtsBackend trait with SapiBackend (current PowerShell), PiperBackend (rhasspy/piper, neural quality), SileroBackend (python subprocess) - JARVIS_TTS env var picks (sapi|piper|silero). Auto-detect Piper if binary + voice present in tools/piper/. Falls back to SAPI on missing. - SpeakOpts {lang, detached, raw} replaces ad-hoc args. text_utils sanitiser applied unless raw=true. - llm_fallback + lua/api/tts both routed through tts::backend(). - tools/piper/install.ps1 downloads piper.exe + ru_RU-irina-medium.onnx from rhasspy releases + huggingface. Smoke-test included. - tools/silero/silero_tts.py helper (PyTorch); rust spawns it as subprocess. IMBA-1 Agentic LLM router - crates/jarvis-app/src/llm_router.rs - When fuzzy/intent matcher fails, LLM picks the closest command from the full registry. Returns JSON {command_id, confidence, reason}. - Threshold-gated re-dispatch via substitute phrase. JARVIS_LLM_ROUTER=1 enables; JARVIS_LLM_ROUTER_THRESHOLD overrides 0.55 default. - Inserted in app.rs::execute_command between "no match" and existing llm_fallback chat fallback. IMBA-2 Long-term memory - crates/jarvis-core/src/long_term_memory.rs — JSON store at APP_CONFIG_DIR/long_term_memory.json. Atomic write-through. - remember/recall/search/forget/all/build_context API. - Lua bindings: jarvis.memory.* (5 functions). - llm_fallback auto-injects relevant facts (substring search of prompt) into system message before LLM call. - Pack resources/commands/memory_pack/ with 4 commands: remember, recall, forget, list. IMBA-3 Profile switching (work/game/sleep/driving/default) - crates/jarvis-core/src/profiles.rs — JSON profiles at APP_CONFIG_DIR/profiles/ Auto-seeds 5 defaults on first run with personality + allow/deny lists + greetings + emoji icons. - active_profile.txt persists choice across restart. - Lua bindings: jarvis.profile.{active,set,list,allows,active_name}. - llm_fallback prepends profile personality to system prompt. - Pack resources/commands/profile_switch/ with 6 voice triggers. IMBA-4 Multimodal screenshot + vision LLM - crates/jarvis-core/src/lua/api/vision.rs — gated on HTTP sandbox. - jarvis.vision.screenshot() captures via PowerShell System.Drawing. - jarvis.vision.describe(prompt?) sends base64 PNG to Groq vision model (default llama-3.2-11b-vision-preview, override via GROQ_VISION_MODEL). - Pack resources/commands/vision/ with 2 commands: describe + read_error. P0.2 Continuous conversation grace window - config::CONVERSATION_GRACE_MS = 30_000. - app.rs: after command result, if grace_ms > 0 keep listening WITHOUT re-wake for the grace duration. Existing CMS_WAIT_DELAY back-dated so the existing timeout fires at start + grace_ms. Tests: 24/24 jarvis-core unit tests pass (including 5 text_utils). Build: cargo build --release -p jarvis-app and -p jarvis-gui both succeed on Windows MSVC (VS 2026 Enterprise vcvars64). Notes for setup: - Piper voice install: pwsh tools/piper/install.ps1 (downloads ~90 MB). - GROQ_TOKEN needed for IMBA-1 (router) and IMBA-4 (vision). - All features are opt-in via env vars or auto-detect; existing SAPI + fuzzy match path remains the default.
2026-05-15 15:32:44 +03:00
api::vision::register(&self.lua, &jarvis)?;
}
if self.sandbox.allows_state() {
api::state::register(&self.lua, &jarvis, &context.command_path)?;
}
if self.sandbox.allows_fs() {
api::fs::register(&self.lua, &jarvis, &context.command_path, self.sandbox)?;
}
api::system::register(&self.lua, &jarvis, self.sandbox)?;
globals.set("jarvis", jarvis)
.map_err(|e| LuaError::InitError(e.to_string()))?;
Ok(())
}
// Main LUA executor
pub fn execute(
&self,
script_path: &PathBuf,
context: CommandContext,
timeout: Duration,
) -> Result<CommandResult, LuaError> {
// register APIs
self.register_api(&context)?;
// load script
let script_content = fs::read_to_string(script_path)
.map_err(|e| LuaError::LoadError(format!("{}: {}", script_path.display(), e)))?;
let script_name = script_path.file_name()
.unwrap()
.to_string_lossy()
.to_string();
// set up timeout hook
let start = std::time::Instant::now();
self.lua.set_hook(mlua::HookTriggers {
every_nth_instruction: Some(1000),
..Default::default()
}, move |_lua, _debug| {
if start.elapsed() > timeout {
Err(mlua::Error::runtime("Script timeout"))
} else {
Ok(mlua::VmState::Continue)
}
}).map_err(|e| LuaError::InitError(e.to_string()))?;
// execute script
let result = self.lua.load(&script_content)
.set_name(&script_name)
.eval::<Value>();
// remove hook
let _ = self.lua.remove_hook();
// result
match result {
Ok(value) => Ok(self.parse_result(value)),
Err(e) => {
if e.to_string().contains("timeout") {
Err(LuaError::Timeout)
} else {
Err(LuaError::RuntimeError(e.to_string()))
}
}
}
}
// Parse Lua return value into CommandResult
fn parse_result(&self, value: Value) -> CommandResult {
match value {
// return { chain = false }
Value::Table(t) => {
let chain = t.get::<bool>("chain").unwrap_or(true);
CommandResult { chain }
}
// return false (shorthand for no chain)
Value::Boolean(chain) => CommandResult { chain },
// return nil or no return = chain
_ => CommandResult::default(),
}
}
}