From 69a38b0d893287175fe8432999f3bbeb5bf95799 Mon Sep 17 00:00:00 2001 From: Bossiara13 <236771060+DmitryBykov-ISPO@users.noreply.github.com> Date: Fri, 15 May 2026 01:39:21 +0300 Subject: [PATCH] feat: LLM auto-fallback + codegen pack + OCR pack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LLM auto-fallback (jarvis-app/src/app.rs + jarvis-core/src/config.rs): - When neither intent classifier nor levenshtein finds a command match, route the utterance straight to the LLM instead of just playing the "not found" sound. Triggers ("скажи X", "answer Y") still work and take precedence — they short-circuit before command lookup. - Two new config knobs: LLM_AUTO_FALLBACK = true — master toggle LLM_AUTO_FALLBACK_MIN_CHARS = 4 — suppress for very short utterances so background noise doesn't burn Groq quota - Requires GROQ_TOKEN; if absent, behaviour is unchanged (play_not_found). codegen/ command pack: - Phrases: "напиши код X", "сгенерируй скрипт Y", "write code Z", etc. - Strips the trigger from the recognized phrase, sends what remains to Groq with a strict system prompt ("return ONLY code, no fences, no commentary") at temperature 0.2. - Parses the JSON content, unescapes \n / \" / \\ / \t, strips any remaining ```lang ... ``` fences, drops the result into the clipboard via jarvis.system.clipboard.set. - Notifies a 120-char preview + plays ok-sound. Works for any language the model handles (Python by default if unspecified). - GROQ_TOKEN / GROQ_MODEL / GROQ_BASE_URL read from env at call time — same envvars the voice-loop fallback already uses. ocr/ command pack: - Phrases: "прочитай экран", "что на экране", "read screen", etc. - Captures the primary screen via System.Windows.Forms + System.Drawing to a temp PNG, then shells to tesseract.exe (-l rus+eng for ru/ua, -l eng for en). - Resolves Tesseract by PATH first, then C:\Program Files\Tesseract-OCR and the x86 install dir; if none works the user gets a friendly "winget install UB-Mannheim.TesseractOCR" hint in a notification. - Recognized text goes to clipboard + a 200-char preview notification. All three features are portable (env-var resolution, no hardcoded user paths). Command-pack total is now 11. cargo test -p jarvis-core --lib commands::tests passes 3/3. --- crates/jarvis-app/src/app.rs | 19 +++-- crates/jarvis-core/src/config.rs | 7 ++ resources/commands/codegen/codegen.lua | 99 +++++++++++++++++++++++++ resources/commands/codegen/command.toml | 27 +++++++ resources/commands/ocr/command.toml | 26 +++++++ resources/commands/ocr/read_screen.lua | 82 ++++++++++++++++++++ 6 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 resources/commands/codegen/codegen.lua create mode 100644 resources/commands/codegen/command.toml create mode 100644 resources/commands/ocr/command.toml create mode 100644 resources/commands/ocr/read_screen.lua diff --git a/crates/jarvis-app/src/app.rs b/crates/jarvis-app/src/app.rs index 0c48826..2dd2d86 100644 --- a/crates/jarvis-app/src/app.rs +++ b/crates/jarvis-app/src/app.rs @@ -459,12 +459,21 @@ fn execute_command(text: &str, rt: &tokio::runtime::Runtime) -> bool { } } else { info!("No command found for: {}", text); - voices::play_not_found(); - ipc::send(IpcEvent::Error { - message: format!("Command not found: {}", text) - }); + + if jarvis_core::config::LLM_AUTO_FALLBACK + && crate::llm_fallback::is_enabled() + && text.chars().count() >= jarvis_core::config::LLM_AUTO_FALLBACK_MIN_CHARS + { + info!("Auto-routing to LLM (no command match): {}", text); + crate::llm_fallback::handle(text); + } else { + voices::play_not_found(); + ipc::send(IpcEvent::Error { + message: format!("Command not found: {}", text) + }); + } } - + ipc::send(IpcEvent::Idle); false // no chain on error or not found } diff --git a/crates/jarvis-core/src/config.rs b/crates/jarvis-core/src/config.rs index a975cfc..2a0b228 100644 --- a/crates/jarvis-core/src/config.rs +++ b/crates/jarvis-core/src/config.rs @@ -197,6 +197,13 @@ pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(1 pub const LLM_DEFAULT_ENABLED: bool = true; pub const LLM_DEFAULT_MAX_HISTORY: usize = 8; pub const LLM_DEFAULT_MAX_TOKENS: u32 = 256; + +// Auto-route unrecognized commands to the LLM (no "скажи" trigger required). +// Triggers still work and take precedence — this only kicks in when neither +// the intent classifier nor the levenshtein fallback found a command match. +pub const LLM_AUTO_FALLBACK: bool = true; +// Don't auto-fallback for very short utterances — too noisy. +pub const LLM_AUTO_FALLBACK_MIN_CHARS: usize = 4; pub const LLM_SYSTEM_PROMPT_RU: &str = "Ты — J.A.R.V.I.S. (Just A Rather Very Intelligent System), \ ИИ-ассистент Тони Старка из киновселенной Marvel (до событий Age of Ultron — ты НЕ Vision и НЕ FRIDAY). \ Ведёшь себя как британский дворецкий: вежливо, иронично, с лёгким сарказмом, обращаешься к пользователю «сэр». \ diff --git a/resources/commands/codegen/codegen.lua b/resources/commands/codegen/codegen.lua new file mode 100644 index 0000000..36f2c5f --- /dev/null +++ b/resources/commands/codegen/codegen.lua @@ -0,0 +1,99 @@ +local lang = jarvis.context.language +local phrase = (jarvis.context.phrase or ""):lower() + +local triggers = { + "сгенерируй скрипт", "напиши скрипт", "сделай скрипт", + "сгенерируй код", "набросай код", "напиши код", + "generate code", "write code", "make script", "write script", + "згенеруй код", "напиши код", "зроби скрипт", +} + +local request = phrase +for _, t in ipairs(triggers) do + local s, e = string.find(request, t, 1, true) + if s == 1 then + request = request:sub(e + 1) + break + end +end +request = request:gsub("^%s+", ""):gsub("%s+$", "") + +if request == "" then + jarvis.system.notify( + lang == "ru" and "Codegen" or "Codegen", + lang == "ru" and "Что писать?" or "What to write?" + ) + jarvis.audio.play_error() + return { chain = false } +end + +local token = jarvis.system.env("GROQ_TOKEN") +if not token or token == "" then + jarvis.log("error", "GROQ_TOKEN not set") + jarvis.system.notify("Codegen", "GROQ_TOKEN не задан") + jarvis.audio.play_error() + return { chain = false } +end + +local base = jarvis.system.env("GROQ_BASE_URL") +if not base or base == "" then base = "https://api.groq.com/openai/v1" end + +local model = jarvis.system.env("GROQ_MODEL") +if not model or model == "" then model = "llama-3.3-70b-versatile" end + +local system_prompt = lang == "ru" + and "Ты Senior Engineer. На запрос пользователя верни ТОЛЬКО исходный код, без обрамления ``` и без объяснений. Если язык не задан — выбери самый подходящий (по умолчанию Python). Не добавляй комментарии-болтовню." + or "You are a Senior Engineer. Reply with ONLY source code, no ``` fences, no commentary. Pick the most fitting language if unspecified (Python by default). Skip chatty comments." + +jarvis.log("info", "codegen request: " .. request) + +local payload = { + model = model, + messages = { + { role = "system", content = system_prompt }, + { role = "user", content = request }, + }, + max_tokens = 1024, + temperature = 0.2, +} + +local headers = { Authorization = "Bearer " .. token } +local res = jarvis.http.post_json(base .. "/chat/completions", payload, headers) + +if not res.ok then + jarvis.log("error", "codegen API failed: status=" .. tostring(res.status) .. " body=" .. tostring(res.body):sub(1, 200)) + jarvis.system.notify("Codegen", "Ошибка API: " .. tostring(res.status)) + jarvis.audio.play_error() + return { chain = false } +end + +local body = res.body or "" +local content = body:match('"content"%s*:%s*"(.-[^\\])"') +if not content then + jarvis.log("error", "codegen: could not parse content from " .. body:sub(1, 200)) + jarvis.system.notify("Codegen", "Не смог распарсить ответ") + jarvis.audio.play_error() + return { chain = false } +end + +content = content:gsub('\\n', '\n') +content = content:gsub('\\"', '"') +content = content:gsub('\\\\', '\\') +content = content:gsub('\\t', '\t') + +content = content:gsub("^```[%w]*\n?", "") +content = content:gsub("\n?```%s*$", "") +content = content:gsub("^%s+", ""):gsub("%s+$", "") + +jarvis.system.clipboard.set(content) +jarvis.log("info", "codegen result copied to clipboard (" .. #content .. " bytes)") + +local preview = content:sub(1, 120) +if #content > 120 then preview = preview .. "..." end +jarvis.system.notify( + lang == "ru" and "Код в буфере" or "Code in clipboard", + preview +) +jarvis.audio.play_ok() + +return { chain = false } diff --git a/resources/commands/codegen/command.toml b/resources/commands/codegen/command.toml new file mode 100644 index 0000000..a2f6dda --- /dev/null +++ b/resources/commands/codegen/command.toml @@ -0,0 +1,27 @@ +[[commands]] +id = "codegen_clipboard" +type = "lua" +script = "codegen.lua" +sandbox = "full" +timeout = 30000 + +[commands.phrases] +ru = [ + "напиши код", + "сгенерируй код", + "набросай код", + "сделай скрипт", + "напиши скрипт", + "сгенерируй скрипт", +] +en = [ + "write code", + "generate code", + "make script", + "write script", +] +ua = [ + "напиши код", + "згенеруй код", + "зроби скрипт", +] diff --git a/resources/commands/ocr/command.toml b/resources/commands/ocr/command.toml new file mode 100644 index 0000000..8890cc9 --- /dev/null +++ b/resources/commands/ocr/command.toml @@ -0,0 +1,26 @@ +[[commands]] +id = "read_screen" +type = "lua" +script = "read_screen.lua" +sandbox = "full" +timeout = 30000 + +[commands.phrases] +ru = [ + "прочитай экран", + "что на экране", + "распознай текст с экрана", + "что написано на экране", + "сними текст с экрана", +] +en = [ + "read screen", + "what is on screen", + "ocr screen", + "extract text from screen", +] +ua = [ + "прочитай екран", + "що на екрані", + "розпізнай текст з екрана", +] diff --git a/resources/commands/ocr/read_screen.lua b/resources/commands/ocr/read_screen.lua new file mode 100644 index 0000000..1c6f753 --- /dev/null +++ b/resources/commands/ocr/read_screen.lua @@ -0,0 +1,82 @@ +local lang = jarvis.context.language + +local temp = jarvis.system.env("TEMP") or jarvis.system.env("TMP") or "C:\\Windows\\Temp" +local ts = jarvis.context.time.year .. jarvis.context.time.month .. jarvis.context.time.day + .. "_" .. jarvis.context.time.hour .. jarvis.context.time.minute .. jarvis.context.time.second +local img = temp .. "\\jarvis_ocr_" .. ts .. ".png" + +local capture_ps = string.format( + [[Add-Type -AssemblyName System.Windows.Forms,System.Drawing; $b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bmp=New-Object Drawing.Bitmap $b.Width,$b.Height; $g=[Drawing.Graphics]::FromImage($bmp); $g.CopyFromScreen($b.Location,[Drawing.Point]::Empty,$b.Size); $bmp.Save('%s'); $g.Dispose(); $bmp.Dispose()]], + img:gsub("\\", "\\\\") +) +local capture_cmd = string.format('powershell -NoProfile -ExecutionPolicy Bypass -Command "%s"', capture_ps:gsub('"', '\\"')) +local cap = jarvis.system.exec(capture_cmd) + +if not cap.success or not jarvis.fs.exists(img) then + jarvis.log("error", "screen capture failed: " .. tostring(cap.stderr)) + jarvis.system.notify("OCR", lang == "ru" and "Не смог снять экран" or "Could not capture screen") + jarvis.audio.play_error() + return { chain = false } +end + +local tesseract_paths = { + "tesseract.exe", + "C:\\Program Files\\Tesseract-OCR\\tesseract.exe", + "C:\\Program Files (x86)\\Tesseract-OCR\\tesseract.exe", +} + +local tess = nil +for _, p in ipairs(tesseract_paths) do + local check + if p == "tesseract.exe" then + check = jarvis.system.exec("where tesseract.exe") + else + check = { success = jarvis.fs.exists(p), stdout = p } + end + if check.success and (check.stdout or ""):gsub("[\r\n]+$", "") ~= "" then + tess = p == "tesseract.exe" and "tesseract.exe" or '"' .. p .. '"' + break + end +end + +if not tess then + jarvis.log("error", "tesseract not found") + jarvis.system.notify( + "OCR", + lang == "ru" + and "Tesseract не установлен. Установи: winget install UB-Mannheim.TesseractOCR" + or "Tesseract not installed. Install: winget install UB-Mannheim.TesseractOCR" + ) + jarvis.audio.play_error() + return { chain = false } +end + +local langs = lang == "en" and "eng" or "rus+eng" +local ocr_cmd = string.format('%s "%s" - -l %s', tess, img, langs) +local ocr = jarvis.system.exec(ocr_cmd) + +if not ocr.success then + jarvis.log("error", "tesseract failed: " .. tostring(ocr.stderr):sub(1, 200)) + jarvis.system.notify("OCR", lang == "ru" and "Tesseract упал" or "Tesseract failed") + jarvis.audio.play_error() + return { chain = false } +end + +local text = (ocr.stdout or ""):gsub("^%s+", ""):gsub("%s+$", "") +if text == "" then + jarvis.system.notify("OCR", lang == "ru" and "Текста не найдено" or "No text found") + jarvis.audio.play_not_found() + return { chain = false } +end + +jarvis.system.clipboard.set(text) +jarvis.log("info", "OCR result (" .. #text .. " bytes) copied to clipboard") + +local preview = text:gsub("\r?\n", " "):sub(1, 200) +if #text > 200 then preview = preview .. "..." end +jarvis.system.notify( + lang == "ru" and "Текст с экрана" or "Screen text", + preview +) +jarvis.audio.play_ok() +return { chain = false }