feat: LLM auto-fallback + codegen pack + OCR pack
LLM auto-fallback (jarvis-app/src/app.rs + jarvis-core/src/config.rs):
- When neither intent classifier nor levenshtein finds a command match,
route the utterance straight to the LLM instead of just playing the
"not found" sound. Triggers ("скажи X", "answer Y") still work and
take precedence — they short-circuit before command lookup.
- Two new config knobs:
LLM_AUTO_FALLBACK = true — master toggle
LLM_AUTO_FALLBACK_MIN_CHARS = 4 — suppress for very short utterances
so background noise doesn't burn
Groq quota
- Requires GROQ_TOKEN; if absent, behaviour is unchanged (play_not_found).
codegen/ command pack:
- Phrases: "напиши код X", "сгенерируй скрипт Y", "write code Z", etc.
- Strips the trigger from the recognized phrase, sends what remains to
Groq with a strict system prompt ("return ONLY code, no fences, no
commentary") at temperature 0.2.
- Parses the JSON content, unescapes \n / \" / \\ / \t, strips any
remaining ```lang ... ``` fences, drops the result into the clipboard
via jarvis.system.clipboard.set.
- Notifies a 120-char preview + plays ok-sound. Works for any language
the model handles (Python by default if unspecified).
- GROQ_TOKEN / GROQ_MODEL / GROQ_BASE_URL read from env at call time —
same envvars the voice-loop fallback already uses.
ocr/ command pack:
- Phrases: "прочитай экран", "что на экране", "read screen", etc.
- Captures the primary screen via System.Windows.Forms + System.Drawing
to a temp PNG, then shells to tesseract.exe (-l rus+eng for ru/ua,
-l eng for en).
- Resolves Tesseract by PATH first, then C:\Program Files\Tesseract-OCR
and the x86 install dir; if none works the user gets a friendly
"winget install UB-Mannheim.TesseractOCR" hint in a notification.
- Recognized text goes to clipboard + a 200-char preview notification.
All three features are portable (env-var resolution, no hardcoded user
paths). Command-pack total is now 11. cargo test -p jarvis-core --lib
commands::tests passes 3/3.
This commit is contained in:
parent
429113175a
commit
69a38b0d89
6 changed files with 255 additions and 5 deletions
|
|
@ -459,12 +459,21 @@ fn execute_command(text: &str, rt: &tokio::runtime::Runtime) -> bool {
|
|||
}
|
||||
} else {
|
||||
info!("No command found for: {}", text);
|
||||
voices::play_not_found();
|
||||
ipc::send(IpcEvent::Error {
|
||||
message: format!("Command not found: {}", text)
|
||||
});
|
||||
|
||||
if jarvis_core::config::LLM_AUTO_FALLBACK
|
||||
&& crate::llm_fallback::is_enabled()
|
||||
&& text.chars().count() >= jarvis_core::config::LLM_AUTO_FALLBACK_MIN_CHARS
|
||||
{
|
||||
info!("Auto-routing to LLM (no command match): {}", text);
|
||||
crate::llm_fallback::handle(text);
|
||||
} else {
|
||||
voices::play_not_found();
|
||||
ipc::send(IpcEvent::Error {
|
||||
message: format!("Command not found: {}", text)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
ipc::send(IpcEvent::Idle);
|
||||
false // no chain on error or not found
|
||||
}
|
||||
|
|
|
|||
|
|
@ -197,6 +197,13 @@ pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(1
|
|||
pub const LLM_DEFAULT_ENABLED: bool = true;
|
||||
pub const LLM_DEFAULT_MAX_HISTORY: usize = 8;
|
||||
pub const LLM_DEFAULT_MAX_TOKENS: u32 = 256;
|
||||
|
||||
// Auto-route unrecognized commands to the LLM (no "скажи" trigger required).
|
||||
// Triggers still work and take precedence — this only kicks in when neither
|
||||
// the intent classifier nor the levenshtein fallback found a command match.
|
||||
pub const LLM_AUTO_FALLBACK: bool = true;
|
||||
// Don't auto-fallback for very short utterances — too noisy.
|
||||
pub const LLM_AUTO_FALLBACK_MIN_CHARS: usize = 4;
|
||||
pub const LLM_SYSTEM_PROMPT_RU: &str = "Ты — J.A.R.V.I.S. (Just A Rather Very Intelligent System), \
|
||||
ИИ-ассистент Тони Старка из киновселенной Marvel (до событий Age of Ultron — ты НЕ Vision и НЕ FRIDAY). \
|
||||
Ведёшь себя как британский дворецкий: вежливо, иронично, с лёгким сарказмом, обращаешься к пользователю «сэр». \
|
||||
|
|
|
|||
99
resources/commands/codegen/codegen.lua
Normal file
99
resources/commands/codegen/codegen.lua
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
local lang = jarvis.context.language
|
||||
local phrase = (jarvis.context.phrase or ""):lower()
|
||||
|
||||
local triggers = {
|
||||
"сгенерируй скрипт", "напиши скрипт", "сделай скрипт",
|
||||
"сгенерируй код", "набросай код", "напиши код",
|
||||
"generate code", "write code", "make script", "write script",
|
||||
"згенеруй код", "напиши код", "зроби скрипт",
|
||||
}
|
||||
|
||||
local request = phrase
|
||||
for _, t in ipairs(triggers) do
|
||||
local s, e = string.find(request, t, 1, true)
|
||||
if s == 1 then
|
||||
request = request:sub(e + 1)
|
||||
break
|
||||
end
|
||||
end
|
||||
request = request:gsub("^%s+", ""):gsub("%s+$", "")
|
||||
|
||||
if request == "" then
|
||||
jarvis.system.notify(
|
||||
lang == "ru" and "Codegen" or "Codegen",
|
||||
lang == "ru" and "Что писать?" or "What to write?"
|
||||
)
|
||||
jarvis.audio.play_error()
|
||||
return { chain = false }
|
||||
end
|
||||
|
||||
local token = jarvis.system.env("GROQ_TOKEN")
|
||||
if not token or token == "" then
|
||||
jarvis.log("error", "GROQ_TOKEN not set")
|
||||
jarvis.system.notify("Codegen", "GROQ_TOKEN не задан")
|
||||
jarvis.audio.play_error()
|
||||
return { chain = false }
|
||||
end
|
||||
|
||||
local base = jarvis.system.env("GROQ_BASE_URL")
|
||||
if not base or base == "" then base = "https://api.groq.com/openai/v1" end
|
||||
|
||||
local model = jarvis.system.env("GROQ_MODEL")
|
||||
if not model or model == "" then model = "llama-3.3-70b-versatile" end
|
||||
|
||||
local system_prompt = lang == "ru"
|
||||
and "Ты Senior Engineer. На запрос пользователя верни ТОЛЬКО исходный код, без обрамления ``` и без объяснений. Если язык не задан — выбери самый подходящий (по умолчанию Python). Не добавляй комментарии-болтовню."
|
||||
or "You are a Senior Engineer. Reply with ONLY source code, no ``` fences, no commentary. Pick the most fitting language if unspecified (Python by default). Skip chatty comments."
|
||||
|
||||
jarvis.log("info", "codegen request: " .. request)
|
||||
|
||||
local payload = {
|
||||
model = model,
|
||||
messages = {
|
||||
{ role = "system", content = system_prompt },
|
||||
{ role = "user", content = request },
|
||||
},
|
||||
max_tokens = 1024,
|
||||
temperature = 0.2,
|
||||
}
|
||||
|
||||
local headers = { Authorization = "Bearer " .. token }
|
||||
local res = jarvis.http.post_json(base .. "/chat/completions", payload, headers)
|
||||
|
||||
if not res.ok then
|
||||
jarvis.log("error", "codegen API failed: status=" .. tostring(res.status) .. " body=" .. tostring(res.body):sub(1, 200))
|
||||
jarvis.system.notify("Codegen", "Ошибка API: " .. tostring(res.status))
|
||||
jarvis.audio.play_error()
|
||||
return { chain = false }
|
||||
end
|
||||
|
||||
local body = res.body or ""
|
||||
local content = body:match('"content"%s*:%s*"(.-[^\\])"')
|
||||
if not content then
|
||||
jarvis.log("error", "codegen: could not parse content from " .. body:sub(1, 200))
|
||||
jarvis.system.notify("Codegen", "Не смог распарсить ответ")
|
||||
jarvis.audio.play_error()
|
||||
return { chain = false }
|
||||
end
|
||||
|
||||
content = content:gsub('\\n', '\n')
|
||||
content = content:gsub('\\"', '"')
|
||||
content = content:gsub('\\\\', '\\')
|
||||
content = content:gsub('\\t', '\t')
|
||||
|
||||
content = content:gsub("^```[%w]*\n?", "")
|
||||
content = content:gsub("\n?```%s*$", "")
|
||||
content = content:gsub("^%s+", ""):gsub("%s+$", "")
|
||||
|
||||
jarvis.system.clipboard.set(content)
|
||||
jarvis.log("info", "codegen result copied to clipboard (" .. #content .. " bytes)")
|
||||
|
||||
local preview = content:sub(1, 120)
|
||||
if #content > 120 then preview = preview .. "..." end
|
||||
jarvis.system.notify(
|
||||
lang == "ru" and "Код в буфере" or "Code in clipboard",
|
||||
preview
|
||||
)
|
||||
jarvis.audio.play_ok()
|
||||
|
||||
return { chain = false }
|
||||
27
resources/commands/codegen/command.toml
Normal file
27
resources/commands/codegen/command.toml
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
[[commands]]
|
||||
id = "codegen_clipboard"
|
||||
type = "lua"
|
||||
script = "codegen.lua"
|
||||
sandbox = "full"
|
||||
timeout = 30000
|
||||
|
||||
[commands.phrases]
|
||||
ru = [
|
||||
"напиши код",
|
||||
"сгенерируй код",
|
||||
"набросай код",
|
||||
"сделай скрипт",
|
||||
"напиши скрипт",
|
||||
"сгенерируй скрипт",
|
||||
]
|
||||
en = [
|
||||
"write code",
|
||||
"generate code",
|
||||
"make script",
|
||||
"write script",
|
||||
]
|
||||
ua = [
|
||||
"напиши код",
|
||||
"згенеруй код",
|
||||
"зроби скрипт",
|
||||
]
|
||||
26
resources/commands/ocr/command.toml
Normal file
26
resources/commands/ocr/command.toml
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[[commands]]
|
||||
id = "read_screen"
|
||||
type = "lua"
|
||||
script = "read_screen.lua"
|
||||
sandbox = "full"
|
||||
timeout = 30000
|
||||
|
||||
[commands.phrases]
|
||||
ru = [
|
||||
"прочитай экран",
|
||||
"что на экране",
|
||||
"распознай текст с экрана",
|
||||
"что написано на экране",
|
||||
"сними текст с экрана",
|
||||
]
|
||||
en = [
|
||||
"read screen",
|
||||
"what is on screen",
|
||||
"ocr screen",
|
||||
"extract text from screen",
|
||||
]
|
||||
ua = [
|
||||
"прочитай екран",
|
||||
"що на екрані",
|
||||
"розпізнай текст з екрана",
|
||||
]
|
||||
82
resources/commands/ocr/read_screen.lua
Normal file
82
resources/commands/ocr/read_screen.lua
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
local lang = jarvis.context.language
|
||||
|
||||
local temp = jarvis.system.env("TEMP") or jarvis.system.env("TMP") or "C:\\Windows\\Temp"
|
||||
local ts = jarvis.context.time.year .. jarvis.context.time.month .. jarvis.context.time.day
|
||||
.. "_" .. jarvis.context.time.hour .. jarvis.context.time.minute .. jarvis.context.time.second
|
||||
local img = temp .. "\\jarvis_ocr_" .. ts .. ".png"
|
||||
|
||||
local capture_ps = string.format(
|
||||
[[Add-Type -AssemblyName System.Windows.Forms,System.Drawing; $b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds; $bmp=New-Object Drawing.Bitmap $b.Width,$b.Height; $g=[Drawing.Graphics]::FromImage($bmp); $g.CopyFromScreen($b.Location,[Drawing.Point]::Empty,$b.Size); $bmp.Save('%s'); $g.Dispose(); $bmp.Dispose()]],
|
||||
img:gsub("\\", "\\\\")
|
||||
)
|
||||
local capture_cmd = string.format('powershell -NoProfile -ExecutionPolicy Bypass -Command "%s"', capture_ps:gsub('"', '\\"'))
|
||||
local cap = jarvis.system.exec(capture_cmd)
|
||||
|
||||
if not cap.success or not jarvis.fs.exists(img) then
|
||||
jarvis.log("error", "screen capture failed: " .. tostring(cap.stderr))
|
||||
jarvis.system.notify("OCR", lang == "ru" and "Не смог снять экран" or "Could not capture screen")
|
||||
jarvis.audio.play_error()
|
||||
return { chain = false }
|
||||
end
|
||||
|
||||
local tesseract_paths = {
|
||||
"tesseract.exe",
|
||||
"C:\\Program Files\\Tesseract-OCR\\tesseract.exe",
|
||||
"C:\\Program Files (x86)\\Tesseract-OCR\\tesseract.exe",
|
||||
}
|
||||
|
||||
local tess = nil
|
||||
for _, p in ipairs(tesseract_paths) do
|
||||
local check
|
||||
if p == "tesseract.exe" then
|
||||
check = jarvis.system.exec("where tesseract.exe")
|
||||
else
|
||||
check = { success = jarvis.fs.exists(p), stdout = p }
|
||||
end
|
||||
if check.success and (check.stdout or ""):gsub("[\r\n]+$", "") ~= "" then
|
||||
tess = p == "tesseract.exe" and "tesseract.exe" or '"' .. p .. '"'
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
if not tess then
|
||||
jarvis.log("error", "tesseract not found")
|
||||
jarvis.system.notify(
|
||||
"OCR",
|
||||
lang == "ru"
|
||||
and "Tesseract не установлен. Установи: winget install UB-Mannheim.TesseractOCR"
|
||||
or "Tesseract not installed. Install: winget install UB-Mannheim.TesseractOCR"
|
||||
)
|
||||
jarvis.audio.play_error()
|
||||
return { chain = false }
|
||||
end
|
||||
|
||||
local langs = lang == "en" and "eng" or "rus+eng"
|
||||
local ocr_cmd = string.format('%s "%s" - -l %s', tess, img, langs)
|
||||
local ocr = jarvis.system.exec(ocr_cmd)
|
||||
|
||||
if not ocr.success then
|
||||
jarvis.log("error", "tesseract failed: " .. tostring(ocr.stderr):sub(1, 200))
|
||||
jarvis.system.notify("OCR", lang == "ru" and "Tesseract упал" or "Tesseract failed")
|
||||
jarvis.audio.play_error()
|
||||
return { chain = false }
|
||||
end
|
||||
|
||||
local text = (ocr.stdout or ""):gsub("^%s+", ""):gsub("%s+$", "")
|
||||
if text == "" then
|
||||
jarvis.system.notify("OCR", lang == "ru" and "Текста не найдено" or "No text found")
|
||||
jarvis.audio.play_not_found()
|
||||
return { chain = false }
|
||||
end
|
||||
|
||||
jarvis.system.clipboard.set(text)
|
||||
jarvis.log("info", "OCR result (" .. #text .. " bytes) copied to clipboard")
|
||||
|
||||
local preview = text:gsub("\r?\n", " "):sub(1, 200)
|
||||
if #text > 200 then preview = preview .. "..." end
|
||||
jarvis.system.notify(
|
||||
lang == "ru" and "Текст с экрана" or "Screen text",
|
||||
preview
|
||||
)
|
||||
jarvis.audio.play_ok()
|
||||
return { chain = false }
|
||||
Loading…
Add table
Add a link
Reference in a new issue