feat: vision + macros + scheduler in Python (129 → 142 commands)
Three big rust packs ported. Python parity now covers the FULL imba lineup
except codebase_qa and github_pr.
vision_handler.py (new, ~125 lines)
- Screenshot via PowerShell System.Drawing → temp PNG → base64 → Groq
vision API (llama-3.2-11b-vision-preview, override via GROQ_VISION_MODEL).
- do_vision_describe "что на экране" / "опиши экран"
- do_vision_read_error "прочитай ошибку"
- Requires GROQ_TOKEN; speaks "Не удалось" on missing.
macros_store.py (new, ~170 lines)
- JSON store at <here>/macros.json (atomic write-through).
- start(name) / save() / cancel() / replay(name, dispatch_fn)
- is_recording / recording_name / list_names / get / delete
- record_step(phrase) called by execute_cmd hook (see main.py)
- is_macro_control filter blocks "запиши макрос ..." from recording itself
(no infinite recursion).
- replay spawns daemon thread, fires each step with 800ms delay.
scheduler_store.py (new, ~220 lines)
- JSON store at <here>/schedule.json.
- parse_schedule(spec) handles "daily HH:MM", "at HH:MM",
"every/in N minutes|hours|seconds" — Russian (минут/час/секунд) + English.
- Background daemon thread ticks every 30s, fires Speak actions through the
same speak callback used by main.py.
- Once-tasks auto-delete after firing; daily/interval cycle forever.
- add/remove/clear/remove_by_text/list_all.
extensions.py (+13 handlers)
do_vision_describe / do_vision_read_error → thin proxies to vision_handler
do_macros_start / save / cancel / replay / list / delete
do_scheduler_add_reminder / add_recurring / list / clear / cancel_by_text
main.py
- extensions.init_background_services() at startup → scheduler thread boots.
- Wraps execute_cmd so every successful dispatch feeds macros_store.record_step.
- set_dispatch_fn(_execute_phrase_for_macro) enables macro replay through the
normal command pipeline (recognize_cmd → execute_cmd).
commands.yaml (+13 entries, 129 → 142)
vision_describe, vision_read_error, macros_start, macros_save, macros_cancel,
macros_replay, macros_list, macros_delete, scheduler_reminder,
scheduler_recurring, scheduler_list, scheduler_clear, scheduler_cancel_text.
Tests: ast.parse passes for all five .py files. yaml.safe_load = 142 entries.
Python parity status vs rust (72 packs):
✓ memory_pack, profile_switch, vision, scheduler, macros, llm switch
(via env), llm_context (basic), media_keys (existing), screenshot, net_info,
unit_convert, generators, disk, date_math, sleep_timer (via shutdown),
interesting_fact, mailto, self_check, ssl_check, intro/echo
✗ codebase_qa, github_pr (require gh CLI + file walk — TODO)
✗ pomodoro / daily_briefing / habit_nudge / quick_search / diagnostics
(could be ported but need scheduler integration + LLM glue)
This commit is contained in:
parent
02c6f2f90d
commit
6a328077bc
6 changed files with 1015 additions and 0 deletions
146
vision_handler.py
Normal file
146
vision_handler.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
"""Vision handler — port of `crates/jarvis-core/src/lua/api/vision.rs`.
|
||||
|
||||
Captures the primary screen via PowerShell System.Drawing, base64-encodes
|
||||
the PNG, sends to Groq's vision-capable model. Requires GROQ_TOKEN.
|
||||
|
||||
Action types in commands.yaml:
|
||||
vision_describe "что на экране" / "опиши экран"
|
||||
vision_read_error "прочитай ошибку"
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
_speak_fn = print
|
||||
|
||||
|
||||
def set_speak(fn):
|
||||
global _speak_fn
|
||||
_speak_fn = fn
|
||||
|
||||
|
||||
def _speak(text):
|
||||
try:
|
||||
_speak_fn(text)
|
||||
except Exception as exc:
|
||||
print(f"[vision] speak: {exc}")
|
||||
|
||||
|
||||
_SCREENSHOT_PS_TEMPLATE = """
|
||||
Add-Type -AssemblyName System.Windows.Forms;
|
||||
Add-Type -AssemblyName System.Drawing;
|
||||
$b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds;
|
||||
$bmp = New-Object System.Drawing.Bitmap $b.Width, $b.Height;
|
||||
$g = [System.Drawing.Graphics]::FromImage($bmp);
|
||||
$g.CopyFromScreen($b.Location, [System.Drawing.Point]::Empty, $b.Size);
|
||||
$bmp.Save('{path}', [System.Drawing.Imaging.ImageFormat]::Png);
|
||||
$g.Dispose(); $bmp.Dispose();
|
||||
"""
|
||||
|
||||
|
||||
def _take_screenshot() -> str | None:
|
||||
"""Save full screen to a temp PNG, return path or None on failure."""
|
||||
fd, path = tempfile.mkstemp(prefix='jarvis-screen-', suffix='.png')
|
||||
os.close(fd)
|
||||
ps = _SCREENSHOT_PS_TEMPLATE.format(path=path.replace("'", "''"))
|
||||
try:
|
||||
subprocess.run(
|
||||
['powershell', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', ps],
|
||||
capture_output=True, timeout=10, check=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[vision] screenshot: {exc}")
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
if not os.path.isfile(path) or os.path.getsize(path) < 1000:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def _vision_call(prompt: str, image_b64: str) -> str | None:
|
||||
import config as cfg
|
||||
if not getattr(cfg, 'GROQ_TOKEN', None):
|
||||
return None
|
||||
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
client = OpenAI(
|
||||
api_key=cfg.GROQ_TOKEN,
|
||||
base_url=getattr(cfg, 'GROQ_BASE_URL', 'https://api.groq.com/openai/v1'),
|
||||
)
|
||||
|
||||
model = os.environ.get('GROQ_VISION_MODEL', 'llama-3.2-11b-vision-preview')
|
||||
|
||||
try:
|
||||
resp = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{
|
||||
'role': 'user',
|
||||
'content': [
|
||||
{'type': 'text', 'text': prompt},
|
||||
{'type': 'image_url', 'image_url': {
|
||||
'url': f'data:image/png;base64,{image_b64}',
|
||||
}},
|
||||
],
|
||||
}],
|
||||
max_tokens=400, temperature=0.2, timeout=30,
|
||||
)
|
||||
return resp.choices[0].message.content.strip()
|
||||
except Exception as exc:
|
||||
print(f"[vision] LLM call: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def _describe(prompt: str) -> str | None:
|
||||
"""Take a screenshot and ask vision-LLM. Returns description or None."""
|
||||
path = _take_screenshot()
|
||||
if not path:
|
||||
return None
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode('ascii')
|
||||
finally:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
return _vision_call(prompt, b64)
|
||||
|
||||
|
||||
def do_vision_describe(action, voice):
|
||||
_speak("Сейчас посмотрю.")
|
||||
prompt = (
|
||||
"Опиши коротко (1-3 предложения) что сейчас на экране. По-русски."
|
||||
)
|
||||
result = _describe(prompt)
|
||||
if not result:
|
||||
_speak("Не удалось разобрать экран. Проверь GROQ_TOKEN.")
|
||||
return
|
||||
_speak(result)
|
||||
|
||||
|
||||
def do_vision_read_error(action, voice):
|
||||
_speak("Смотрю на ошибку.")
|
||||
prompt = (
|
||||
"Найди на экране текст ошибки (сообщение об ошибке, stack trace, диалог "
|
||||
"об исключении). Прочитай ключевое сообщение и одним коротким "
|
||||
"предложением подскажи возможную причину. Если ошибки нет — скажи "
|
||||
"'ошибок не вижу'. Отвечай по-русски. 1-3 предложения максимум."
|
||||
)
|
||||
result = _describe(prompt)
|
||||
if not result:
|
||||
_speak("Не получилось прочитать.")
|
||||
return
|
||||
_speak(result)
|
||||
Loading…
Add table
Add a link
Reference in a new issue