From 6a328077bced00624af0273bd2a2dade04d73cee Mon Sep 17 00:00:00 2001 From: Bossiara13 <236771060+DmitryBykov-ISPO@users.noreply.github.com> Date: Sat, 16 May 2026 00:00:22 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20vision=20+=20macros=20+=20scheduler=20i?= =?UTF-8?q?n=20Python=20(129=20=E2=86=92=20142=20commands)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three big rust packs ported. Python parity now covers the FULL imba lineup except codebase_qa and github_pr. vision_handler.py (new, ~125 lines) - Screenshot via PowerShell System.Drawing → temp PNG → base64 → Groq vision API (llama-3.2-11b-vision-preview, override via GROQ_VISION_MODEL). - do_vision_describe "что на экране" / "опиши экран" - do_vision_read_error "прочитай ошибку" - Requires GROQ_TOKEN; speaks "Не удалось" on missing. macros_store.py (new, ~170 lines) - JSON store at /macros.json (atomic write-through). - start(name) / save() / cancel() / replay(name, dispatch_fn) - is_recording / recording_name / list_names / get / delete - record_step(phrase) called by execute_cmd hook (see main.py) - is_macro_control filter blocks "запиши макрос ..." from recording itself (no infinite recursion). - replay spawns daemon thread, fires each step with 800ms delay. scheduler_store.py (new, ~220 lines) - JSON store at /schedule.json. - parse_schedule(spec) handles "daily HH:MM", "at HH:MM", "every/in N minutes|hours|seconds" — Russian (минут/час/секунд) + English. - Background daemon thread ticks every 30s, fires Speak actions through the same speak callback used by main.py. - Once-tasks auto-delete after firing; daily/interval cycle forever. - add/remove/clear/remove_by_text/list_all. extensions.py (+13 handlers) do_vision_describe / do_vision_read_error → thin proxies to vision_handler do_macros_start / save / cancel / replay / list / delete do_scheduler_add_reminder / add_recurring / list / clear / cancel_by_text main.py - extensions.init_background_services() at startup → scheduler thread boots. - Wraps execute_cmd so every successful dispatch feeds macros_store.record_step. - set_dispatch_fn(_execute_phrase_for_macro) enables macro replay through the normal command pipeline (recognize_cmd → execute_cmd). commands.yaml (+13 entries, 129 → 142) vision_describe, vision_read_error, macros_start, macros_save, macros_cancel, macros_replay, macros_list, macros_delete, scheduler_reminder, scheduler_recurring, scheduler_list, scheduler_clear, scheduler_cancel_text. Tests: ast.parse passes for all five .py files. yaml.safe_load = 142 entries. Python parity status vs rust (72 packs): ✓ memory_pack, profile_switch, vision, scheduler, macros, llm switch (via env), llm_context (basic), media_keys (existing), screenshot, net_info, unit_convert, generators, disk, date_math, sleep_timer (via shutdown), interesting_fact, mailto, self_check, ssl_check, intro/echo ✗ codebase_qa, github_pr (require gh CLI + file walk — TODO) ✗ pomodoro / daily_briefing / habit_nudge / quick_search / diagnostics (could be ported but need scheduler integration + LLM glue) --- commands.yaml | 107 ++++++++++++++++ extensions.py | 242 +++++++++++++++++++++++++++++++++++ macros_store.py | 179 ++++++++++++++++++++++++++ main.py | 34 +++++ scheduler_store.py | 307 +++++++++++++++++++++++++++++++++++++++++++++ vision_handler.py | 146 +++++++++++++++++++++ 6 files changed, 1015 insertions(+) create mode 100644 macros_store.py create mode 100644 scheduler_store.py create mode 100644 vision_handler.py diff --git a/commands.yaml b/commands.yaml index c835c3f..1286b0f 100644 --- a/commands.yaml +++ b/commands.yaml @@ -1113,3 +1113,110 @@ profile_what: - текущий режим - какой режим action: {type: profile_what} + +vision_describe: + phrases: + - что на экране + - опиши экран + - посмотри на экран + - что у меня на экране + - посмотри на мой экран + - глянь на экран + action: {type: vision_describe} + +vision_read_error: + phrases: + - прочитай ошибку + - что за ошибка + - помоги с ошибкой + action: {type: vision_read_error} + +macros_start: + phrases: + - запиши макрос + - начни запись макроса + - записывай макрос + - новый макрос + action: {type: macros_start} + +macros_save: + phrases: + - сохрани макрос + - стоп макрос + - стоп запись макроса + - закончи запись + - макрос готов + action: {type: macros_save} + +macros_cancel: + phrases: + - отмени макрос + - отмени запись макроса + - забудь макрос + action: {type: macros_cancel} + +macros_replay: + phrases: + - запусти макрос + - выполни макрос + - воспроизведи макрос + - сыграй макрос + - проиграй макрос + action: {type: macros_replay} + +macros_list: + phrases: + - какие у меня макросы + - список макросов + - покажи макросы + action: {type: macros_list} + +macros_delete: + phrases: + - удали макрос + - забудь о макросе + action: {type: macros_delete} + +scheduler_reminder: + phrases: + - напомни мне через + - напомни через + - поставь напоминалку через + - поставь напоминание через + - разбуди через + action: {type: scheduler_add_reminder} + +scheduler_recurring: + phrases: + - напоминай каждые + - напоминай каждый + - каждые + - каждый час + action: {type: scheduler_add_recurring} + +scheduler_list: + phrases: + - что у меня запланировано + - какие напоминания + - покажи расписание + - какие задачи у меня + - что в расписании + action: {type: scheduler_list} + +scheduler_clear: + phrases: + - очисти расписание + - удали все напоминания + - отмени все напоминания + - сбрось расписание + action: {type: scheduler_clear} + +scheduler_cancel_text: + phrases: + - отмени напоминание про + - отмени напоминание о + - удали задачу про + - удали задачу о + - отмени про + - забудь напоминание про + action: {type: scheduler_cancel_by_text} diff --git a/extensions.py b/extensions.py index 383a125..0612faf 100644 --- a/extensions.py +++ b/extensions.py @@ -22,6 +22,9 @@ import urllib.request import memory_store import profiles_store +import macros_store +import scheduler_store +import vision_handler _speak_fn = print _set_clipboard_fn = None @@ -31,6 +34,8 @@ def set_speak(fn): """Register the main.py _speak function as our voice output.""" global _speak_fn _speak_fn = fn + scheduler_store.set_speak(fn) + vision_handler.set_speak(fn) def set_clipboard_fn(fn): @@ -39,6 +44,11 @@ def set_clipboard_fn(fn): _set_clipboard_fn = fn +def init_background_services(): + """Boot scheduler. Idempotent. Called once from main.py at startup.""" + scheduler_store.init() + + def _speak(text): try: _speak_fn(text) @@ -581,6 +591,238 @@ def do_profile_what(action, voice): _speak(msg) +# ── vision ────────────────────────────────────────────────────────────────── + +def do_vision_describe(action, voice): + vision_handler.do_vision_describe(action, voice) + + +def do_vision_read_error(action, voice): + vision_handler.do_vision_read_error(action, voice) + + +# ── macros ────────────────────────────────────────────────────────────────── + +def _strip_trig_lower(voice: str, triggers): + low = (voice or '').lower() + for t in triggers: + idx = low.find(t) + if idx >= 0: + return (voice or '')[idx + len(t):].strip(' ,.:') + return voice or '' + + +def do_macros_start(action, voice): + name = _strip_trig_lower(voice, ( + 'начни запись макроса', 'записывай макрос', + 'запиши макрос', 'новый макрос', + )).strip() + if not name: + _speak("Как назвать макрос?") + return + try: + macros_store.start(name) + except ValueError as exc: + _speak(f"Не получилось: {exc}.") + return + _speak(f"Записываю макрос {name}. Говорите команды. Когда закончите — скажите сохрани макрос.") + + +def do_macros_save(action, voice): + try: + count = macros_store.save() + except (RuntimeError,) as exc: + _speak(str(exc)) + return + _speak(f"Сохранил {count} шагов.") + + +def do_macros_cancel(action, voice): + if macros_store.cancel(): + _speak("Запись макроса отменена.") + else: + _speak("Записи не было.") + + +_DISPATCH_FN = None + + +def set_dispatch_fn(fn): + """Register the main.py `execute_cmd(matched_phrase, voice)` for replay.""" + global _DISPATCH_FN + _DISPATCH_FN = fn + + +def do_macros_replay(action, voice): + name = _strip_trig_lower(voice, ( + 'выполни макрос', 'воспроизведи макрос', + 'запусти макрос', 'сыграй макрос', 'проиграй макрос', + )).strip() + if not name: + _speak("Какой макрос запустить?") + return + if _DISPATCH_FN is None: + _speak("Replay не подключен.") + return + try: + count = macros_store.replay(name, _DISPATCH_FN) + except KeyError as exc: + _speak(str(exc)) + return + _speak(f"Запускаю {name}, шагов: {count}.") + + +def do_macros_list(action, voice): + names = macros_store.list_names() + if not names: + _speak("Макросов нет.") + return + sample = names[:5] + line = f"Макросов: {len(names)}. Первые: " + ", ".join(sample) + "." + _speak(line) + + +def do_macros_delete(action, voice): + name = _strip_trig_lower(voice, ( + 'удали макрос', 'забудь о макросе', + )).strip() + if not name: + _speak("Какой макрос удалить?") + return + if macros_store.delete(name): + _speak(f"Удалил макрос {name}.") + else: + _speak(f"Макрос {name} не найден.") + + +# ── scheduler ─────────────────────────────────────────────────────────────── + +def _parse_duration_phrase(voice: str) -> tuple[int, str] | None: + """From 'через 5 минут' / 'через 2 часа', return (count, unit_word).""" + low = (voice or '').lower() + m = re.search(r'(\d+)\s+(\S+)', low) + if not m: + return None + return (int(m.group(1)), m.group(2)) + + +def do_scheduler_add_reminder(action, voice): + body = _strip_trig_lower(voice, ( + 'напомни мне через', 'напомни через', + 'поставь напоминалку через', 'поставь напоминание через', + 'разбуди через', + )) + body = body.strip(' ,.:') + parsed = _parse_duration_phrase(body) + if not parsed: + _speak("Не понял через сколько.") + return + n, unit = parsed + if unit.startswith('минут'): + spec = f"in {n} minutes" + label = f"{n} минут" + elif unit.startswith('час'): + spec = f"in {n} hours" + label = f"{n} час" if n == 1 else (f"{n} часа" if n < 5 else f"{n} часов") + elif unit.startswith('секунд'): + spec = f"in {n} seconds" + label = f"{n} секунд" + else: + _speak("Не понял единицу. Минуты или часы.") + return + + # Body after the number+unit is the reminder text. + text = re.sub(r'^\d+\s+\S+\s*', '', body).strip() or "Напоминание." + try: + scheduler_store.add({ + 'name': 'Reminder', + 'schedule': spec, + 'action': {'type': 'speak', 'text': text}, + }) + except ValueError as exc: + _speak(f"Не получилось: {exc}.") + return + _speak(f"Напомню через {label}.") + + +def do_scheduler_add_recurring(action, voice): + body = _strip_trig_lower(voice, ( + 'напоминай каждые', 'напоминай каждый', + 'каждые', 'каждый час', + )) + body = body.strip(' ,.:') + parsed = _parse_duration_phrase(body) + if not parsed: + # try implicit 1 + m = re.match(r'^(\S+)', body) + if m and m.group(1) in ('час', 'минуту', 'минута'): + n = 1 + unit = m.group(1) + else: + _speak("Не понял через сколько повторять.") + return + else: + n, unit = parsed + + if unit.startswith('минут'): + spec = f"every {n} minutes" + sample = f"{n} минут" + elif unit.startswith('час'): + spec = f"every {n} hours" + sample = f"{n} час" if n == 1 else (f"{n} часа" if n < 5 else f"{n} часов") + else: + _speak("Не понял единицу.") + return + + text = re.sub(r'^\d+\s+\S+\s*', '', body).strip() or "Напоминание." + text = re.sub(r'^напоминай\s+', '', text).strip() or "Напоминание." + try: + scheduler_store.add({ + 'name': 'Recurring', + 'schedule': spec, + 'action': {'type': 'speak', 'text': text}, + }) + except ValueError as exc: + _speak(f"Не получилось: {exc}.") + return + _speak(f"Буду напоминать каждые {sample}.") + + +def do_scheduler_list(action, voice): + tasks = scheduler_store.list_all() + if not tasks: + _speak("Расписание пустое.") + return + line = f"Запланировано {len(tasks)}." + _speak(line) + + +def do_scheduler_clear(action, voice): + n = scheduler_store.clear() + if n == 0: + _speak("В расписании и так пусто.") + else: + _speak(f"Удалил {n} задач.") + + +def do_scheduler_cancel_by_text(action, voice): + query = _strip_trig_lower(voice, ( + 'отмени напоминание про', 'отмени напоминание о', + 'удали задачу про', 'удали задачу о', + 'отмени про', 'забудь напоминание про', + )).strip() + if not query: + _speak("Что именно отменить?") + return + n = scheduler_store.remove_by_text(query) + if n == 0: + _speak(f"Не нашёл напоминаний про {query}.") + elif n == 1: + _speak("Отменил напоминание.") + else: + _speak(f"Отменил {n} напоминаний про {query}.") + + # ── interesting fact (LLM-dependent) ─────────────────────────────────────── def do_interesting_fact(action, voice): diff --git a/macros_store.py b/macros_store.py new file mode 100644 index 0000000..05566a2 --- /dev/null +++ b/macros_store.py @@ -0,0 +1,179 @@ +"""Voice-macro recorder — port of `crates/jarvis-core/src/macros.rs`. + +Records successful command phrases, replays them through the dispatch later. +JSON file at `/macros.json`. + +Public API: + is_recording() / recording_name() + start(name) / save() / cancel() + list_names() / get(name) / delete(name) + record_step(phrase) # called by main.py on each successful command + replay(name, dispatch_fn) # dispatch_fn fires one phrase +""" + +import json +import os +import time +import threading + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_PATH = os.path.join(_HERE, 'macros.json') + +_recording: dict | None = None # {name, steps: list[str]} +_store: dict[str, dict] = {} +_loaded = False +_recording_lock = threading.Lock() + + +def _load(): + global _store, _loaded + if _loaded: + return + if os.path.isfile(_PATH): + try: + with open(_PATH, encoding='utf-8') as f: + data = json.load(f) + _store = data.get('macros', {}) if isinstance(data, dict) else {} + except (OSError, json.JSONDecodeError) as exc: + print(f"[macros] corrupt: {exc}") + _store = {} + _loaded = True + + +def _save(): + tmp = _PATH + '.tmp' + try: + with open(tmp, 'w', encoding='utf-8') as f: + json.dump({'macros': _store}, f, ensure_ascii=False, indent=2) + os.replace(tmp, _PATH) + except OSError as exc: + print(f"[macros] save: {exc}") + + +def _normalize(s: str) -> str: + return (s or '').strip().lower() + + +_CONTROL_TOKENS = ( + 'запиши макрос', 'сохрани макрос', 'отмени макрос', + 'прекрати макрос', 'отмени запись', 'запусти макрос', + 'воспроизведи макрос', 'удали макрос', +) + + +def _is_control(phrase: str) -> bool: + low = (phrase or '').lower() + return any(tok in low for tok in _CONTROL_TOKENS) + + +def is_recording() -> bool: + with _recording_lock: + return _recording is not None + + +def recording_name() -> str | None: + with _recording_lock: + return _recording['name'] if _recording else None + + +def start(name: str) -> None: + nname = _normalize(name) + if not nname: + raise ValueError("empty macro name") + with _recording_lock: + global _recording + _recording = {'name': nname, 'steps': []} + + +def cancel() -> bool: + with _recording_lock: + global _recording + if _recording is None: + return False + _recording = None + return True + + +def save() -> int: + """Stop recording, persist, return step count. Raises if nothing recorded.""" + with _recording_lock: + global _recording + if _recording is None: + raise RuntimeError("не было активной записи") + if not _recording['steps']: + _recording = None + raise RuntimeError("макрос пустой") + + _load() + name = _recording['name'] + steps = list(_recording['steps']) + _recording = None + + _store[name] = { + 'name': name, + 'steps': steps, + 'created_at': int(time.time()), + 'last_run': None, + } + _save() + return len(steps) + + +def record_step(phrase: str) -> None: + """Called by main.py after every successful command execution.""" + if not phrase: + return + if _is_control(phrase): + return + with _recording_lock: + if _recording is None: + return + _recording['steps'].append(phrase.strip()) + + +def list_names() -> list[str]: + _load() + return sorted(_store.keys()) + + +def get(name: str) -> dict | None: + _load() + return _store.get(_normalize(name)) + + +def delete(name: str) -> bool: + _load() + nname = _normalize(name) + if nname in _store: + del _store[nname] + _save() + return True + return False + + +def replay(name: str, dispatch_fn) -> int: + """Replay macro by name. dispatch_fn(phrase) fires a synthetic command. + Returns step count. Raises if macro missing. + Runs in a background thread with 800ms inter-step delay.""" + _load() + nname = _normalize(name) + m = _store.get(nname) + if not m: + raise KeyError(f"Макрос '{name}' не найден.") + + steps = list(m['steps']) + + def runner(): + for i, step in enumerate(steps): + try: + dispatch_fn(step) + except Exception as exc: + print(f"[macros] replay step {i}: {exc}") + if i + 1 < len(steps): + time.sleep(0.8) + m['last_run'] = int(time.time()) + _save() + + threading.Thread(target=runner, name=f"macro-replay-{nname}", + daemon=True).start() + return len(steps) diff --git a/main.py b/main.py index 953863e..d6942c7 100644 --- a/main.py +++ b/main.py @@ -248,6 +248,7 @@ def _set_clipboard(text: str): # Wire extensions module to our voice + clipboard helpers extensions.set_speak(_speak) extensions.set_clipboard_fn(_set_clipboard) +extensions.init_background_services() # boots scheduler thread _CODEGEN_TRIGGERS = ( @@ -1615,6 +1616,39 @@ print(f" J.A.R.V.I.S. v{config.VA_VER} [Python edition]") print(f" Total commands: {len(VA_CMD_LIST)}") print("=" * 60) + +# Hook macros to the dispatch path: registers a callback so macros::replay +# can re-fire commands, and so the recorder can capture successful phrases. +def _execute_phrase_for_macro(phrase: str): + """Replay step: re-fire a phrase through the normal dispatch path.""" + try: + cmd = recognize_cmd(phrase) + except Exception as exc: + print(f"[macros] replay recognize: {exc}") + return + if cmd: + execute_cmd(cmd, phrase) + + +extensions.set_dispatch_fn(_execute_phrase_for_macro) + + +# Wrap execute_cmd to also feed the macro recorder on success. +_orig_execute_cmd = execute_cmd + + +def execute_cmd(cmd: str, voice: str): # noqa: F811 + spec = VA_CMD_LIST.get(cmd) + _orig_execute_cmd(cmd, voice) + if spec: + # On every successful dispatch, feed the recorder. Filter inside + # macros_store.record_step blocks macro-control phrases. + try: + import macros_store + macros_store.record_step(voice) + except Exception as exc: + print(f"[macros] record_step: {exc}") + recorder = PvRecorder(device_index=config.MICROPHONE_INDEX, frame_length=512) recorder.start() print('Using device: %s' % recorder.selected_device) diff --git a/scheduler_store.py b/scheduler_store.py new file mode 100644 index 0000000..51deb8c --- /dev/null +++ b/scheduler_store.py @@ -0,0 +1,307 @@ +"""Proactive scheduler — port of `crates/jarvis-core/src/scheduler.rs`. + +JSON store at /schedule.json. Background daemon thread ticks every 30s +and fires due tasks via the speak callback. + +Schedule formats (parsed by `parse_schedule`): + "daily HH:MM" + "at HH:MM" + "every N minutes" / "every N hours" / "every N seconds" + "in N minutes" / "in N hours" / "in N seconds" + +Public API: + init() # start the background thread, load state + add(task_dict) -> str # returns id + remove(id) -> bool + list_all() -> list[dict] + remove_by_text(query) -> int + clear() -> int +""" + +import json +import os +import re +import threading +import time + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_PATH = os.path.join(_HERE, 'schedule.json') +_TICK_SECS = 30 + +_speak_fn = print +_tasks: list[dict] = [] +_lock = threading.Lock() +_thread_started = False + + +def set_speak(fn): + global _speak_fn + _speak_fn = fn + + +def _speak(text): + try: + _speak_fn(text) + except Exception as exc: + print(f"[scheduler] speak: {exc}") + + +# ── persistence ─────────────────────────────────────────────────────────── + +def _load(): + global _tasks + if os.path.isfile(_PATH): + try: + with open(_PATH, encoding='utf-8') as f: + data = json.load(f) + _tasks = data.get('tasks', []) if isinstance(data, dict) else [] + except (OSError, json.JSONDecodeError) as exc: + print(f"[scheduler] corrupt: {exc}") + _tasks = [] + + +def _save(): + tmp = _PATH + '.tmp' + try: + with open(tmp, 'w', encoding='utf-8') as f: + json.dump({'tasks': _tasks}, f, ensure_ascii=False, indent=2) + os.replace(tmp, _PATH) + except OSError as exc: + print(f"[scheduler] save: {exc}") + + +# ── parsing ─────────────────────────────────────────────────────────────── + +_RU_UNITS = { + 'минут': 60, 'минуту': 60, 'минуты': 60, + 'час': 3600, 'часа': 3600, 'часов': 3600, + 'секунд': 1, 'секунду': 1, 'секунды': 1, +} +_EN_UNITS = { + 'minute': 60, 'minutes': 60, + 'hour': 3600, 'hours': 3600, + 'second': 1, 'seconds': 1, +} + + +def _unit_to_secs(token: str) -> int | None: + """Map a unit word to seconds. Tries Russian stems first.""" + low = token.lower() + for stem, n in _RU_UNITS.items(): + if low.startswith(stem): + return n + for stem, n in _EN_UNITS.items(): + if low.startswith(stem): + return n + return None + + +def parse_schedule(spec: str) -> dict: + """Returns a task-schedule dict, raises ValueError on bad input. + + Schedule shape: + {'kind': 'daily', 'hour': H, 'minute': M} + {'kind': 'interval', 'seconds': S} + {'kind': 'once', 'at': ts} + """ + s = spec.strip().lower() + now = int(time.time()) + + # daily HH:MM + m = re.match(r'^daily\s+(\d{1,2}):(\d{2})$', s) + if m: + h, mn = int(m.group(1)), int(m.group(2)) + if not (0 <= h <= 23 and 0 <= mn <= 59): + raise ValueError("hour/minute out of range") + return {'kind': 'daily', 'hour': h, 'minute': mn} + + # at HH:MM + m = re.match(r'^at\s+(\d{1,2}):(\d{2})$', s) + if m: + h, mn = int(m.group(1)), int(m.group(2)) + if not (0 <= h <= 23 and 0 <= mn <= 59): + raise ValueError("hour/minute out of range") + import datetime as dt + target = dt.datetime.now().replace(hour=h, minute=mn, second=0, microsecond=0) + if target.timestamp() <= now: + target = target + dt.timedelta(days=1) + return {'kind': 'once', 'at': int(target.timestamp())} + + # every N + m = re.match(r'^every\s+(\d+)\s+(\S+)$', s) + if m: + n = int(m.group(1)) + secs_per = _unit_to_secs(m.group(2)) + if secs_per is None: + raise ValueError(f"unknown unit: {m.group(2)}") + return {'kind': 'interval', 'seconds': n * secs_per} + + # in N + m = re.match(r'^in\s+(\d+)\s+(\S+)$', s) + if m: + n = int(m.group(1)) + secs_per = _unit_to_secs(m.group(2)) + if secs_per is None: + raise ValueError(f"unknown unit: {m.group(2)}") + return {'kind': 'once', 'at': now + n * secs_per} + + raise ValueError(f"unknown schedule spec: {spec}") + + +def _next_fire(schedule: dict, last_fired: int | None, now: int) -> int | None: + kind = schedule.get('kind') + if kind == 'once': + if last_fired is not None: + return None + return schedule.get('at') + if kind == 'interval': + anchor = last_fired if last_fired is not None else now + return anchor + int(schedule['seconds']) + if kind == 'daily': + import datetime as dt + h, mn = int(schedule['hour']), int(schedule['minute']) + today = dt.datetime.now().replace(hour=h, minute=mn, second=0, microsecond=0) + today_ts = int(today.timestamp()) + if today_ts > now and (last_fired is None or + dt.datetime.fromtimestamp(last_fired).date() != today.date()): + return today_ts + return today_ts + 86400 + return None + + +# ── public API ──────────────────────────────────────────────────────────── + +def _gen_id() -> str: + import uuid as uuidlib + return f"task-{uuidlib.uuid4().hex[:12]}" + + +def add(task: dict) -> str: + """Add task. Required keys: name, schedule (dict or string), action. + Action is a dict with at least {'type': 'speak', 'text': '...'}. + """ + sched = task.get('schedule') + if isinstance(sched, str): + sched = parse_schedule(sched) + if not isinstance(sched, dict): + raise ValueError("missing or invalid schedule") + + tid = task.get('id') or _gen_id() + record = { + 'id': tid, + 'name': task.get('name', 'task'), + 'schedule': sched, + 'action': task.get('action', {'type': 'speak', 'text': 'Напоминание.'}), + 'last_fired': None, + 'enabled': task.get('enabled', True), + 'created_at': int(time.time()), + } + with _lock: + _tasks[:] = [t for t in _tasks if t['id'] != tid] + _tasks.append(record) + _save() + return tid + + +def remove(task_id: str) -> bool: + with _lock: + before = len(_tasks) + _tasks[:] = [t for t in _tasks if t['id'] != task_id] + if len(_tasks) != before: + _save() + return True + return False + + +def list_all() -> list[dict]: + with _lock: + return [dict(t) for t in _tasks] + + +def clear() -> int: + with _lock: + n = len(_tasks) + _tasks.clear() + _save() + return n + + +def remove_by_text(query: str) -> int: + nq = query.strip().lower() + if not nq: + return 0 + with _lock: + before = len(_tasks) + def matches(t): + if nq in t['name'].lower(): + return True + action = t.get('action', {}) + if action.get('type') == 'speak': + if nq in (action.get('text') or '').lower(): + return True + return False + _tasks[:] = [t for t in _tasks if not matches(t)] + removed = before - len(_tasks) + if removed: + _save() + return removed + + +# ── background tick ──────────────────────────────────────────────────────── + +def _fire_action(action: dict) -> None: + atype = action.get('type', 'speak') + if atype == 'speak': + _speak(action.get('text', '...')) + else: + print(f"[scheduler] unknown action type: {atype}") + + +def _tick(): + now = int(time.time()) + fired_ids = [] + expired_once = [] + with _lock: + for task in list(_tasks): + if not task.get('enabled', True): + continue + nf = _next_fire(task['schedule'], task.get('last_fired'), now) + if nf is not None and nf <= now: + fired_ids.append(task['id']) + if task['schedule'].get('kind') == 'once': + expired_once.append(task['id']) + + # Fire outside the lock — speak() can block. + for task in list_all(): + if task['id'] in fired_ids: + print(f"[scheduler] firing {task['id']} '{task['name']}'") + _fire_action(task['action']) + + if fired_ids: + with _lock: + for task in _tasks: + if task['id'] in fired_ids: + task['last_fired'] = now + _tasks[:] = [t for t in _tasks if t['id'] not in expired_once] + _save() + + +def _tick_loop(): + while True: + try: + time.sleep(_TICK_SECS) + _tick() + except Exception as exc: + print(f"[scheduler] tick err: {exc}") + + +def init(): + """Load tasks + start the background thread (idempotent).""" + global _thread_started + _load() + if _thread_started: + return + _thread_started = True + threading.Thread(target=_tick_loop, name='jarvis-scheduler', + daemon=True).start() + print(f"[scheduler] thread started ({len(_tasks)} tasks loaded)") diff --git a/vision_handler.py b/vision_handler.py new file mode 100644 index 0000000..a3d17af --- /dev/null +++ b/vision_handler.py @@ -0,0 +1,146 @@ +"""Vision handler — port of `crates/jarvis-core/src/lua/api/vision.rs`. + +Captures the primary screen via PowerShell System.Drawing, base64-encodes +the PNG, sends to Groq's vision-capable model. Requires GROQ_TOKEN. + +Action types in commands.yaml: + vision_describe "что на экране" / "опиши экран" + vision_read_error "прочитай ошибку" +""" + +import base64 +import os +import subprocess +import tempfile + +_speak_fn = print + + +def set_speak(fn): + global _speak_fn + _speak_fn = fn + + +def _speak(text): + try: + _speak_fn(text) + except Exception as exc: + print(f"[vision] speak: {exc}") + + +_SCREENSHOT_PS_TEMPLATE = """ +Add-Type -AssemblyName System.Windows.Forms; +Add-Type -AssemblyName System.Drawing; +$b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds; +$bmp = New-Object System.Drawing.Bitmap $b.Width, $b.Height; +$g = [System.Drawing.Graphics]::FromImage($bmp); +$g.CopyFromScreen($b.Location, [System.Drawing.Point]::Empty, $b.Size); +$bmp.Save('{path}', [System.Drawing.Imaging.ImageFormat]::Png); +$g.Dispose(); $bmp.Dispose(); +""" + + +def _take_screenshot() -> str | None: + """Save full screen to a temp PNG, return path or None on failure.""" + fd, path = tempfile.mkstemp(prefix='jarvis-screen-', suffix='.png') + os.close(fd) + ps = _SCREENSHOT_PS_TEMPLATE.format(path=path.replace("'", "''")) + try: + subprocess.run( + ['powershell', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', ps], + capture_output=True, timeout=10, check=True, + ) + except Exception as exc: + print(f"[vision] screenshot: {exc}") + try: + os.unlink(path) + except OSError: + pass + return None + if not os.path.isfile(path) or os.path.getsize(path) < 1000: + try: + os.unlink(path) + except OSError: + pass + return None + return path + + +def _vision_call(prompt: str, image_b64: str) -> str | None: + import config as cfg + if not getattr(cfg, 'GROQ_TOKEN', None): + return None + + try: + from openai import OpenAI + except ImportError: + return None + + client = OpenAI( + api_key=cfg.GROQ_TOKEN, + base_url=getattr(cfg, 'GROQ_BASE_URL', 'https://api.groq.com/openai/v1'), + ) + + model = os.environ.get('GROQ_VISION_MODEL', 'llama-3.2-11b-vision-preview') + + try: + resp = client.chat.completions.create( + model=model, + messages=[{ + 'role': 'user', + 'content': [ + {'type': 'text', 'text': prompt}, + {'type': 'image_url', 'image_url': { + 'url': f'data:image/png;base64,{image_b64}', + }}, + ], + }], + max_tokens=400, temperature=0.2, timeout=30, + ) + return resp.choices[0].message.content.strip() + except Exception as exc: + print(f"[vision] LLM call: {exc}") + return None + + +def _describe(prompt: str) -> str | None: + """Take a screenshot and ask vision-LLM. Returns description or None.""" + path = _take_screenshot() + if not path: + return None + try: + with open(path, 'rb') as f: + b64 = base64.b64encode(f.read()).decode('ascii') + finally: + try: + os.unlink(path) + except OSError: + pass + return _vision_call(prompt, b64) + + +def do_vision_describe(action, voice): + _speak("Сейчас посмотрю.") + prompt = ( + "Опиши коротко (1-3 предложения) что сейчас на экране. По-русски." + ) + result = _describe(prompt) + if not result: + _speak("Не удалось разобрать экран. Проверь GROQ_TOKEN.") + return + _speak(result) + + +def do_vision_read_error(action, voice): + _speak("Смотрю на ошибку.") + prompt = ( + "Найди на экране текст ошибки (сообщение об ошибке, stack trace, диалог " + "об исключении). Прочитай ключевое сообщение и одним коротким " + "предложением подскажи возможную причину. Если ошибки нет — скажи " + "'ошибок не вижу'. Отвечай по-русски. 1-3 предложения максимум." + ) + result = _describe(prompt) + if not result: + _speak("Не получилось прочитать.") + return + _speak(result)