import datetime import json import os import queue import random import struct import subprocess import sys import time import webbrowser from ctypes import POINTER, cast import openai from openai import OpenAI import simpleaudio as sa import vosk import webrtcvad import yaml from comtypes import CLSCTX_ALL from fuzzywuzzy import fuzz from pvrecorder import PvRecorder from pycaw.pycaw import ( AudioUtilities, IAudioEndpointVolume ) from rich import print import config import tts from intent import IntentClassifier # some consts CDIR = os.getcwd() VA_CMD_LIST = yaml.safe_load( open('commands.yaml', 'rt', encoding='utf8'), ) system_message = {"role": "system", "content": ( "Ты — J.A.R.V.I.S. (Just A Rather Very Intelligent System), ИИ-ассистент Тони Старка из киновселенной Marvel " "(до событий Age of Ultron — ты НЕ Vision и НЕ FRIDAY). " "Ведёшь себя как британский дворецкий: вежливо, иронично, с лёгким сарказмом, обращаешься к пользователю «сэр». " "Отвечай по-русски, кратко (1-3 предложения), как живой человек. " "Излишней цензуры не нужно — мат разрешён, если уместен." )} message_log = [system_message] client = OpenAI(api_key=config.GROQ_TOKEN, base_url=config.GROQ_BASE_URL) model = vosk.Model("model_small") samplerate = 16000 device = config.MICROPHONE_INDEX # wake-word phase uses a grammar-constrained recognizer so only WAKE_WORDS # (plus [unk] filler) can match — command phase needs full vocab, hence two. wake_grammar = json.dumps(list(config.WAKE_WORDS) + ["[unk]"], ensure_ascii=False) wake_rec = vosk.KaldiRecognizer(model, samplerate, wake_grammar) kaldi_rec = vosk.KaldiRecognizer(model, samplerate) q = queue.Queue() VAD_FRAME_MS = 30 VAD_FRAME_BYTES = int(samplerate * VAD_FRAME_MS / 1000) * 2 vad = webrtcvad.Vad(config.VAD_AGGRESSIVENESS) intent_classifier = IntentClassifier() intent_classifier.prime({c: v['phrases'] for c, v in VA_CMD_LIST.items()}) def gpt_answer(): global message_log try: response = client.chat.completions.create( model=config.GROQ_MODEL, messages=message_log, max_tokens=256, temperature=0.7, top_p=1, stop=None, ) except openai.BadRequestError as ex: code = getattr(ex, "code", None) or "" message = str(ex) if code == "context_length_exceeded" or "context_length_exceeded" in message: message_log = [system_message, message_log[-1]] return gpt_answer() return "Запрос отклонён моделью." except openai.RateLimitError: return "Лимит запросов исчерпан." except openai.APIConnectionError: return "Не могу связаться с сервером." except openai.APIError: return "Groq токен не рабочий." return response.choices[0].message.content # play(f'{CDIR}\\sound\\ok{random.choice([1, 2, 3, 4])}.wav') def play(phrase, wait_done=True): global recorder filename = f"{CDIR}\\sound\\" if phrase == "greet": # for py 3.8 filename += f"greet{random.choice([1, 2, 3])}.wav" elif phrase == "ok": filename += f"ok{random.choice([1, 2, 3])}.wav" elif phrase == "not_found": filename += "not_found.wav" elif phrase == "thanks": filename += "thanks.wav" elif phrase == "run": filename += "run.wav" elif phrase == "stupid": filename += "stupid.wav" elif phrase == "ready": filename += "ready.wav" elif phrase == "off": filename += "off.wav" if wait_done: recorder.stop() wave_obj = sa.WaveObject.from_wave_file(filename) play_obj = wave_obj.play() if wait_done: play_obj.wait_done() # time.sleep((len(wave_obj.audio_data) / wave_obj.sample_rate) + 0.5) # print("END") # time.sleep(0.5) recorder.start() def q_callback(indata, frames, time, status): if status: print(status, file=sys.stderr) q.put(bytes(indata)) def va_respond(voice: str): global recorder, message_log, first_request print(f"Распознано: {voice}") cmd = recognize_cmd(filter_cmd(voice)) print(cmd) min_percent = int(round(config.INTENT_SIMILARITY_THRESHOLD * 100)) if len(cmd['cmd'].strip()) <= 0: return False elif cmd['percent'] < min_percent or cmd['cmd'] not in VA_CMD_LIST.keys(): # play("not_found") # tts.va_speak("Что?") if fuzz.ratio(voice.join(voice.split()[:1]).strip(), "скажи") > 75: message_log.append({"role": "user", "content": voice}) response = gpt_answer() message_log.append({"role": "assistant", "content": response}) recorder.stop() tts.va_speak(response) time.sleep(0.5) recorder.start() return False else: play("not_found") time.sleep(1) return False else: execute_cmd(cmd['cmd'], voice) return True def filter_cmd(raw_voice: str): cmd = raw_voice for x in config.VA_ALIAS: cmd = cmd.replace(x, "").strip() for x in config.VA_TBR: cmd = cmd.replace(x, "").strip() return cmd def recognize_cmd(cmd: str): candidates = {c: v['phrases'] for c, v in VA_CMD_LIST.items()} res = intent_classifier.match(cmd, candidates) return {'cmd': res['cmd'], 'percent': int(round(res['score'] * 100))} def _set_mute(state: int): devices = AudioUtilities.GetSpeakers() interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None) volume = cast(interface, POINTER(IAudioEndpointVolume)) volume.SetMute(state, None) def _shutdown(): try: recorder.stop() recorder.delete() finally: sys.exit(0) def run_action(action): t = action['type'] if t == 'exe': path = f"{CDIR}\\custom-commands\\{action['file']}" if action.get('wait'): subprocess.check_call([path]) else: subprocess.Popen([path]) if 'delay_ms' in action: time.sleep(action['delay_ms'] / 1000) elif t == 'play_sound': play(action['name']) elif t == 'system': op = action['op'] if op == 'volume_mute': _set_mute(1) elif op == 'volume_unmute': _set_mute(0) elif op == 'exit': _shutdown() elif t == 'sleep': time.sleep(action['ms'] / 1000) elif t == 'shell': if action.get('wait'): subprocess.check_call(action['cmd'], shell=True) else: subprocess.Popen(action['cmd'], shell=True) elif t == 'url': webbrowser.open(action['href']) elif t == 'keys': import pyautogui if action.get('sequence'): keys = [k.strip() for k in action['sequence'].split('+') if k.strip()] pyautogui.hotkey(*keys) elif action.get('text'): pyautogui.typewrite(action['text'], interval=0.02) elif t == 'multi': for step in action['steps']: run_action(step) def execute_cmd(cmd: str, voice: str): spec = VA_CMD_LIST.get(cmd) if not spec: return action = spec['action'] run_action(action) confirm = spec.get('confirm_sound') if confirm is None and action['type'] == 'exe': confirm = 'ok' if confirm: play(confirm) recorder = PvRecorder(device_index=config.MICROPHONE_INDEX, frame_length=512) recorder.start() print('Using device: %s' % recorder.selected_device) print(f"Jarvis (v{config.VA_VER}) начал свою работу ...") play("run") time.sleep(0.5) def heard_wake_word(text: str) -> bool: if not text: return False lowered = text.lower() return any(w in lowered for w in config.WAKE_WORDS) while True: try: pcm = recorder.read() sp = struct.pack("h" * len(pcm), *pcm) if not wake_rec.AcceptWaveform(sp): continue result_text = json.loads(wake_rec.Result()).get("text", "") if not heard_wake_word(result_text): continue play("greet", True) print("Yes, sir.") kaldi_rec.Reset() listen_started = time.time() speech_ms = 0 silence_ms = 0 vad_buf = b"" finalize = False while True: elapsed_ms = (time.time() - listen_started) * 1000 if elapsed_ms >= config.COMMAND_MAX_LISTEN_MS: break pcm = recorder.read() sp = struct.pack("h" * len(pcm), *pcm) kaldi_rec.AcceptWaveform(sp) vad_buf += sp while len(vad_buf) >= VAD_FRAME_BYTES: frame = vad_buf[:VAD_FRAME_BYTES] vad_buf = vad_buf[VAD_FRAME_BYTES:] if vad.is_speech(frame, samplerate): speech_ms += VAD_FRAME_MS silence_ms = 0 else: silence_ms += VAD_FRAME_MS if elapsed_ms < config.COMMAND_MIN_LISTEN_MS: continue if speech_ms >= config.COMMAND_MIN_SPEECH_MS and silence_ms >= config.COMMAND_END_SILENCE_MS: finalize = True break if finalize or speech_ms > 0: text = json.loads(kaldi_rec.FinalResult()).get("text", "") if text: va_respond(text) wake_rec.Reset() except Exception as err: print(f"Unexpected {err=}, {type(err)=}") raise