Compare commits

...
Sign in to create a new pull request.

64 commits
main ... master

Author SHA1 Message Date
Priler
5c3031c977 AI models shared registry + Code cleanup + Better async handling + Some fixes, etc 2026-02-18 21:08:48 +05:00
Priler
b427eacf35 some fixes + gliner first implementation 2026-02-11 07:21:50 +05:00
Priler
e367d11b7c Fix intent classifier init 2026-02-08 07:30:46 +05:00
Priler
5e18503704 New intent classification engine - MiniLM L6v2 and MiniLM L12v2 ONNX 2026-02-08 07:16:03 +05:00
Priler
f9cb13eb25 Bump to Lua 5.5 2026-02-08 06:42:58 +05:00
Priler
d132ffacac Merge branch 'master' of https://github.com/Priler/jarvis 2026-02-08 06:37:55 +05:00
Priler
0ddfa35047 Improved wake-word & command detection, with dual mode 2026-02-08 06:37:39 +05:00
Abraham Tugalov
b20805b2f5 Mark README as outdated
Updated the README to indicate that it is outdated.
2026-01-17 06:27:35 +05:00
Priler
05e3c5c8e7 library files required for the build added 2026-01-17 06:08:09 +05:00
Priler
c4a774b5cf basic Lua 5.4 implementation with few example commands 2026-01-17 05:46:38 +05:00
Priler
c9b9482cc8 Commands/voices multilanguage support + Ukranian vosk model added 2026-01-13 02:21:59 +05:00
Priler
64eb1093d5 VAD fixes + some calibrations 2026-01-08 00:35:21 +05:00
Priler
82eddf5d4c Tray options implementation + Voice system rewrite + build fixes 2026-01-07 23:29:46 +05:00
Priler
44ea5e46e2 Multilingual support via Fluent + Frontend improvements + Rewrite of ArcReactor 2026-01-07 05:04:04 +05:00
Priler
9b310fd831 websockets based IPC added in order to bind GUI/app together 2026-01-07 00:15:36 +05:00
Priler
88ecf21b2c noise suppression added via nnnoiseless + vad + gain-normalizer + few frontend changes 2026-01-06 23:32:58 +05:00
Priler
bd8e2c989d vosk usage optimization 2026-01-05 04:20:43 +05:00
Priler
75edd47d18 vosk model selection in GUI 2026-01-05 03:38:04 +05:00
Priler
f9fa4a9fc1 jarvis-cli added for dev tests 2026-01-05 02:23:32 +05:00
Priler
1ee944ce7a intent recognition added (first implementation) 2026-01-05 01:23:13 +05:00
Priler
6570348130 intent recognition added (first implementation) 2026-01-05 01:22:45 +05:00
Priler
4ceaa7ff20 frontend cleanup 2026-01-04 22:50:34 +05:00
Priler
78eb0c0080 frontend fixes & db rewrite with serde 2026-01-04 09:00:51 +05:00
Priler
a425d8f834 rustpotter detections optimization + few code improvements 2026-01-04 06:14:05 +05:00
Priler
612ed2f1b0 project restructure with Rust workspaces 2026-01-04 05:26:40 +05:00
Priler
24f8c55837 project restructure with Rust workspaces 2026-01-04 05:24:25 +05:00
Priler
6e585dd37d project restructure with Rust workspaces 2026-01-04 05:19:47 +05:00
Priler
1f03a44f33 Draft successful build, versions bump, crates up to date + custom forks of certain crates 2025-12-12 02:05:37 +05:00
Priler
15d33d9bc5 Draft code updates for kira/rodio (new versions) 2025-12-12 01:59:07 +05:00
Priler
a445b8984b Custom pv_recorder git repo (reverted) 2025-12-12 01:36:39 +05:00
Priler
b2fcc900dd Bruh 2025-12-12 00:31:17 +05:00
Abraham Tugalov
e3b3a3ab9c Merge pull request #164 from Alim4567/patch-2
Update command.yaml
2025-12-12 00:30:06 +05:00
Abraham Tugalov
696383059a Merge pull request #90 from benzlokzik/patch-1
Markup correction
2025-12-12 00:28:50 +05:00
Abraham Tugalov
3a0a12c34b Merge pull request #82 from el1sha256/linux_build
Linux build files
2025-12-12 00:28:23 +05:00
Priler
c083c76412 Merge branch 'sawixy/master' 2025-12-12 00:24:03 +05:00
Priler
fe4d4306ce Adjust PR 175: keep only selected changes 2025-12-12 00:19:29 +05:00
Priler
f50029e639 Crates versions bump (picovoice removed) 2025-12-12 00:11:32 +05:00
Priler
3fcd83da80 Accept selected changes from PR #175 2025-12-12 00:09:51 +05:00
Priler
28670a54f8 Accept selected changes from PR #175 2025-12-12 00:01:22 +05:00
Priler
703e316257 Latest changes 2025-12-11 23:43:50 +05:00
sawixy
37e1fe6958 gtk linux fix 2025-09-07 20:01:44 +03:00
Alim4567
afe5f36489 Update command.yaml 2025-06-13 19:15:41 +04:00
Abraham
909b1ab0e4 App architecture modifications.
Now GUI and the app itself is divided into two different binaries.
The app also provides system tray icon.
Whereas the GUI can be used to configure the app.
2023-06-11 19:22:02 +05:00
Abraham
24c9d8f58a App architecture modifications.
Now GUI and the app itself is divided into two different binaries.
The app also provides system tray icon.
Whereas the GUI can be used to configure the app.
2023-06-11 19:18:58 +05:00
Abraham
1323505b0a App architecture modifications.
Now GUI and the app itself is divided into two different binaries.
The app also provides system tray icon.
Whereas the GUI can be used to configure the app.
2023-06-11 19:17:50 +05:00
Abraham
ea489d5719 pre 0.0.3 2023-06-08 21:01:29 +05:00
benzlokzik
aa2d51e98f Markup correction
Small corrections in the markdown markup related to the use of strikethrough lines with list of TTS and wake-word neural links
2023-05-07 06:15:09 +03:00
Abraham
9b7636df2e README typo fixes & some little changes 2023-05-04 01:06:44 +05:00
Abraham
5b636f6763 App window size lowered
Open logs button added
Better links handling
Multiple UI fixes/improvements
etc
2023-05-01 16:54:50 +05:00
Abraham
229d7194a5 Commands fixes + New commands 2023-05-01 04:24:06 +05:00
Abraham
1ef42e27e2 if you have README problems I feel bad for you son,
I've got 99 problems but the b**ch aint one ...
2023-05-01 02:00:15 +05:00
Abraham
278b5cd615 Some funky stuff, bruh 2023-05-01 01:52:47 +05:00
Abraham
85eb06fa06 Recorder rewritten. + Attempt to integrate cpal and portaudio. 2023-04-30 22:32:48 +05:00
Elisha Kravchuk
8b5cd04b22 Merge branch 'master' of github.com:el1sha-git/jarvis into linux_build 2023-04-29 16:39:37 +03:00
Elisha Kravchuk
2934e613fb add files for linux build 2023-04-29 16:34:02 +03:00
Abraham
20081161f3 bruh 2023-04-29 16:02:29 +05:00
Abraham
75691211ad Vosk added as wake-word engine option 2023-04-29 16:00:49 +05:00
Abraham
d8b8ba2617 App icon, logs are now stored to log.txt file, work in progress on intergration with rustpotter 2023-04-29 14:58:16 +05:00
Abraham
49a7a7ef2a Readme fix 2023-04-28 19:49:22 +05:00
Abraham
f0468f9899 Readme fixes 2023-04-28 19:42:55 +05:00
Abraham
1180d11e8f Readme fixes 2023-04-28 19:41:44 +05:00
Abraham
57033334f8 Readme and License 2023-04-28 19:39:48 +05:00
Abraham
52fc5fc06d BETA Fix4 changes 2023-04-28 19:05:35 +05:00
Abraham
54613e3628 Update to Rust programming language. 2023-04-27 00:28:36 +05:00
449 changed files with 686314 additions and 1685 deletions

223
.gitignore vendored
View file

@ -1,188 +1,55 @@
# Created by .ignore support plugin (hsz.mobi)
### Python template
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Custom
model_small/
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
# Logs
logs
*.log
local_settings.py
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
# Flask stuff:
instance/
.webassets-cache
# Editor directories and files
.vscode
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
# Scrapy stuff:
.scrapy
# Etc
__other
psd
list.py
tree.txt
# Sphinx documentation
docs/_build/
# Rust / Cargo 🔧
# Ignore Cargo build output, generated files, and common Rust artifacts
/target/
**/target/
/target/doc/
# PyBuilder
target/
# Binaries and dynamic libraries
*.exe
# *.dll
*.so
*.dylib
# IPython Notebook
.ipynb_checkpoints
# Static libraries
*.a
# *.lib
# pyenv
.python-version
# Rust compilation artifacts
*.rlib
*.rmeta
# celery beat schedule file
celerybeat-schedule
# Backup files and editor temp files
*.rs.bk
# dotenv
.env
# Ignore packaged crates
*.crate
# virtualenv
venv/
ENV/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
### VirtualEnv template
# Virtualenv
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
[Bb]in
[Ii]nclude
[Ll]ib
[Ll]ib64
[Ll]ocal
[Ss]cripts
pyvenv.cfg
.venv
pip-selfcheck.json
### JetBrains template
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# AWS User-specific
.idea/**/aws.xml
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# SonarLint plugin
.idea/sonarlint/
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
# idea folder, uncomment if you don't need it
# .idea
# Ignore AI models
/resources/models/*

3
.idea/.gitignore generated vendored
View file

@ -1,3 +0,0 @@
# Default ignored files
/shelf/
/workspace.xml

View file

@ -1,42 +0,0 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="6">
<item index="0" class="java.lang.String" itemvalue="aiogram" />
<item index="1" class="java.lang.String" itemvalue="aiohttp" />
<item index="2" class="java.lang.String" itemvalue="uvloop" />
<item index="3" class="java.lang.String" itemvalue="uvjson" />
<item index="4" class="java.lang.String" itemvalue="ujson" />
<item index="5" class="java.lang.String" itemvalue="pynput" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="E501" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N806" />
<option value="N802" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="str.strip" />
<option value="eventlet.wsgi" />
</list>
</option>
</inspection_tool>
</profile>
</component>

View file

@ -1,6 +0,0 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

10
.idea/jarvis.iml generated
View file

@ -1,10 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="jdk" jdkName="Python 3.10 (jarvis)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

4
.idea/misc.xml generated
View file

@ -1,4 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (jarvis)" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml generated
View file

@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/jarvis.iml" filepath="$PROJECT_DIR$/.idea/jarvis.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml generated
View file

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

9582
Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

55
Cargo.toml Normal file
View file

@ -0,0 +1,55 @@
[workspace]
members = [
"crates/jarvis-core",
"crates/jarvis-app",
"crates/jarvis-gui",
"crates/jarvis-cli"
]
resolver = "2"
[workspace.package]
version = "0.1.0"
authors = ["Abraham Tugalov"]
license = "GPL-3.0-only"
repository = "https://github.com/Priler/jarvis"
edition = "2021"
[workspace.dependencies]
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
serde_yaml = "0.9"
once_cell = "1.19"
log = "0.4"
rand = "0.8"
seqdiff = "0.3"
hound = "3.5"
platform-dirs = "0.3"
rodio = "0.21"
kira = "0.11"
pv_recorder = { git = "https://github.com/Priler/pvrecorder" }
vosk = "0.3"
rustpotter = { git = "https://github.com/Priler/rustpotter" }
image = "0.25"
parking_lot = "0.12.5"
toml = "0.9.8"
sha2 = "0.10"
nnnoiseless = "0.5"
sysinfo = "0.37.2"
tokio-tungstenite = "0.28.0"
futures-util = "0.3"
fluent = "0.17.0"
fluent-bundle = "0.16.0"
unic-langid = "0.9"
chrono = "0.4"
mlua = { version = "0.11.5", features = ["lua55", "vendored", "async", "serde"] }
reqwest = { version = "0.13.1", features = ["blocking", "json"] }
tempfile = "^3.24"
winrt-notification = "0.5"
fastembed = { version = "^5.8.1", default-features = false, features = ["ort-download-binaries"] }
ort = { version = "=2.0.0-rc.11" }
ndarray = "0.17"
tokenizers = { version = "0.22", default-features = false }
regex = "1"
sys-locale = "0.3"

File diff suppressed because it is too large Load diff

View file

@ -1,30 +1,65 @@
# Kesha v3.0 very early (aka Jarvis update)
Simple Voice Assistant made as an experiment using [Silero](https://github.com/snakers4/silero-models) & [Vosk](https://pypi.org/project/vosk/).
<br>Later on [Picovoice Porcupine Wake Word Detection](https://picovoice.ai/platform/porcupine/) & [ChatGPT](https://chat.openai.com/) was added.
# JARVIS Voice Assistant (this readme is outdated)
![image](https://i.pinimg.com/originals/63/e9/b7/63e9b72b983793f64bffc07fd14a0e62.jpg)
![We are NOT limited by the technology of our time!](poster.jpg)
`The code has NOT been polished and is provided "as is". There are a lot of code that are redundant and there are tons of improvements that can be made.`
`Jarvis` - is a voice assistant made as an experiment using neural networks for things like **STT/TTS/Wake Word/NLU** etc.
# Installation
First, install the requirements, the `requirements.txt` file is just an output of `pip freeze` from my test venv 'k.<br>
Second, check `config.py` and set required values (api key, device index).<br>
Next, run the `main.py` script and Voilà, as simple as that.<br><br>
The main project challenges we try to achieve is:
- 100% offline *(no cloud)*
- Open source *(full transparency)*
- No data collection *(we respect your privacy)*
And don't forget to put models of Vosk to main folder.<br>
You can get the latest from the [official website.](https://alphacephei.com/vosk/models)
<br>The one I was using is `small`.
<br>p.s. If you don't understand how to install or where to put the Vosk model, I've made a [screenshot](https://i.imgur.com/N3bu2lC.png) for you.
Our backend stack is 🦀 **[Rust](https://www.rust-lang.org/)** with ❤️ **[Tauri](https://tauri.app/)**.<br>
For the frontend we use ⚡️ **[Vite](https://vitejs.dev/)** + 🛠️ **[Svelte](https://svelte.dev/)**.
# Python version
I was using Python `3.8.3`, but it should work on any newer version.
*Other libraries, tools and packages can be found in source code.*
# ToDo
- Адекватная архитектура кода, собрать всё и переписать from the ground up.
- Задержка воспроизведения звука на основе реальной длительности .wav файла (прогружать при запуске?)
- Speech to intent?
- Отключать self listening protection во время воспроизведения с наушников.
- Указание из списка или по имени будет реализовано позже.
## Neural Networks
# Author
(2022) Abraham Tugalov
This are the neural networks we are currently using:
- Speech-To-Text
- [Vosk Speech Recognition Toolkit](https://github.com/alphacep/vosk-api) via [Vosk-rs](https://github.com/Bear-03/vosk-rs)
- Text-To-Speech
- [~~Silero TTS~~](https://github.com/snakers4/silero-models) *(currently not used)*
- [~~Coqui TTS~~](https://github.com/coqui-ai/TTS) *(currently not used)*
- [~~WinRT~~](https://github.com/ndarilek/tts-rs) *(currently not used)*
- [~gTTS~](https://github.com/nightlyistaken/tts_rust) *(currently not used)*
- [~~SAM~~](https://github.com/s-macke/SAM) *(currently not used)*
- Wake Word
- [Rustpotter](https://github.com/GiviMAD/rustpotter) *(Partially implemented, still WIP)*
- [Picovoice Porcupine](https://github.com/Picovoice/porcupine) via [official SDK](https://github.com/Picovoice/porcupine#rust) *(requires API key)*
- [Vosk Speech Recognition Toolkit](https://github.com/alphacep/vosk-api) via [Vosk-rs](https://github.com/Bear-03/vosk-rs) *(very slow)*
- [~~Snowboy~~](https://github.com/Kitt-AI/snowboy) *(currently not used)*
- NLU
- Nothing yet.
- Chat
- [~~ChatGPT~~](https://chat.openai.com/) (coming soon)
## Supported Languages
Currently, only Russian language is supported.<br>
But soon, Ukranian and English will be added for the interface, wake-word detection and speech recognition.
## How to build?
Nothing special was used to build this project.<br>
You need only Rust and NodeJS installed on your system.<br>
Other than that, all you need is to install all the dependencies and then compile the code with `cargo tauri build` command.<br>
Or run dev with `cargo tauri dev`.
<br><br>
*Thought you might need some of the platform specific libraries for [PvRecorder](https://github.com/Picovoice/pvrecorder) and [Vosk](https://github.com/alphacep/vosk-api).*
## Author
Abraham Tugalov
## Python version?
Old version of Jarvis was built with Python.<br>
The last Python version commit can be found [here](https://github.com/Priler/jarvis/tree/943efbfbdb8aeb5889fa5e2dc7348ca4ea0b81df).
## License
[Attribution-NonCommercial-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-nc-sa/4.0/)<br>
See LICENSE.txt file for more details.

View file

@ -1,157 +0,0 @@
gaming_mode_off:
- рабочий режим
- вернись в рабочий режим
- переключись на компьютер
- обратно на компьютер
- отключи игровой режим
- выйди из игрового режима
- вернись на компьютер
- выход с игрового режима
- рабочее пространство
- вернись в игровой режим
- выключи игровой режим
gaming_mode_on:
- включи игровой режим
- перейди в игровой режим
- я хочу поиграть
- переключись на телевизор
music:
- давай музыку
- музыка
- яндекс музыка
- включи музыку
- открой музыку
- послушаем музыку
- хочу музыку
- врубай музло
- включи что-то послушать
- давай что-то послушаем
music_next:
- следующий трек
- следующая мелодия
- следующая песня
- переключи трек
- переключи музыку
- переключи мелодию
- переключи песню
- следующая музыка
music_off:
- выключи музыку
- отруби музыку
- убери музыку
- отключи музыку
- закрой музыку
- останови музыку
- хватит музыки
music_prev:
- предыдущая музыка
- предыдущий трек
- предыдущая мелодия
- предыдущая песня
- верни трек
- верни мелодию
- верни песню
- верни прошлый трек
- верни прошлую мелодию
- верни прошлую песню
- верни прошлую музыку
- верни то что играло
- давай предыдущий трек
- переключи музыку обратно
- переключи на прошлый трек
- переключи на прошлую мелодию
- переключи на прошлую песню
music_save:
- сохрани трек
- мне нравится трек
- сохрани песню
- мне нравится песня
- добавь трек в избранное
- добавь песню в избранное
- запомни мелодию
- запомни трек
- запомни песню
- добавь мелодию в избранное
- сохрани то что сейчас играет
- добавь то что сейчас играет в избранное
- лайкни трек
- лайкни мелодию
- лайкни песню
'off':
- выключись
- вырубись
- завершить работу
- закройся
- отключись
- заверши свою работу
- на сегодня хватит
- выгрузи себя из памяти
- ты мне надоел
- пора спать
open_browser:
- открой браузер
- запусти браузер
- открой гугл хром
- гугл хром
open_google:
- открой гугл
- гугл
- запусти гугл
open_youtube:
- открой ютуб
- ютуб
- запусти ютуб
sound_off:
- выключи звук
- беззвучный режим
- режим без звука
- отключи звук
sound_on:
- включи звук
- режим со звуком
- верни звук
stupid:
- ты дурак
- ты дебил
- ты глупый
- ты тупой
switch_to_dynamics:
- переключи на динамики
- включи динамики
- перейди на динамики
- давай звук в динамики
- давай звук на динамики
- переключи звук в динамики
- переключи звук на динамики
- переключи на колонки
- включи колонки
- перейди на колонки
- давай звук в колонки
- давай звук на колонки
- переключи звук в колонки
- переключи звук на колонки
switch_to_headphones:
- переключи на наушники
- включи наушники
- перейди на наушники
- давай звук в наушники
- давай звук на наушники
- переключи звук в наушники
- переключи звук на наушники
- переключи на кракены
- включи кракены
- перейди на кракены
- давай звук в кракены
- давай звук на кракены
- переключи звук в кракены
- переключи звук на кракены
thanks:
- спасибо
- молодец
- респект
- ты супер
- отличная работа
- ты крут
- ты большой молодец
- ты реально крут
- ты афигенный

View file

@ -1,24 +0,0 @@
from dotenv import load_dotenv
import os
# Find .env file with os variables
load_dotenv("dev.env")
# Конфигурация
VA_NAME = 'Jarvis'
VA_VER = "3.0"
VA_ALIAS = ('джарвис',)
VA_TBR = ('скажи', 'покажи', 'ответь', 'произнеси', 'расскажи', 'сколько', 'слушай')
# ID микрофона (можете просто менять ID пока при запуске не отобразится нужный)
# -1 это стандартное записывающее устройство
MICROPHONE_INDEX = -1
# Путь к браузеру Google Chrome
CHROME_PATH = 'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe %s'
# Токен Picovoice
PICOVOICE_TOKEN = os.getenv('PICOVOICE_TOKEN')
# Токен OpenAI
OPENAI_TOKEN = os.getenv('OPENAI_TOKEN')

View file

@ -0,0 +1,16 @@
# Cargo configuration file
[build]
jobs = 4
rustflags = [
# "-Ctarget-feature=+fp16,+fhm"
]
[profile.dev]
opt-level = 0
debug = true
[profile.release]
opt-level = 3
debug = false
lto = true

View file

@ -0,0 +1,32 @@
[package]
name = "jarvis-app"
version.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
edition.workspace = true
[dependencies]
jarvis-core = { path = "../jarvis-core", features = ["intent"] }
once_cell.workspace = true
log.workspace = true
simple-log = "2.4"
tray-icon = "0.21"
winit = "0.30"
image.workspace = true
platform-dirs.workspace = true
rand.workspace = true
parking_lot.workspace = true
tokio = { version = "1", features = ["rt-multi-thread"] }
[target.'cfg(windows)'.dependencies.winit]
version = "0.30"
features = []
[target.'cfg(target_os = "windows")'.dependencies]
winapi = { version = "0.3", features = ["winuser"] }
[target.'cfg(target_os = "linux")'.dependencies]
gtk = "0.18"
glib = "0.21.5"

View file

@ -0,0 +1,45 @@
[tasks.format]
install_crate = "rustfmt"
command = "cargo"
args = ["fmt", "--", "--emit=files"]
[tasks.clean]
command = "cargo"
args = ["clean"]
[tasks.build_debug]
command = "cargo"
args = ["build"]
[tasks.run]
command = "cargo"
args = ["run"]
[tasks.build_release]
command = "cargo"
args = ["build", "--release"]
dependencies = ["clean"]
[tasks.test]
command = "cargo"
args = ["test"]
# dependencies = ["clean"]
[tasks.post_build]
script_runner = "python"
script_extension = "py"
script = { file = "post_build.py" }
[tasks.debug]
dependencies = [
"format",
"build_debug",
"post_build"
]
[tasks.release]
dependencies = [
"format",
"build_release",
"post_build"
]

View file

@ -0,0 +1,10 @@
fn main() {
// link to Vosk lib
// println!("cargo:rustc-link-lib=libvosk.dll");
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
let lib_path = std::path::Path::new(&manifest_dir)
.join("..\\..\\lib\\windows\\amd64");
println!("cargo:rustc-link-search=native={}", lib_path.display());
}

View file

@ -0,0 +1,337 @@
use std::sync::mpsc::Receiver;
use std::time::SystemTime;
use jarvis_core::{audio_buffer::AudioRingBuffer, audio, audio_processing, commands, config, listener, recorder, stt, COMMANDS_LIST, intent, voices, ipc::{self, IpcEvent}};
use rand::prelude::*;
use crate::should_stop;
// VAD state machine
#[derive(Debug, Clone, Copy, PartialEq)]
enum VadState {
WaitingForVoice,
VoiceActive,
}
pub fn start(text_cmd_rx: Receiver<String>) -> Result<(), ()> {
// start the loop
main_loop(text_cmd_rx)
}
fn main_loop(text_cmd_rx: Receiver<String>) -> Result<(), ()> {
let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime");
let mut start: SystemTime;
// let sounds_directory = audio::get_sound_directory().unwrap();
let frame_length: usize = 512; // default for every wake-word engine
let sample_rate: usize = 16000;
let mut frame_buffer: Vec<i16> = vec![0; frame_length];
// ring buffer: keep last 2 seconds of audio
let mut audio_buffer = AudioRingBuffer::new(2.0, frame_length, sample_rate);
// VAD state
let mut vad_state = VadState::WaitingForVoice;
let mut silence_frames: u32 = 0;
// how many frames of silence before we consider speech ended
// 1.5 seconds = 1.5 * (16000 / 512) ≈ 47 frames
// @TODO: Put this to config
let silence_threshold: u32 = ((1.5 * sample_rate as f32) / frame_length as f32) as u32;
// play some startup phrase
// audio::play_sound(&sounds_directory.join("run.wav"));
voices::play_greet();
// start recording
match recorder::start_recording() {
Ok(_) => info!("Recording started."),
Err(_) => {
error!("Cannot start recording.");
return Err(()); // quit
}
}
// notify GUI we're ready
ipc::send(IpcEvent::Idle);
// DEBUG counter
let mut frame_count: u32 = 0;
// the loop
'wake_word: loop {
// check for stop signal
if should_stop() {
info!("Stop signal received, shutting down...");
voices::play_goodbye();
ipc::send(IpcEvent::Stopping);
break;
}
// check for text commands
if let Ok(text) = text_cmd_rx.try_recv() {
process_text_command(&text, &rt);
continue 'wake_word;
}
// read from microphone
recorder::read_microphone(&mut frame_buffer);
// DEBUG: check raw audio
frame_count += 1;
let raw_rms = calculate_rms(&frame_buffer);
if frame_count % 100 == 0 {
info!("DEBUG [{}]: raw_rms={:.0}", frame_count, raw_rms);
}
// check if we're getting any audio at all
if frame_count == 100 && raw_rms < 10.0 {
warn!("WARNING: Microphone appears to be silent! RMS={:.0}", raw_rms);
}
// process audio (gain -> noise suppression -> VAD)
let processed = audio_processing::process(&frame_buffer);
if frame_count % 100 == 0 {
info!("DEBUG [{}]: is_voice={}, vad_conf={:.2}, processed_rms={:.0}",
frame_count,
processed.is_voice,
processed.vad_confidence,
calculate_rms(&processed.samples)
);
}
// skip if no voice detected (vad)
if !processed.is_voice {
continue 'wake_word;
}
// DEBUG: we passed VAD
if frame_count % 50 == 0 {
info!("DEBUG: Voice detected, checking wake word...");
}
// recognize wake-word
match listener::data_callback(&frame_buffer) {
Some(_keyword_index) => {
// notify GUI
ipc::send(IpcEvent::WakeWordDetected);
// reset some things
stt::reset_wake_recognizer();
stt::reset_speech_recognizer();
audio_processing::reset();
// wake-word activated, process further commands
// capture current time
start = SystemTime::now();
silence_frames = 0;
// play some reply phrase
// @TODO. Make it via commands or upcoming events system.
voices::play_reply();
// notify GUI we're listening
ipc::send(IpcEvent::Listening);
// wait for voice commands
'voice_recognition: loop {
// check for stop
if should_stop() {
break 'wake_word;
}
// read from microphone
recorder::read_microphone(&mut frame_buffer);
// process first
let processed = audio_processing::process(&frame_buffer);
// detect silence, return to wake-word if silence
if processed.is_voice {
silence_frames = 0;
} else {
silence_frames += 1;
if silence_frames > config::VAD_SILENCE_FRAMES * 2 {
info!("Long silence detected, returning to wake word mode.");
break 'voice_recognition;
}
}
// stt part (without partials)
if let Some(mut recognized_voice) = stt::recognize(&frame_buffer, false) {
// something was recognized
info!("Recognized voice: {}", recognized_voice);
// notify GUI
ipc::send(IpcEvent::SpeechRecognized {
text: recognized_voice.clone(),
});
// filter recognized voice
// @TODO. Better recognized voice filtration.
recognized_voice = recognized_voice.to_lowercase();
// answer again if it's activation phrase repeated
if recognized_voice.contains(config::VOSK_FETCH_PHRASE) {
info!("Wake word detected during chaining, reactivating...");
// play greet sound
// audio::play_sound(&sounds_directory.join(format!(
// "{}.wav",
// config::ASSISTANT_GREET_PHRASES
// .choose(&mut rand::thread_rng())
// .unwrap()
// )));
voices::play_reply();
// reset timer and continue listening
start = SystemTime::now();
silence_frames = 0;
stt::reset_speech_recognizer();
ipc::send(IpcEvent::Listening);
continue 'voice_recognition;
}
// filter out activation phrase from command
for tbr in config::ASSISTANT_PHRASES_TBR {
recognized_voice = recognized_voice.replace(tbr, "");
}
recognized_voice = recognized_voice.trim().into();
// skip if nothing left after filtering (*evil laugh*)
if recognized_voice.is_empty() {
continue 'voice_recognition;
}
// execute command (shared executor)
execute_command(&recognized_voice, &rt);
// return to wake-word listening after command execution (no matter successful or not)
break 'voice_recognition;
}
// only recognize voice for a certain period of time
match start.elapsed() {
Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
// return to wake-word listening after N seconds
break 'voice_recognition;
}
_ => (),
}
// reset things
stt::reset_wake_recognizer();
audio_processing::reset();
ipc::send(IpcEvent::Idle);
}
}
None => (),
}
}
// cleanup
recorder::stop_recording().ok();
ipc::send(IpcEvent::Stopping);
Ok(())
}
// process text command from GUI
fn process_text_command(text: &str, rt: &tokio::runtime::Runtime) {
info!("Processing text command: {}", text);
ipc::send(IpcEvent::SpeechRecognized { text: text.to_string() });
// filter text same as voice
let mut filtered = text.to_lowercase();
for tbr in config::ASSISTANT_PHRASES_TBR {
filtered = filtered.replace(tbr, "");
}
let filtered = filtered.trim();
if filtered.is_empty() {
ipc::send(IpcEvent::Idle);
return;
}
execute_command(filtered, rt);
}
// shared command execution logic (manual & voice)
fn execute_command(text: &str, rt: &tokio::runtime::Runtime) {
let commands_list = match COMMANDS_LIST.get() {
Some(c) => c,
None => {
ipc::send(IpcEvent::Error { message: "Commands not loaded".to_string() });
ipc::send(IpcEvent::Idle);
return;
}
};
// let sounds_directory = audio::get_sound_directory().unwrap();
// try intent recognition first, fallback to levenshtein
let cmd_result = if let Some((intent_id, confidence)) =
rt.block_on(intent::classify(text))
{
info!("Intent recognized: {} (confidence: {:.2})", intent_id, confidence);
intent::get_command_by_intent(commands_list, &intent_id)
} else {
info!("Intent not recognized, trying levenshtein fallback...");
commands::fetch_command(text, commands_list)
};
if let Some((cmd_path, cmd_config)) = cmd_result {
info!("Command found: {:?}", cmd_path);
match commands::execute_command(&cmd_path, &cmd_config) {
Ok(_) => {
info!("Command executed successfully");
voices::play_ok(); // command executed sound
ipc::send(IpcEvent::CommandExecuted {
id: cmd_config.id.clone(),
success: true,
});
}
Err(msg) => {
error!("Error executing command: {}", msg);
voices::play_error();
ipc::send(IpcEvent::CommandExecuted {
id: cmd_config.id.clone(),
success: false,
});
ipc::send(IpcEvent::Error { message: msg.to_string() });
}
}
} else {
info!("No command found for: {}", text);
// play "not understood" sound
// audio::play_sound(&sounds_directory.join("not_understand.wav"));
voices::play_not_found();
ipc::send(IpcEvent::Error {
message: format!("Command not found: {}", text)
});
}
ipc::send(IpcEvent::Idle);
}
fn keyword_callback(keyword_index: i32) {}
pub fn close(code: i32) {
info!("Closing application.");
voices::play_goodbye();
ipc::send(IpcEvent::Stopping);
std::process::exit(code);
}
fn calculate_rms(samples: &[i16]) -> f32 {
if samples.is_empty() { return 0.0; }
let sum: f64 = samples.iter().map(|&s| (s as f64).powi(2)).sum();
(sum / samples.len() as f64).sqrt() as f32
}

View file

@ -0,0 +1,92 @@
#[macro_use]
extern crate lazy_static; // better switch to once_cell ?
use pickledb::{PickleDb, PickleDbDumpPolicy, SerializationMethod};
use log::{info};
use log::LevelFilter;
use std::sync::Mutex;
// include assistant commands
mod assistant_commands;
use assistant_commands::AssistantCommand;
// include vosk
mod vosk;
// include events
mod events;
// include recorder
mod recorder;
// init PickleDb connection
lazy_static! {
static ref DB: Mutex<PickleDb> = Mutex::new(
PickleDb::load(
format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME),
PickleDbDumpPolicy::AutoDump,
SerializationMethod::Json
)
.unwrap_or_else(|_x: _| {
info!("Creating new db file at {} ...", format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME));
PickleDb::new(
format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME),
PickleDbDumpPolicy::AutoDump,
SerializationMethod::Json,
)
})
);
}
fn main() {
// init vosk
vosk::init_vosk();
// run the app
tauri::Builder::default()
.setup(|app| {
std::fs::create_dir_all(app.path_resolver().app_config_dir().unwrap())?;
APP_CONFIG_DIR.lock().unwrap().push_str(app.path_resolver().app_config_dir().unwrap().to_str().unwrap());
std::fs::create_dir_all(app.path_resolver().app_log_dir().unwrap())?;
APP_LOG_DIR.lock().unwrap().push_str(app.path_resolver().app_log_dir().unwrap().to_str().unwrap());
// log to file
let log_file_path = format!("{}/{}", APP_LOG_DIR.lock().unwrap(), config::LOG_FILE_NAME);
println!("!!!===============!!!\nLOGGING TO {}\n!!!===============!!!\n", &log_file_path);
simple_logging::log_to_file(&log_file_path, LevelFilter::max()).expect("Failed to start logger ... is directory writable?");
Ok(())
})
.invoke_handler(tauri::generate_handler![
// db commands
tauri_commands::db_read,
tauri_commands::db_write,
// recorder commands
tauri_commands::pv_get_audio_devices,
tauri_commands::pv_get_audio_device_name,
// listener commands
tauri_commands::start_listening,
tauri_commands::stop_listening,
tauri_commands::is_listening,
// sys commands
tauri_commands::get_current_ram_usage,
tauri_commands::get_peak_ram_usage,
tauri_commands::get_cpu_temp,
tauri_commands::get_cpu_usage,
// sound commands
tauri_commands::play_sound,
// fs commands
tauri_commands::show_in_folder,
// etc commands
tauri_commands::get_app_version,
tauri_commands::get_author_name,
tauri_commands::get_repository_link,
tauri_commands::get_tg_official_link,
tauri_commands::get_feedback_link,
tauri_commands::get_log_file_path
])
.run(tauri::generate_context!())
.expect("error while running tauri application");
}

View file

@ -0,0 +1,418 @@
use std::sync::mpsc::Receiver;
use std::time::SystemTime;
use jarvis_core::{audio_buffer::AudioRingBuffer, audio_processing, commands, config, listener, recorder, stt, COMMANDS_LIST, intent, voices, ipc::{self, IpcEvent}, i18n, slots};
use rand::seq::SliceRandom;
use crate::should_stop;
// VAD state machine
#[derive(Debug, Clone, Copy, PartialEq)]
enum VadState {
WaitingForVoice,
VoiceActive,
}
pub fn start(text_cmd_rx: Receiver<String>, rt: &tokio::runtime::Runtime) -> Result<(), ()> {
main_loop(text_cmd_rx, rt)
}
fn main_loop(text_cmd_rx: Receiver<String>, rt: &tokio::runtime::Runtime) -> Result<(), ()> {
let frame_length: usize = 512;
let sample_rate: usize = 16000;
let mut frame_buffer: Vec<i16> = vec![0; frame_length];
// ring buffer: keeps last 5 seconds of audio (pre-roll)
let mut audio_buffer = AudioRingBuffer::new(5.0, frame_length, sample_rate);
// VAD state
let mut vad_state = VadState::WaitingForVoice;
let mut silence_frames: u32 = 0;
// how many frames of silence before we consider speech ended
// 1.5 seconds = 1.5 * (16000 / 512) ≈ 47 frames
let silence_threshold: u32 = ((1.5 * sample_rate as f32) / frame_length as f32) as u32;
voices::play_greet();
match recorder::start_recording() {
Ok(_) => info!("Recording started. Microphone: {}",
recorder::get_audio_device_name(recorder::get_selected_microphone_index())),
Err(_) => {
error!("Cannot start recording.");
return Err(());
}
}
ipc::send(IpcEvent::Idle);
// ### WAKE WORD DETECTION LOOP
'wake_word: loop {
if should_stop() {
info!("Stop signal received, shutting down...");
voices::play_goodbye();
ipc::send(IpcEvent::Stopping);
break;
}
if let Ok(text) = text_cmd_rx.try_recv() {
process_text_command(&text, &rt);
continue 'wake_word;
}
recorder::read_microphone(&mut frame_buffer);
let processed = audio_processing::process(&frame_buffer);
match vad_state {
VadState::WaitingForVoice => {
// always buffer audio
audio_buffer.push(&frame_buffer);
if processed.is_voice {
// voice started! flush buffer to Vosk
info!("VAD: Voice started, flushing {} buffered frames", audio_buffer.len());
for buffered_frame in audio_buffer.drain_all() {
listener::data_callback(&buffered_frame);
}
vad_state = VadState::VoiceActive;
silence_frames = 0;
}
}
VadState::VoiceActive => {
// dual-feed: speech recognizer gets frames in parallel with wake word detector
let _ = stt::recognize(&frame_buffer, false);
// feed to wake word detector
if let Some(_keyword_index) = listener::data_callback(&frame_buffer) {
// WAKE WORD DETECTED!
info!("Wake word activated!");
ipc::send(IpcEvent::WakeWordDetected);
stt::reset_wake_recognizer();
audio_processing::reset();
// brief sniff to keep feeding STT while transitioning
let sniff_frames = ((0.3 * sample_rate as f32) / frame_length as f32) as u32;
for _ in 0..sniff_frames {
recorder::read_microphone(&mut frame_buffer);
audio_processing::process(&frame_buffer);
stt::recognize(&frame_buffer, false);
}
ipc::send(IpcEvent::Listening);
recognize_command(&mut frame_buffer, &rt, frame_length, sample_rate, true);
// reset state after command
vad_state = VadState::WaitingForVoice;
silence_frames = 0;
audio_buffer.clear();
stt::reset_wake_recognizer();
stt::reset_speech_recognizer(); // NOW reset, after command is done
audio_processing::reset();
ipc::send(IpcEvent::Idle);
continue 'wake_word;
}
// track silence
if processed.is_voice {
silence_frames = 0;
} else {
silence_frames += 1;
if silence_frames > silence_threshold {
debug!("VAD: Silence timeout, returning to wait state");
vad_state = VadState::WaitingForVoice;
silence_frames = 0;
stt::reset_wake_recognizer();
stt::reset_speech_recognizer(); // reset since we were dual-feeding
}
}
}
}
}
recorder::stop_recording().ok();
ipc::send(IpcEvent::Stopping);
Ok(())
}
// Voice recognition for command after wake word
fn recognize_command(
frame_buffer: &mut [i16],
rt: &tokio::runtime::Runtime,
frame_length: usize,
sample_rate: usize,
prefed_audio: bool
) {
let mut audio_buffer = AudioRingBuffer::new(2.0, frame_length, sample_rate);
let mut vad_state = if prefed_audio {
VadState::VoiceActive
} else {
VadState::WaitingForVoice
};
let mut silence_frames: u32 = 0;
let mut start = SystemTime::now();
let mut first_recognition = prefed_audio;
// longer silence threshold for commands (user might pause to think)
// 5 seconds
let silence_threshold: u32 = ((5.0 * sample_rate as f32) / frame_length as f32) as u32;
loop {
if crate::should_stop() {
return;
}
recorder::read_microphone(frame_buffer);
let processed = audio_processing::process(frame_buffer);
match vad_state {
VadState::WaitingForVoice => {
audio_buffer.push(frame_buffer);
if processed.is_voice {
// flush buffer to STT
for buffered_frame in audio_buffer.drain_all() {
stt::recognize(&buffered_frame, false);
}
vad_state = VadState::VoiceActive;
silence_frames = 0;
} else {
silence_frames += 1;
if silence_frames > silence_threshold {
info!("Long silence detected, returning to wake word mode.");
return;
}
}
}
VadState::VoiceActive => {
// feed to STT
if let Some(mut recognized_voice) = stt::recognize(frame_buffer, false) {
info!("Recognized voice: {}", recognized_voice);
ipc::send(IpcEvent::SpeechRecognized {
text: recognized_voice.clone(),
});
recognized_voice = recognized_voice.to_lowercase();
// check if wake word repeated (reactivate)
let wake_phrases = config::get_wake_phrases(&i18n::get_language());
let contains_wake = wake_phrases.iter().any(|wp| recognized_voice.contains(wp));
if contains_wake {
// strip the wake word
let mut remaining = recognized_voice.clone();
for wp in wake_phrases {
remaining = remaining.replace(wp, "");
}
let remaining = remaining.trim();
if remaining.is_empty() {
if first_recognition {
// leftover wake word from dual-feed, just discard it
info!("Discarding initial wake word from prefed audio");
first_recognition = false;
stt::reset_speech_recognizer();
voices::play_reply();
vad_state = VadState::WaitingForVoice;
silence_frames = 0;
start = SystemTime::now();
audio_buffer.clear();
continue;
}
// just wake word, no command - reactivate
info!("Wake word repeated during chaining, reactivating...");
voices::play_reply();
stt::reset_speech_recognizer();
ipc::send(IpcEvent::Listening);
vad_state = VadState::WaitingForVoice;
silence_frames = 0;
start = SystemTime::now();
audio_buffer.clear();
continue;
} else {
// wake word + command in one phrase - execute the command part
info!("Wake word + command during chaining: '{}'", remaining);
recognized_voice = remaining.to_string();
// fall through to command execution below
}
}
first_recognition = false;
// filter activation phrases
// for tbr in config::ASSISTANT_PHRASES_TBR {
// recognized_voice = recognized_voice.replace(tbr, "");
// }
for tbr in config::get_phrases_to_remove(&i18n::get_language()) {
recognized_voice = recognized_voice.replace(tbr, "");
}
recognized_voice = recognized_voice.trim().to_string();
if recognized_voice.len() < 5 {
debug!("Ignoring too short recognition: '{}'", recognized_voice);
continue;
}
if recognized_voice.is_empty() {
continue;
}
// execute command and check if we should chain
let should_chain = execute_command(&recognized_voice, rt);
if should_chain {
// chain: reset and continue listening
info!("Chaining enabled, continuing to listen...");
stt::reset_speech_recognizer();
vad_state = VadState::WaitingForVoice;
silence_frames = 0;
start = SystemTime::now();
audio_buffer.clear();
ipc::send(IpcEvent::Listening);
continue;
} else {
// no chain: return to wake word
info!("No chain, returning to wake word mode.");
return;
}
}
// track silence
if processed.is_voice {
silence_frames = 0;
} else {
silence_frames += 1;
if silence_frames > silence_threshold {
info!("Long silence detected, returning to wake word mode.");
return;
}
}
}
}
// timeout
if let Ok(elapsed) = start.elapsed() {
if elapsed > config::CMS_WAIT_DELAY {
info!("Command timeout, returning to wake word mode.");
return;
}
}
}
}
fn process_text_command(text: &str, rt: &tokio::runtime::Runtime) {
info!("Processing text command: {}", text);
ipc::send(IpcEvent::SpeechRecognized { text: text.to_string() });
let mut filtered = text.to_lowercase();
// for tbr in config::ASSISTANT_PHRASES_TBR {
// filtered = filtered.replace(tbr, "");
// }
for tbr in config::get_phrases_to_remove(&i18n::get_language()) {
filtered = filtered.replace(tbr, "");
}
let filtered = filtered.trim();
if filtered.is_empty() {
ipc::send(IpcEvent::Idle);
return;
}
// text commands never chain
execute_command(filtered, rt);
}
// Execute command, returns true if chaining should continue
fn execute_command(text: &str, rt: &tokio::runtime::Runtime) -> bool {
let commands_list = match COMMANDS_LIST.get() {
Some(c) => c,
None => {
ipc::send(IpcEvent::Error { message: "Commands not loaded".to_string() });
ipc::send(IpcEvent::Idle);
return false;
}
};
let cmd_result = if let Some((intent_id, confidence)) =
rt.block_on(intent::classify(text))
{
info!("Intent recognized: {} (confidence: {:.2})", intent_id, confidence);
intent::get_command_by_intent(commands_list, &intent_id)
} else {
info!("Intent not recognized, trying levenshtein fallback...");
commands::fetch_command(text, commands_list)
};
if let Some((cmd_path, cmd_config)) = cmd_result {
info!("Command found: {:?}", cmd_path);
// extract slots if needed
let extracted_slots = if !cmd_config.slots.is_empty() {
let s = slots::extract(text, &cmd_config.slots);
if !s.is_empty() {
info!("Extracted slots: {:?}", s);
}
Some(s)
} else {
None
};
match commands::execute_command(&cmd_path, &cmd_config, Some(&text), extracted_slots.as_ref()) {
Ok(chain) => {
info!("Command executed successfully");
// voices::play_ok();
voices::play_random_from(cmd_config.get_sounds(&i18n::get_language()).as_slice());
ipc::send(IpcEvent::CommandExecuted {
id: cmd_config.id.clone(),
success: true,
});
ipc::send(IpcEvent::Idle);
return chain; // return chain status from command
}
Err(msg) => {
error!("Error executing command: {}", msg);
voices::play_error();
ipc::send(IpcEvent::CommandExecuted {
id: cmd_config.id.clone(),
success: false,
});
ipc::send(IpcEvent::Error { message: msg.to_string() });
}
}
} else {
info!("No command found for: {}", text);
voices::play_not_found();
ipc::send(IpcEvent::Error {
message: format!("Command not found: {}", text)
});
}
ipc::send(IpcEvent::Idle);
false // no chain on error or not found
}
pub fn close(code: i32) {
info!("Closing application.");
voices::play_goodbye();
ipc::send(IpcEvent::Stopping);
std::process::exit(code);
}

View file

@ -0,0 +1,25 @@
use simple_log::LogConfigBuilder;
use crate::config;
use crate::APP_LOG_DIR;
pub fn init_logging() -> Result<(), String> {
// configure logging
let config = LogConfigBuilder::builder()
.path(format!(
"{}/{}",
APP_LOG_DIR.get().unwrap().display(),
config::LOG_FILE_NAME
))
.size(1 * 100)
.roll_count(10)
.time_format("%Y-%m-%d %H:%M:%S.%f")
.level("debug")?
.output_file()
.output_console()
.build();
simple_log::new(config)?;
Ok(())
}

View file

@ -0,0 +1,173 @@
use jarvis_core::slots;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc;
// include core
use jarvis_core::{
audio, audio_processing, commands, config, db, listener, recorder, stt, intent,
ipc::{self, IpcAction},
i18n, voices, models,
APP_CONFIG_DIR, APP_LOG_DIR, COMMANDS_LIST, DB,
};
// include log
#[macro_use]
extern crate simple_log;
mod log;
// include app
mod app;
// include tray
// @TODO. macOS currently not supported for tray functionality.
#[cfg(not(target_os = "macos"))]
mod tray;
static SHOULD_STOP: AtomicBool = AtomicBool::new(false);
fn main() -> Result<(), String> {
// initialize directories
config::init_dirs()?;
// initialize logging
log::init_logging()?;
// log some base info
info!("Starting Jarvis v{} ...", config::APP_VERSION.unwrap());
info!("Config directory is: {}", APP_CONFIG_DIR.get().unwrap().display());
info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display());
// initialize settings
let settings = db::init();
// set global DB (for core modules that read settings at init time)
DB.set(settings.arc().clone())
.expect("DB already initialized");
// init voices
let voice_id = settings.lock().voice.clone();
let language = settings.lock().language.clone();
if let Err(e) = voices::init(&voice_id, &language) {
warn!("Failed to init voices: {}", e);
}
// init i18n
i18n::init(&settings.lock().language);
// init recorder
if recorder::init().is_err() {
app::close(1);
}
// init models registry (scans available AI models)
if let Err(e) = models::init() {
warn!("Models registry init failed: {}", e);
}
// init stt engine
if stt::init().is_err() {
// @TODO. Allow continuing even without STT, if commands is using keywords or smthng?
app::close(1); // cannot continue without stt
}
// init commands
info!("Initializing commands.");
let cmds = match commands::parse_commands() {
Ok(c) => c,
Err(e) => {
warn!("Failed to parse commands: {}. Starting with empty command list.", e);
Vec::new()
}
};
info!("Commands initialized. Count: {}, List: {:?}", cmds.len(), commands::list_paths(&cmds));
COMMANDS_LIST.set(cmds).unwrap();
// init audio
if audio::init().is_err() {
// @TODO. Allow continuing even without audio?
app::close(1); // cannot continue without audio
}
// init wake-word engine
if let Err(e) = listener::init() {
error!("Wake-word engine init failed: {}", e);
app::close(1);
}
// shared async runtime for intent classification, IPC, etc.
let rt = Arc::new(
tokio::runtime::Runtime::new().expect("Failed to create tokio runtime")
);
// init intent-recognition engine
rt.block_on(async {
if let Err(e) = intent::init(COMMANDS_LIST.get().unwrap()).await {
error!("Failed to initialize intent classifier: {}", e);
app::close(1);
}
});
// init slots parsing engine
slots::init().map_err(|e| error!("Slot extraction init failed: {}", e)).ok();
// init audio processing
info!("Initializing audio processing...");
if let Err(e) = audio_processing::init() {
warn!("Audio processing init failed: {}", e);
}
// init IPC
info!("Initializing IPC...");
ipc::init();
// channel for text commands (manually written in the GUI)
let (text_cmd_tx, text_cmd_rx) = mpsc::channel::<String>();
ipc::set_action_handler(move |action| {
match action {
IpcAction::Stop => {
info!("Received stop command from GUI");
SHOULD_STOP.store(true, Ordering::SeqCst);
}
IpcAction::ReloadCommands => {
info!("Received reload commands request");
// TODO: implement reload
}
IpcAction::SetMuted { muted } => {
info!("Received mute request: {}", muted);
// TODO: implement mute
}
IpcAction::TextCommand { text } => {
info!("Received text command: {}", text);
if let Err(e) = text_cmd_tx.send(text) {
error!("Failed to send text command to app: {}", e);
}
}
IpcAction::Ping => {
// handled internally by server
}
_ => {}
}
});
// start WebSocket server on the shared runtime
let ipc_rt = Arc::clone(&rt);
std::thread::spawn(move || {
ipc_rt.block_on(ipc::start_server());
});
// start the app (in the background thread)
let app_rt = Arc::clone(&rt);
std::thread::spawn(move || {
let _ = app::start(text_cmd_rx, &app_rt);
});
tray::init_blocking(settings);
Ok(())
}
pub fn should_stop() -> bool {
SHOULD_STOP.load(Ordering::SeqCst)
}

View file

@ -0,0 +1,226 @@
mod menu;
use tray_icon::{
menu::MenuEvent,
TrayIconBuilder,
};
use image;
use std::process::Command;
#[cfg(target_os="windows")]
use winit::platform::windows::EventLoopBuilderExtWindows;
use jarvis_core::{config, i18n, voices, ipc::{self, IpcEvent}, SettingsManager};
const TRAY_ICON_BYTES: &[u8] = include_bytes!("../../../resources/icons/32x32.png");
pub fn init_blocking(settings: SettingsManager) {
let icon = load_icon_from_bytes(TRAY_ICON_BYTES);
// build menu with settings submenus
let tray_menu = menu::build(&settings);
let menu::TrayMenu { menu, state: tray_state } = tray_menu;
let _tray_icon = TrayIconBuilder::new()
.with_menu(Box::new(menu))
.with_tooltip(i18n::t("tray-tooltip"))
.with_icon(icon)
.build()
.unwrap();
let menu_channel = MenuEvent::receiver();
#[cfg(target_os = "linux")]
{
gtk::init().unwrap();
glib::timeout_add_local(std::time::Duration::from_millis(100), move || {
if let Ok(event) = menu_channel.try_recv() {
handle_menu_event(&event, &settings, &tray_state);
}
glib::ControlFlow::Continue
});
gtk::main();
}
#[cfg(target_os = "macos")]
{
use winit::event_loop::{EventLoop, ControlFlow};
let event_loop = EventLoop::new().unwrap();
event_loop.run(move |_event, elwt| {
elwt.set_control_flow(ControlFlow::Wait);
if let Ok(event) = menu_channel.try_recv() {
handle_menu_event(&event, &settings, &tray_state);
}
}).unwrap();
}
#[cfg(target_os = "windows")]
{
loop {
if let Ok(event) = menu_channel.try_recv() {
handle_menu_event(&event, &settings, &tray_state);
}
// pump Windows messages
unsafe {
let mut msg: winapi::um::winuser::MSG = std::mem::zeroed();
while winapi::um::winuser::PeekMessageW(
&mut msg,
std::ptr::null_mut(),
0, 0,
winapi::um::winuser::PM_REMOVE
) != 0 {
winapi::um::winuser::TranslateMessage(&msg);
winapi::um::winuser::DispatchMessageW(&msg);
}
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
}
info!("Tray initialized.");
}
fn handle_menu_event(event: &MenuEvent, settings: &SettingsManager, tray_state: &menu::TrayState) {
let id = event.id.0.as_str();
// -- radio group: "set:key:value"
if let Some(rest) = id.strip_prefix("set:") {
if let Some((key, value)) = rest.split_once(':') {
match settings.write(key, value) {
Ok(()) => {
info!("Tray: {} = {}", key, value);
// update check marks in the radio group
for group in &tray_state.radio_groups {
if group.setting_key == key {
group.select(value);
break;
}
}
// apply side effects
match key {
"language" => {
i18n::set_language(value);
}
"assistant_voice" => {
voices::set_current_voice(value);
}
_ => {}
}
}
Err(e) => {
warn!("Tray: failed to set {} = {}: {}", key, value, e);
}
}
return;
}
}
// -- toggle: "toggle:key"
if let Some(key) = id.strip_prefix("toggle:") {
match key {
"gain_normalizer" => {
// CheckMenuItem auto-toggles on click, just read the new state
let new_val = tray_state.gain_toggle.is_checked();
let val_str = if new_val { "true" } else { "false" };
if let Err(e) = settings.write(key, val_str) {
warn!("Tray: failed to toggle {}: {}", key, e);
// revert visual state on error
tray_state.gain_toggle.set_checked(!new_val);
} else {
info!("Tray: {} = {}", key, val_str);
}
}
_ => {}
}
return;
}
// -- action items
match id {
"exit" => std::process::exit(0),
"restart" => {
info!("Restarting from tray menu...");
restart_app();
}
"settings" => {
info!("Opening settings from tray menu...");
open_settings();
}
_ => {}
}
}
// HELPERS
fn load_icon_from_bytes(bytes: &[u8]) -> tray_icon::Icon {
let image = image::load_from_memory(bytes)
.expect("Failed to load icon")
.into_rgba8();
let (width, height) = image.dimensions();
let rgba = image.into_raw();
tray_icon::Icon::from_rgba(rgba, width, height).expect("Failed to create icon")
}
fn restart_app() {
let exe_path = match std::env::current_exe() {
Ok(path) => path,
Err(e) => {
error!("Failed to get executable path: {}", e);
return;
}
};
match Command::new(&exe_path).spawn() {
Ok(_) => {
info!("Spawned new instance, exiting current...");
std::process::exit(0);
}
Err(e) => {
error!("Failed to restart: {}", e);
}
}
}
fn open_settings() {
if ipc::has_clients() {
info!("GUI is connected, sending reveal event");
ipc::send(IpcEvent::RevealWindow);
} else {
info!("GUI not connected, launching jarvis-gui");
launch_gui();
}
}
fn launch_gui() {
let exe_path = match std::env::current_exe() {
Ok(path) => path,
Err(e) => {
error!("Failed to get executable path: {}", e);
return;
}
};
let gui_path = exe_path.parent()
.map(|p| p.join(get_gui_executable_name()))
.unwrap_or_else(|| get_gui_executable_name().into());
info!("Launching GUI: {:?}", gui_path);
match Command::new(&gui_path).spawn() {
Ok(_) => info!("Launched jarvis-gui"),
Err(e) => error!("Failed to launch jarvis-gui: {}", e),
}
}
#[cfg(target_os = "windows")]
fn get_gui_executable_name() -> &'static str {
"jarvis-gui.exe"
}
#[cfg(not(target_os = "windows"))]
fn get_gui_executable_name() -> &'static str {
"jarvis-gui"
}

View file

@ -0,0 +1,182 @@
use tray_icon::menu::{CheckMenuItem, Menu, MenuItem, PredefinedMenuItem, Submenu};
use jarvis_core::{i18n, voices, SettingsManager};
use jarvis_core::config::structs::{WakeWordEngine, NoiseSuppressionBackend};
// RADIO GROUP
// a group of check menu items where only one can be active at a time.
// stores (menu_item, setting_value) pairs.
pub struct RadioGroup {
pub setting_key: String,
pub items: Vec<(CheckMenuItem, String)>,
}
impl RadioGroup {
pub fn select(&self, value: &str) {
for (item, val) in &self.items {
item.set_checked(val == value);
}
}
}
// TRAY MENU STATE
pub struct TrayMenu {
pub menu: Menu,
pub state: TrayState,
}
// holds references to menu items for updating check marks after build
pub struct TrayState {
pub radio_groups: Vec<RadioGroup>,
pub gain_toggle: CheckMenuItem,
}
// BUILD
pub fn build(settings: &SettingsManager) -> TrayMenu {
let menu = Menu::new();
let mut radio_groups = Vec::new();
// -- language submenu
let lang_sub = Submenu::new(i18n::t("tray-language"), true);
let current_lang = settings.read("language").unwrap_or_default();
let mut lang_items = Vec::new();
for &lang in i18n::SUPPORTED_LANGUAGES {
let label = match lang {
"ru" => "Русский",
"en" => "English",
"ua" => "Українська",
_ => lang,
};
let item = CheckMenuItem::with_id(
format!("set:language:{}", lang),
label,
true,
lang == current_lang,
None,
);
let _ = lang_sub.append(&item);
lang_items.push((item, lang.to_string()));
}
radio_groups.push(RadioGroup {
setting_key: "language".to_string(),
items: lang_items,
});
// -- voice submenu
let voice_sub = Submenu::new(i18n::t("tray-voice"), true);
let current_voice = voices::get_current_voice()
.map(|v| v.voice.id.clone())
.unwrap_or_default();
let mut voice_items = Vec::new();
for voice in voices::list_voices() {
let item = CheckMenuItem::with_id(
format!("set:assistant_voice:{}", voice.voice.id),
&voice.voice.name,
true,
voice.voice.id == current_voice,
None,
);
let _ = voice_sub.append(&item);
voice_items.push((item, voice.voice.id.clone()));
}
radio_groups.push(RadioGroup {
setting_key: "assistant_voice".to_string(),
items: voice_items,
});
// -- wake word engine submenu
let ww_sub = Submenu::new(i18n::t("tray-wake-word"), true);
let current_ww = settings.read("selected_wake_word_engine").unwrap_or_default();
let mut ww_items = Vec::new();
for (label, value) in &[("Rustpotter", "Rustpotter"), ("Vosk", "Vosk")] {
let item = CheckMenuItem::with_id(
format!("set:selected_wake_word_engine:{}", value.to_lowercase()),
*label,
true,
current_ww == *label,
None,
);
let _ = ww_sub.append(&item);
ww_items.push((item, value.to_lowercase()));
}
radio_groups.push(RadioGroup {
setting_key: "selected_wake_word_engine".to_string(),
items: ww_items,
});
// -- noise suppression submenu
let ns_sub = Submenu::new(i18n::t("tray-noise-suppression"), true);
let current_ns = settings.read("noise_suppression").unwrap_or_default();
let mut ns_items = Vec::new();
for (label, value) in &[("None", "none"), ("Nnnoiseless", "nnnoiseless")] {
let item = CheckMenuItem::with_id(
format!("set:noise_suppression:{}", value),
*label,
true,
current_ns.to_lowercase() == *value,
None,
);
let _ = ns_sub.append(&item);
ns_items.push((item, value.to_string()));
}
radio_groups.push(RadioGroup {
setting_key: "noise_suppression".to_string(),
items: ns_items,
});
// -- vad submenu
let vad_sub = Submenu::new(i18n::t("tray-vad"), true);
let current_vad = settings.read("vad_backend").unwrap_or_default();
let mut vad_items = Vec::new();
for (label, value) in &[("None", "none"), ("Energy", "energy"), ("Nnnoiseless", "nnnoiseless")] {
let item = CheckMenuItem::with_id(
format!("set:vad_backend:{}", value),
*label,
true,
current_vad == *value,
None,
);
let _ = vad_sub.append(&item);
vad_items.push((item, value.to_string()));
}
radio_groups.push(RadioGroup {
setting_key: "vad_backend".to_string(),
items: vad_items,
});
// -- gain normalizer toggle
let gain_on = settings.read("gain_normalizer")
.map(|v| v == "true")
.unwrap_or(true);
let gain_toggle = CheckMenuItem::with_id(
"toggle:gain_normalizer",
i18n::t("tray-gain-normalizer"),
true,
gain_on,
None,
);
// -- assemble main menu
let _ = menu.append(&lang_sub);
let _ = menu.append(&voice_sub);
let _ = menu.append(&ww_sub);
let _ = menu.append(&ns_sub);
let _ = menu.append(&vad_sub);
let _ = menu.append(&gain_toggle);
let _ = menu.append(&PredefinedMenuItem::separator());
let _ = menu.append(&MenuItem::with_id("restart", i18n::t("tray-restart"), true, None));
let _ = menu.append(&MenuItem::with_id("settings", i18n::t("tray-settings"), true, None));
let _ = menu.append(&MenuItem::with_id("exit", i18n::t("tray-exit"), true, None));
TrayMenu {
menu,
state: TrayState {
radio_groups,
gain_toggle,
},
}
}

View file

@ -0,0 +1,14 @@
[package]
name = "jarvis-cli"
version.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
edition.workspace = true
[dependencies]
jarvis-core = { path = "../jarvis-core", default-features = false, features = ["intent"]}
tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros"] }
log.workspace = true
env_logger = "0.11"
parking_lot.workspace = true

View file

@ -0,0 +1,203 @@
use std::io::{self, Write};
use jarvis_core::{COMMANDS_LIST, DB, JCommandsList, commands, config, db, intent};
fn print_help() {
println!("
--## Jarvis CLI - Testing Tool ##--
Commands:
classify <text> - Test intent classification
execute <text> - Simulate voice input and execute command
list - List all loaded commands
phrases - List all training phrases
hash - Show commands hash
settings - Dump all settings
help - Show this help
exit - Exit the CLI
");
}
fn list_commands(commands: &[JCommandsList]) {
println!("\n[ Loaded Commands ]");
for cmd_list in commands {
println!(" 📁 {}", cmd_list.path.display());
for cmd in &cmd_list.commands {
println!(" ├─ id: {}", cmd.id);
println!(" ├─ type: {}", cmd.cmd_type);
println!(" └─ phrases: {} languages", cmd.phrases.len());
}
}
println!();
}
fn list_phrases(commands: &[JCommandsList]) {
println!("\n[ Training Phrases ]");
for cmd_list in commands {
for cmd in &cmd_list.commands {
println!(" [{}]", cmd.id);
for (lang, phrases) in &cmd.phrases {
println!(" lang: {}", lang);
for phrase in phrases {
println!(" - {}", phrase);
}
}
}
}
println!();
}
async fn classify_text(text: &str) {
match intent::classify(text).await {
Some((intent_id, confidence)) => {
println!(" ✓ Intent: {} (confidence: {:.2}%)", intent_id, confidence * 100.0);
}
None => {
println!(" ✗ No intent matched (below threshold)");
}
}
}
async fn execute_text(commands: &[JCommandsList], text: &str) {
// try intent classification first
if let Some((intent_id, confidence)) = intent::classify(text).await {
println!(" Intent: {} (confidence: {:.2}%)", intent_id, confidence * 100.0);
if let Some((cmd_path, cmd)) = intent::get_command_by_intent(commands, &intent_id) {
println!(" Command: {:?}", cmd_path);
println!(" Type: {}", cmd.cmd_type);
println!(" Executing...");
match commands::execute_command(cmd_path, cmd, Some(text), None) {
Ok(chain) => println!(" ✓ Success (chain: {})", chain),
Err(e) => println!(" ✗ Error: {}", e),
}
return;
}
}
// fallback to levenshtein
println!(" Intent not matched, trying levenshtein fallback...");
if let Some((cmd_path, cmd)) = commands::fetch_command(text, commands) {
println!(" Command: {:?}", cmd_path);
println!(" Type: {}", cmd.cmd_type);
println!(" Executing...");
match commands::execute_command(cmd_path, cmd, Some(text), None) {
Ok(chain) => println!(" ✓ Success (chain: {})", chain),
Err(e) => println!(" ✗ Error: {}", e),
}
} else {
println!(" ✗ No command matched");
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// init logging
env_logger::Builder::from_env(
env_logger::Env::default().default_filter_or("info")
).init();
println!("Jarvis CLI v{}", config::APP_VERSION.unwrap_or("unknown"));
// init dirs
config::init_dirs()?;
// init settings
let settings = db::init();
DB.set(settings.arc().clone())
.expect("DB already initialized");
// parse commands
println!("\n[*] Loading commands...");
let cmds = match commands::parse_commands() {
Ok(c) => {
println!(" Loaded {} command groups", c.len());
c
}
Err(e) => {
println!(" Warning: {}", e);
Vec::new()
}
};
COMMANDS_LIST.set(cmds).expect("Failed to set commands list");
// init intent classifier
println!("[*] Initializing intent classifier...");
match intent::init(COMMANDS_LIST.get().unwrap()).await {
Ok(_) => println!(" Intent classifier ready"),
Err(e) => println!(" Warning: {}", e),
}
// init sound
println!("[*] Initializing audio...");
if let Err(e) = jarvis_core::audio::init() {
println!(" Warning: Audio init failed: {:?}", e);
}
print_help();
// REPL loop
let mut input = String::new();
loop {
print!("jarvis> ");
io::stdout().flush()?;
input.clear();
io::stdin().read_line(&mut input)?;
let input = input.trim();
if input.is_empty() {
continue;
}
let parts: Vec<&str> = input.splitn(2, ' ').collect();
let cmd = parts[0];
let arg = parts.get(1).copied().unwrap_or("");
match cmd {
"exit" | "quit" | "q" => {
println!("Bye!");
break;
}
"help" | "h" | "?" => print_help(),
"list" | "ls" => list_commands(COMMANDS_LIST.get().unwrap()),
"phrases" => list_phrases(COMMANDS_LIST.get().unwrap()),
"hash" => {
let hash = commands::commands_hash(COMMANDS_LIST.get().unwrap());
println!(" Commands hash: {}", hash);
}
"settings" => {
println!("\n[ Current Settings ]");
for (key, val) in settings.dump() {
println!(" {} = {}", key, val);
}
println!();
}
"classify" | "c" => {
if arg.is_empty() {
println!(" Usage: classify <text>");
} else {
classify_text(arg).await;
}
}
"execute" | "exec" | "e" => {
if arg.is_empty() {
println!(" Usage: execute <text>");
} else {
execute_text(COMMANDS_LIST.get().unwrap(), arg).await;
}
}
"reload" => {
println!(" Note: Reload requires app restart (statics can't be reset)");
}
_ => {
// treat unknown commands as text to classify
classify_text(input).await;
}
}
}
Ok(())
}

View file

@ -0,0 +1,64 @@
[package]
name = "jarvis-core"
version.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
edition.workspace = true
[dependencies]
serde.workspace = true
serde_json.workspace = true
serde_yaml.workspace = true
once_cell.workspace = true
log.workspace = true
rand.workspace = true
seqdiff.workspace = true
hound.workspace = true
platform-dirs.workspace = true
rodio.workspace = true
kira.workspace = true
pv_recorder.workspace = true
rustpotter.workspace = true
parking_lot.workspace = true
toml.workspace = true
sha2.workspace = true
nnnoiseless = { workspace = true, optional = true }
tokio-tungstenite = { workspace = true, optional = true }
futures-util = { workspace = true, optional = true }
fluent.workspace = true
fluent-bundle.workspace = true
unic-langid.workspace = true
chrono.workspace = true
sys-locale.workspace = true
# pv_recorder = { workspace = true, optional = true }
vosk = { version = "0.3.1", optional = true }
intent-classifier = { version = "0.1.0", optional = true }
# rustpotter = { workspace = true, optional = true }
tokio = { version = "1", features = ["sync"], optional = true }
mlua = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
tempfile.workspace = true
fastembed = { workspace = true, optional = true }
ort = { workspace = true, optional = true }
ndarray = { workspace = true, optional = true }
tokenizers = { workspace = true, optional = true }
regex = { workspace = true, optional = true }
[target.'cfg(windows)'.dependencies]
winrt-notification = { workspace = true, optional = true }
[features]
default = ["jarvis_app"]
jarvis_app = [
"vosk", "intent-classifier", "fastembed", "tokio", "nnnoiseless", "tokio-tungstenite", "futures-util",
"lua",
"ort", "ndarray", "tokenizers", "regex",]
intent = ["intent-classifier", "tokio"]
lua = ["mlua", "reqwest", "winrt-notification"]
lua_only = ["lua", "tokio"]

View file

@ -0,0 +1,92 @@
mod kira;
mod rodio;
use once_cell::sync::OnceCell;
use std::path::PathBuf;
use crate::config::structs::AudioType;
use crate::{config, DB, SOUND_DIR};
static AUDIO_TYPE: OnceCell<AudioType> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if AUDIO_TYPE.get().is_some() {
return Ok(());
} // already initialized
// set default audio type
// @TODO. Make it configurable?
AUDIO_TYPE.set(config::DEFAULT_AUDIO_TYPE).unwrap();
// load given audio backend
match AUDIO_TYPE.get().unwrap() {
AudioType::Rodio => {
// Init Rodio
info!("Initializing Rodio audio backend.");
match rodio::init() {
Ok(_) => {
info!("Successfully initialized Rodio audio backend.");
}
Err(()) => {
error!("Failed to initialize Rodio audio backend.");
return Err(());
}
}
}
AudioType::Kira => {
// Init Kira
info!("Initializing Kira audio backend.");
match kira::init() {
Ok(_) => {
info!("Successfully initialized Kira audio backend.");
}
Err(_msg) => {
error!("Failed to initialize Kira audio backend.");
return Err(());
}
}
}
}
Ok(())
}
pub fn play_sound(filename: &PathBuf) {
let audio_type = match AUDIO_TYPE.get() {
Some(t) => t,
None => {
warn!("Audio not initialized, cannot play: {}", filename.display());
return;
}
};
info!("Playing {}", filename.display());
match audio_type {
AudioType::Rodio => {
rodio::play_sound(filename, true);
}
AudioType::Kira => kira::play_sound(filename),
}
}
pub fn get_sound_directory() -> Option<PathBuf> {
let db = DB.get()?;
let voice_path = {
let s = db.read();
SOUND_DIR.join(&s.voice)
};
match voice_path.exists() {
true => Some(voice_path),
_ => {
error!("No sounds folder found. Search path - {:?}", voice_path);
None
}
}
}

View file

@ -0,0 +1,62 @@
use once_cell::sync::OnceCell;
use std::path::PathBuf;
use std::sync::Mutex;
// use kira::{
// manager::{backend::DefaultBackend, AudioManager, AudioManagerSettings},
// sound::static_sound::{StaticSoundData, StaticSoundSettings},
// };
use kira::{
AudioManager, AudioManagerSettings, DefaultBackend,
sound::static_sound::StaticSoundData,
};
static MANAGER: OnceCell<Mutex<AudioManager>> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if MANAGER.get().is_some() {
return Ok(());
} // already initialized
// Create an audio manager. This plays sounds and manages resources.
match AudioManager::<DefaultBackend>::new(AudioManagerSettings::default()) {
Ok(manager) => {
// store
MANAGER.set(Mutex::new(manager)).ok();
// success
Ok(())
}
Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg);
// failed
Err(())
}
}
}
// @TODO. Cache sounds in memory? With a pool of a certain size, for instance.
pub fn play_sound(filename: &PathBuf) {
// load the file
match StaticSoundData::from_file(filename) {
Ok(sound_data) => {
// sound_data.duration() can be used in order to sleep, if (for some reason) blocking behaviour is required
// play it (non-blocking)
if let Some(manager) = MANAGER.get() {
if let Ok(mut audio_manager) = manager.lock() {
if let Err(e) = audio_manager.play(sound_data) {
warn!("Failed to play sound: {}", e);
}
}
} else {
warn!("Audio manager not initialized");
}
}
Err(err) => {
warn!("Cannot find sound file: {} (err: {})", filename.display(), err);
}
}
}

View file

@ -0,0 +1,65 @@
/*
Abandoned temporary.
Problems with blocking behaviour.
Possible fixes are running rodio in a separate thread or smthng.
*/
use once_cell::sync::OnceCell;
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
// use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink};
use rodio::{Decoder, OutputStream, Sink};
// static STREAM: OnceCell<OutputStream> = OnceCell::new();
static STREAM_HANDLE: OnceCell<OutputStream> = OnceCell::new();
static SINK: OnceCell<Sink> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if STREAM_HANDLE.get().is_some() {
return Ok(());
} // already initialized
// get output stream handle to the default physical sound device
match rodio::OutputStreamBuilder::open_default_stream() {
Ok(stream_handle) => {
// create sink
let sink = Sink::connect_new(&stream_handle.mixer());
info!("Sink initialized.");
// store
// STREAM.set(_stream).unwrap();
let _ = STREAM_HANDLE.set(stream_handle);
let _ = SINK.set(sink);
// success
Ok(())
}
Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg);
// failed
Err(())
}
}
}
pub fn play_sound(filename: &PathBuf, sleep: bool) {
// Load a sound from a file, using a path relative to Cargo.toml
// let filepath = format!("{PUBLIC_PATH}/sound/{filename}.wav");
let file = BufReader::new(File::open(&filename).unwrap());
// Decode that sound file into a source
let source = Decoder::new(file).unwrap();
// Play the sound directly on the device
// STREAM_HANDLE.get().unwrap().play_raw(source.convert_samples());
SINK.get().unwrap().append(source);
if sleep {
// The sound plays in a separate thread. This call will block the current thread until the sink
// has finished playing all its queued sounds.
SINK.get().unwrap().sleep_until_end();
}
}

View file

@ -0,0 +1,41 @@
use std::collections::VecDeque;
pub struct AudioRingBuffer {
buffer: VecDeque<Vec<i16>>,
max_frames: usize,
}
impl AudioRingBuffer {
// Create buffer that holds `seconds` worth of audio at given frame_size and sample_rate
pub fn new(seconds: f32, frame_size: usize, sample_rate: usize) -> Self {
let frames_per_second = sample_rate / frame_size;
let max_frames = (frames_per_second as f32 * seconds) as usize;
Self {
buffer: VecDeque::with_capacity(max_frames),
max_frames,
}
}
// Push a frame, dropping oldest if full
pub fn push(&mut self, frame: &[i16]) {
if self.buffer.len() >= self.max_frames {
self.buffer.pop_front();
}
self.buffer.push_back(frame.to_vec());
}
// Drain all buffered frames into a single vec
pub fn drain_all(&mut self) -> Vec<Vec<i16>> {
self.buffer.drain(..).collect()
}
// Get frame count
pub fn len(&self) -> usize {
self.buffer.len()
}
pub fn clear(&mut self) {
self.buffer.clear();
}
}

View file

@ -0,0 +1,117 @@
pub mod noise_suppression;
pub mod vad;
pub mod gain_normalizer;
use once_cell::sync::OnceCell;
use parking_lot::Mutex;
use crate::config::structs::NoiseSuppressionBackend;
use crate::DB;
static PROCESSOR: OnceCell<Mutex<AudioProcessor>> = OnceCell::new();
#[derive(Debug, Clone)]
pub struct ProcessedAudio {
pub samples: Vec<i16>,
pub is_voice: bool,
pub vad_confidence: f32,
}
struct AudioProcessor {
has_gain: bool,
has_ns: bool,
}
impl AudioProcessor {
fn new(ns: NoiseSuppressionBackend, gain: bool) -> Self {
noise_suppression::init(ns);
vad::init();
if gain {
gain_normalizer::init();
}
Self {
has_gain: gain,
has_ns: !matches!(ns, NoiseSuppressionBackend::None),
}
}
fn process(&mut self, input: &[i16]) -> ProcessedAudio {
let gained: Vec<i16>;
let after_gain: &[i16] = if self.has_gain {
gained = gain_normalizer::normalize(input);
&gained
} else {
input
};
let suppressed: Vec<i16>;
let after_ns: &[i16] = if self.has_ns {
suppressed = noise_suppression::process(after_gain);
&suppressed
} else {
after_gain
};
let (is_voice, confidence) = vad::detect(after_ns);
ProcessedAudio {
samples: after_ns.to_vec(),
is_voice,
vad_confidence: confidence,
}
}
fn reset(&mut self) {
noise_suppression::reset();
vad::reset();
gain_normalizer::reset();
}
}
pub fn init() -> Result<(), String> {
if PROCESSOR.get().is_some() {
return Ok(());
}
let (ns, gain) = get_settings();
info!("Initializing audio processing: NS={:?}, Gain={}", ns, gain);
let processor = AudioProcessor::new(ns, gain);
PROCESSOR
.set(Mutex::new(processor))
.map_err(|_| "Audio processor already initialized".to_string())?;
info!("Audio processing initialized.");
Ok(())
}
pub fn process(input: &[i16]) -> ProcessedAudio {
match PROCESSOR.get() {
Some(p) => p.lock().process(input),
None => ProcessedAudio {
samples: input.to_vec(),
is_voice: true,
vad_confidence: 1.0,
},
}
}
pub fn reset() {
if let Some(p) = PROCESSOR.get() {
p.lock().reset();
}
}
fn get_settings() -> (NoiseSuppressionBackend, bool) {
match DB.get() {
Some(db) => {
let settings = db.read();
(settings.noise_suppression, settings.gain_normalizer)
}
None => (
crate::config::DEFAULT_NOISE_SUPPRESSION,
crate::config::DEFAULT_GAIN_NORMALIZER,
),
}
}

View file

@ -0,0 +1,28 @@
mod simple;
use once_cell::sync::OnceCell;
use parking_lot::Mutex;
static NORMALIZER: OnceCell<Mutex<simple::GainNormalizer>> = OnceCell::new();
pub fn init() {
if NORMALIZER.get().is_some() {
return;
}
NORMALIZER.set(Mutex::new(simple::GainNormalizer::new())).ok();
info!("Gain normalizer: enabled");
}
pub fn normalize(input: &[i16]) -> Vec<i16> {
match NORMALIZER.get() {
Some(n) => n.lock().normalize(input),
None => input.to_vec(),
}
}
pub fn reset() {
if let Some(n) = NORMALIZER.get() {
n.lock().reset();
}
}

View file

@ -0,0 +1,47 @@
use crate::config;
pub struct GainNormalizer {
current_gain: f32,
}
impl GainNormalizer {
pub fn new() -> Self {
Self { current_gain: 1.0 }
}
pub fn normalize(&mut self, input: &[i16]) -> Vec<i16> {
let rms = self.calculate_rms(input);
if rms < 1.0 {
return input.to_vec();
}
let target_gain = config::GAIN_TARGET_RMS / rms;
let clamped_gain = target_gain.clamp(config::GAIN_MIN, config::GAIN_MAX);
self.current_gain = self.current_gain * 0.9 + clamped_gain * 0.1;
input.iter()
.map(|&s| {
let amplified = (s as f32) * self.current_gain;
amplified.clamp(i16::MIN as f32, i16::MAX as f32) as i16
})
.collect()
}
pub fn reset(&mut self) {
self.current_gain = 1.0;
}
fn calculate_rms(&self, samples: &[i16]) -> f32 {
if samples.is_empty() {
return 0.0;
}
let sum: f64 = samples.iter()
.map(|&s| (s as f64).powi(2))
.sum();
(sum / samples.len() as f64).sqrt() as f32
}
}

View file

@ -0,0 +1,65 @@
mod none;
use once_cell::sync::OnceCell;
use parking_lot::Mutex;
use crate::config::structs::NoiseSuppressionBackend;
static BACKEND: OnceCell<NoiseSuppressionBackend> = OnceCell::new();
#[cfg(feature = "nnnoiseless")]
static NNNOISELESS_STATE: OnceCell<Mutex<crate::models::nnnoiseless::NnnoiselessNS>> = OnceCell::new();
pub fn init(backend: NoiseSuppressionBackend) {
if BACKEND.get().is_some() {
return;
}
// fallback if nnnoiseless not compiled in
#[cfg(not(feature = "nnnoiseless"))]
if matches!(backend, NoiseSuppressionBackend::Nnnoiseless) {
warn!("Nnnoiseless not compiled in, falling back to None");
backend = NoiseSuppressionBackend::None;
}
BACKEND.set(backend).ok();
match backend {
NoiseSuppressionBackend::None => {
info!("Noise suppression: disabled");
}
#[cfg(feature = "nnnoiseless")]
NoiseSuppressionBackend::Nnnoiseless => {
NNNOISELESS_STATE.set(Mutex::new(crate::models::nnnoiseless::NnnoiselessNS::new())).ok();
info!("Noise suppression: Nnnoiseless");
}
#[cfg(not(feature = "nnnoiseless"))]
_ => {}
}
}
pub fn process(input: &[i16]) -> Vec<i16> {
match BACKEND.get() {
#[cfg(feature = "nnnoiseless")]
Some(NoiseSuppressionBackend::Nnnoiseless) => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().process(input)
} else {
none::process(input)
}
}
_ => none::process(input),
}
}
pub fn reset() {
match BACKEND.get() {
#[cfg(feature = "nnnoiseless")]
Some(NoiseSuppressionBackend::Nnnoiseless) => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().reset();
}
}
_ => {}
}
}

View file

@ -0,0 +1,4 @@
// return unprocessed input
pub fn process(input: &[i16]) -> Vec<i16> {
input.to_vec()
}

View file

@ -0,0 +1,72 @@
mod none;
mod energy;
use once_cell::sync::OnceCell;
use parking_lot::Mutex;
use crate::DB;
static BACKEND: OnceCell<String> = OnceCell::new();
#[cfg(feature = "nnnoiseless")]
static NNNOISELESS_STATE: OnceCell<Mutex<crate::models::nnnoiseless::NnnoiselessVAD>> = OnceCell::new();
pub fn init() {
if BACKEND.get().is_some() {
return;
}
let backend = DB.get()
.map(|db| db.read().vad_backend.clone())
.unwrap_or_else(|| "energy".to_string());
BACKEND.set(backend.clone()).ok();
match backend.as_str() {
"none" => {
info!("VAD: disabled");
}
"energy" => {
info!("VAD: Energy-based");
}
#[cfg(feature = "nnnoiseless")]
"nnnoiseless" => {
NNNOISELESS_STATE.set(Mutex::new(crate::models::nnnoiseless::NnnoiselessVAD::new())).ok();
info!("VAD: Nnnoiseless");
}
other => {
warn!("Unknown VAD backend '{}', falling back to energy", other);
// overwrite with energy
// (BACKEND already set, so energy::detect will be used via fallthrough)
}
}
}
// returns (is_voice, confidence)
pub fn detect(input: &[i16]) -> (bool, f32) {
match BACKEND.get().map(|s| s.as_str()) {
Some("none") | None => none::detect(input),
Some("energy") => energy::detect(input),
#[cfg(feature = "nnnoiseless")]
Some("nnnoiseless") => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().detect(input)
} else {
energy::detect(input)
}
}
_ => energy::detect(input),
}
}
pub fn reset() {
match BACKEND.get().map(|s| s.as_str()) {
#[cfg(feature = "nnnoiseless")]
Some("nnnoiseless") => {
if let Some(state) = NNNOISELESS_STATE.get() {
state.lock().reset();
}
}
_ => {}
}
}

View file

@ -0,0 +1,24 @@
use crate::config;
// Simple energy-based VAD
pub fn detect(input: &[i16]) -> (bool, f32) {
let rms = calculate_rms(input);
let is_voice = rms > config::VAD_ENERGY_THRESHOLD;
// normalize confidence to 0-1 range (rough approximation)
let confidence = (rms / (config::VAD_ENERGY_THRESHOLD * 2.0)).min(1.0);
(is_voice, confidence)
}
fn calculate_rms(samples: &[i16]) -> f32 {
if samples.is_empty() {
return 0.0;
}
let sum: f64 = samples.iter()
.map(|&s| (s as f64).powi(2))
.sum();
(sum / samples.len() as f64).sqrt() as f32
}

View file

@ -0,0 +1,4 @@
// Always returns voice detected (no vad)
pub fn detect(_input: &[i16]) -> (bool, f32) {
(true, 1.0)
}

View file

@ -0,0 +1,319 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::fs;
use std::time::Duration;
use std::process::{Child, Command};
use seqdiff::ratio;
mod structs;
pub use structs::*;
use crate::{config, i18n, APP_DIR};
#[cfg(feature = "lua")]
use crate::lua::{self, SandboxLevel, CommandContext};
pub fn parse_commands() -> Result<Vec<JCommandsList>, String> {
let mut commands: Vec<JCommandsList> = Vec::new();
let commands_path = APP_DIR.join(config::COMMANDS_PATH);
let cmd_dirs = fs::read_dir(&commands_path)
.map_err(|e| format!("Error reading commands directory {:?}: {}", commands_path, e))?;
for entry in cmd_dirs.flatten() {
let cmd_path = entry.path();
let toml_file = cmd_path.join("command.toml");
if !toml_file.exists() {
continue;
}
let content = match fs::read_to_string(&toml_file) {
Ok(c) => c,
Err(e) => {
warn!("Failed to read {}: {}", toml_file.display(), e);
continue;
}
};
let file: JCommandsList = match toml::from_str(&content) {
Ok(f) => f,
Err(e) => {
warn!("Failed to parse {}: {}", toml_file.display(), e);
continue;
}
};
commands.push(JCommandsList {
path: cmd_path,
commands: file.commands,
});
}
if commands.is_empty() {
Err("No commands found".into())
} else {
info!("Loaded {} command pack(s)", commands.len());
Ok(commands)
}
}
pub fn commands_hash(commands: &[JCommandsList]) -> String {
use sha2::{Sha256, Digest};
let mut hasher = Sha256::new();
let lang = i18n::get_language();
hasher.update(lang.as_bytes());
hasher.update(b"|");
// collect all command ids and phrases for current language, sorted
let mut all_data: Vec<(&str, _)> = commands.iter()
.flat_map(|ac| ac.commands.iter().map(|c| (c.id.as_str(), c.get_phrases(&lang))))
.collect();
all_data.sort_by_key(|(id, _)| *id);
for (id, phrases) in all_data {
hasher.update(id.as_bytes());
for phrase in phrases.iter() {
hasher.update(phrase.as_bytes());
}
}
format!("{:x}", hasher.finalize())
}
pub fn fetch_command<'a>(
phrase: &str,
commands: &'a [JCommandsList],
) -> Option<(&'a PathBuf, &'a JCommand)> {
let lang = i18n::get_language();
let phrase = phrase.trim().to_lowercase();
if phrase.is_empty() {
return None;
}
let phrase_chars: Vec<char> = phrase.chars().collect();
let phrase_words: Vec<&str> = phrase.split_whitespace().collect();
let mut result: Option<(&PathBuf, &JCommand)> = None;
let mut best_score = config::CMD_RATIO_THRESHOLD;
for cmd_list in commands {
for cmd in &cmd_list.commands {
let cmd_phrases = cmd.get_phrases(&lang);
for cmd_phrase in cmd_phrases.iter() {
let cmd_phrase_lower = cmd_phrase.trim().to_lowercase();
let cmd_phrase_chars: Vec<char> = cmd_phrase_lower.chars().collect();
// character-level similarity
let char_ratio = ratio(&phrase_chars, &cmd_phrase_chars);
// word-level similarity
let cmd_words: Vec<&str> = cmd_phrase_lower.split_whitespace().collect();
let word_score = word_overlap_score(&phrase_words, &cmd_words);
// combined score
let score = (char_ratio * 0.6) + (word_score * 0.4);
// early exit on perfect match
if score >= 99.0 {
debug!("Perfect match: '{}' -> '{}'", phrase, cmd_phrase_lower);
return Some((&cmd_list.path, cmd));
}
if score > best_score {
best_score = score;
result = Some((&cmd_list.path, cmd));
}
}
}
}
if let Some((_, cmd)) = result {
info!("Fuzzy match: '{}' -> cmd '{}' (score: {:.1}%)", phrase, cmd.id, best_score);
} else {
debug!("No match for '{}' (best: {:.1}%)", phrase, best_score);
}
result
}
fn word_overlap_score(input_words: &[&str], cmd_words: &[&str]) -> f64 {
if input_words.is_empty() || cmd_words.is_empty() {
return 0.0;
}
let mut matched = 0.0;
// pre-compute cmd word chars to avoid repeated allocations
let cmd_word_chars: Vec<Vec<char>> = cmd_words
.iter()
.map(|w| w.chars().collect())
.collect();
for input_word in input_words {
let input_chars: Vec<char> = input_word.chars().collect();
let best_word_match = cmd_word_chars
.iter()
.map(|cw| ratio(&input_chars, cw))
.fold(0.0_f64, f64::max);
if best_word_match > 70.0 {
matched += best_word_match / 100.0;
}
}
let max_words = input_words.len().max(cmd_words.len()) as f64;
(matched / max_words) * 100.0
}
pub fn execute_exe(exe: &str, args: &[String]) -> std::io::Result<Child> {
Command::new(exe).args(args).spawn()
}
pub fn execute_cli(cmd: &str, args: &[String]) -> std::io::Result<Child> {
debug!("Spawning: cmd /C {} {:?}", cmd, args);
if cfg!(target_os = "windows") {
Command::new("cmd").arg("/C").arg(cmd).args(args).spawn()
} else {
Command::new("sh").arg("-c").arg(cmd).args(args).spawn()
}
}
pub fn execute_command(cmd_path: &PathBuf, cmd_config: &JCommand, phrase: Option<&str>, slots: Option<&HashMap<String, SlotValue>>) -> Result<bool, String> {
// execute command by the type
match cmd_config.cmd_type.as_str() {
// BRUH
"voice" => Ok(true),
// LUA command
#[cfg(feature = "lua")]
"lua" => {
execute_lua_command(cmd_path, cmd_config, phrase, slots)
}
// AutoHotkey command
// @TODO: Consider adding ahk source files execution?
"ahk" => {
let exe_path_absolute = Path::new(&cmd_config.exe_path);
let exe_path_local = cmd_path.join(&cmd_config.exe_path);
let exe_path = if exe_path_absolute.exists() {
exe_path_absolute
} else {
exe_path_local.as_path()
};
execute_exe(exe_path.to_str().unwrap(), &cmd_config.exe_args)
.map(|_| true)
.map_err(|e| format!("AHK process spawn error: {}", e))
}
// CLI command type
// @TODO: Consider security restrictions
"cli" => {
execute_cli(&cmd_config.cli_cmd, &cmd_config.cli_args)
.map(|_| true)
.map_err(|e| format!("CLI command error: {}", e))
}
// TERMINATOR command (T1000)
"terminate" => {
std::thread::sleep(Duration::from_secs(2));
std::process::exit(0);
}
// STOP CHANING
"stop_chaining" => Ok(false),
// other
_ => {
error!("Command type unknown: {}", cmd_config.cmd_type);
Err(format!("Command type unknown: {}", cmd_config.cmd_type).into())
}
}
}
// look up a command by its ID
pub fn get_command_by_id<'a>(
commands: &'a [JCommandsList],
id: &str,
) -> Option<(&'a PathBuf, &'a JCommand)> {
for cmd_list in commands {
for cmd in &cmd_list.commands {
if cmd.id == id {
return Some((&cmd_list.path, cmd));
}
}
}
None
}
pub fn list_paths(commands: &[JCommandsList]) -> Vec<&Path> {
commands.iter().map(|x| x.path.as_path()).collect()
}
#[cfg(feature = "lua")]
fn execute_lua_command(
cmd_path: &PathBuf,
cmd_config: &JCommand,
phrase: Option<&str>,
slots: Option<&HashMap<String, SlotValue>>
) -> Result<bool, String> {
// get script path
let script_name = if cmd_config.script.is_empty() {
"script.lua"
} else {
&cmd_config.script
};
let script_path = cmd_path.join(script_name);
if !script_path.exists() {
return Err(format!("Lua script not found: {}", script_path.display()));
}
// parse sandbox level
let sandbox = SandboxLevel::from_str(&cmd_config.sandbox);
// create context
let context = CommandContext {
phrase: phrase.unwrap_or("").to_string(),
command_id: cmd_config.id.clone(),
command_path: cmd_path.clone(),
language: i18n::get_language(),
slots: slots.map(|s| s.clone()),
};
// get timeout
let timeout = Duration::from_millis(cmd_config.timeout);
info!("Executing Lua command: {} (sandbox: {:?}, timeout: {:?})",
cmd_config.id, sandbox, timeout);
// execute
match lua::execute(&script_path, context, sandbox, timeout) {
Ok(result) => {
info!("Lua command {} completed (chain: {})", cmd_config.id, result.chain);
Ok(result.chain)
}
Err(e) => {
error!("Lua command {} failed: {}", cmd_config.id, e);
Err(e.to_string())
}
}
}

View file

@ -0,0 +1,179 @@
use std::{collections::HashMap, path::PathBuf, sync::Arc};
use serde::{Serialize, Deserialize};
use parking_lot::RwLock;
#[derive(Serialize, Deserialize, Debug)]
pub struct JCommandsList {
#[serde(skip)]
pub path: PathBuf,
pub commands: Vec<JCommand>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct JCommand {
pub id: String,
// Available command types are: "lua", "ahk", "cli", "voice", "terminate", "stop_chaining"
#[serde(rename = "type")]
pub cmd_type: String,
#[serde(default)]
pub description: String,
// for "ahk" type
#[serde(default)]
pub exe_path: String,
#[serde(default)]
pub exe_args: Vec<String>,
// for "cli" type
#[serde(default)]
pub cli_cmd: String,
#[serde(default)]
pub cli_args: Vec<String>,
// #[serde(default)]
// pub sounds: Vec<String>,
// for "lua" type
#[serde(default)]
pub script: String,
// Lua sandbox level: "minimal", "standard", "full"
// basically this is an access level
#[serde(default)]
pub sandbox: String,
// Script timeout in milliseconds (default 10000 = 10s)
#[serde(default)]
pub timeout: u64,
// Multi-language sounds
#[serde(default)]
pub sounds: HashMap<String, Vec<String>>,
// Multi-language phrases
#[serde(default)]
pub phrases: HashMap<String, Vec<String>>,
// Slot definitions: slot_name -> how to extract it
#[serde(default)]
pub slots: HashMap<String, SlotDefinition>,
// CACHE
#[serde(skip, default)]
sounds_cache: RwLock<HashMap<String, Arc<Vec<String>>>>,
#[serde(skip, default)]
phrases_cache: RwLock<HashMap<String, Arc<Vec<String>>>>,
}
// custom Clone
impl Clone for JCommand {
fn clone(&self) -> Self {
Self {
id: self.id.clone(),
cmd_type: self.cmd_type.clone(),
description: self.description.clone(),
exe_path: self.exe_path.clone(),
exe_args: self.exe_args.clone(),
cli_cmd: self.cli_cmd.clone(),
cli_args: self.cli_args.clone(),
script: self.script.clone(),
sandbox: self.sandbox.clone(),
timeout: self.timeout.clone(),
sounds: self.sounds.clone(),
phrases: self.phrases.clone(),
slots: self.slots.clone(),
// empty caches for cloned instance
sounds_cache: RwLock::new(HashMap::new()),
phrases_cache: RwLock::new(HashMap::new()),
}
}
}
impl JCommand {
// get phrases for current language
pub fn get_phrases(&self, lang: &str) -> Arc<Vec<String>> {
if let Some(cached) = self.phrases_cache.read().get(lang) {
return Arc::clone(cached);
}
let result = Arc::new(self.resolve_localized(&self.phrases, lang));
self.phrases_cache.write().insert(lang.to_string(), Arc::clone(&result));
result
}
// get all phrases (for backwards compat)
pub fn get_all_phrases(&self) -> Vec<String> {
self.phrases.values().flatten().cloned().collect()
}
// get sounds for current language
pub fn get_sounds(&self, lang: &str) -> Arc<Vec<String>> {
if let Some(cached) = self.sounds_cache.read().get(lang) {
return Arc::clone(cached);
}
let result = Arc::new(self.resolve_localized(&self.sounds, lang));
self.sounds_cache.write().insert(lang.to_string(), Arc::clone(&result));
result
}
// get all sounds (for backwards compat)
pub fn get_all_sounds(&self) -> Vec<String> {
self.sounds.values().flatten().cloned().collect()
}
// shared fallback
fn resolve_localized(&self, map: &HashMap<String, Vec<String>>, lang: &str) -> Vec<String> {
// exact match
if let Some(values) = map.get(lang) {
return values.clone();
}
// fallback to "en"
if lang != "en" {
if let Some(values) = map.get("en") {
return values.clone();
}
}
// fallback to first available
map.values().next().cloned().unwrap_or_default()
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct SlotDefinition {
// Entity label for GLiNER (e.g. "city name", "song title", "number")
// This is a free-form description - GLiNER matches it semantically
#[serde(default)]
pub entity: String,
// Optional: fallback context words for template-based extraction
// e.g. ["in", "for", "at"] for a city slot
#[serde(default)]
pub context: Vec<String>,
}
// Extracted slot value passed to commands
#[derive(Debug, Clone, Serialize)]
#[serde(untagged)]
pub enum SlotValue {
Text(String),
Number(f64),
}

View file

@ -0,0 +1,259 @@
pub mod structs;
use structs::AudioType;
use structs::RecorderType;
use structs::SpeechToTextEngine;
use structs::WakeWordEngine;
use once_cell::sync::Lazy;
use std::env;
use std::fs;
use std::path::PathBuf;
use platform_dirs::AppDirs;
#[cfg(feature="jarvis_app")]
use rustpotter::{
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig,
RustpotterConfig, ScoreMode,
};
use crate::config::structs::NoiseSuppressionBackend;
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
#[allow(dead_code)]
pub fn init_dirs() -> Result<(), String> {
// infer app dirs
if APP_DIRS.get().is_some() {
return Ok(());
}
// cache_dir, config_dir, data_dir, state_dir
APP_DIRS
.set(AppDirs::new(Some(BUNDLE_IDENTIFIER), false).unwrap())
.unwrap();
// setup directories
let mut config_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
let mut log_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
// create dirs, if required
if !config_dir.exists() {
if fs::create_dir_all(&config_dir).is_err() {
config_dir = env::current_dir().expect("Cannot infer the config directory");
fs::create_dir_all(&config_dir)
.expect("Cannot create config directory, access denied?");
}
}
if !log_dir.exists() {
if fs::create_dir_all(&log_dir).is_err() {
log_dir = env::current_dir().expect("Cannot infer the log directory");
fs::create_dir_all(&log_dir).expect("Cannot create log directory, access denied?");
}
}
// store inferred paths
APP_CONFIG_DIR.set(config_dir).unwrap();
APP_LOG_DIR.set(log_dir).unwrap();
Ok(())
}
/*
Defaults.
*/
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Vosk;
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
// backend defaults (string IDs)
pub const DEFAULT_INTENT_BACKEND: &str = "intent-classifier";
pub const DEFAULT_SLOTS_BACKEND: &str = "none";
pub const DEFAULT_VAD_BACKEND: &str = "energy";
pub const DEFAULT_VOICE: &str = "jarvis-remaster";
pub const SOUND_PATH: &str = "resources/sound"; // extended from SOUND_DIR (resources/sound)
pub const VOICES_PATH: &str = "voices"; // extended from SOUND_PATH (resources/sound)
pub const BUNDLE_IDENTIFIER: &str = "com.priler.jarvis";
pub const DB_FILE_NAME: &str = "app.db";
pub const LOG_FILE_NAME: &str = "log.txt";
pub const APP_VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION");
pub const AUTHOR_NAME: Option<&str> = option_env!("CARGO_PKG_AUTHORS");
pub const REPOSITORY_LINK: Option<&str> = option_env!("CARGO_PKG_REPOSITORY");
pub const TG_OFFICIAL_LINK: Option<&str> = Some("https://t.me/howdyho_official");
pub const FEEDBACK_LINK: Option<&str> = Some("https://t.me/jarvis_feedback_bot");
pub const SUPPORT_BOOSTY_LINK: Option<&str> = Some("https://boosty.to/howdyho");
pub const SUPPORT_PATREON_LINK: Option<&str> = Some("https://www.patreon.com/c/priler");
/*
Tray.
*/
pub const TRAY_ICON: &str = "32x32.png";
pub const TRAY_TOOLTIP: &str = "Jarvis Voice Assistant";
// RUSPOTTER
pub const RUSPOTTER_MIN_SCORE: f32 = 0.62;
#[cfg(feature="jarvis_app")]
pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
RustpotterConfig {
fmt: AudioFmt::default(),
detector: DetectorConfig {
avg_threshold: 0.,
threshold: 0.5,
min_scores: 15,
score_ref: 0.22,
band_size: 5,
vad_mode: None,
score_mode: ScoreMode::Max,
eager: false,
// comparator_band_size: 5,
// comparator_ref: 0.22
},
filters: FiltersConfig {
gain_normalizer: GainNormalizationConfig {
// enabled: true,
// gain_ref: None,
// min_gain: 0.7,
// max_gain: 1.0,
enabled: false, // disable, now we have separate gain normalizer implementation
gain_ref: None,
min_gain: 0.7,
max_gain: 1.0,
},
band_pass: BandPassConfig {
enabled: true,
low_cutoff: 80.,
high_cutoff: 400.,
},
},
}
});
// PICOVOICE
pub const COMMANDS_PATH: &str = "resources/commands/";
pub const KEYWORDS_PATH: &str = "resources/picovoice/keywords/";
pub const DEFAULT_KEYWORD: &str = "jarvis_windows.ppn";
pub const DEFAULT_SENSITIVITY: f32 = 1.0;
// VOSK
// pub const VOSK_MODEL_PATH: &str = const_concat!(PUBLIC_PATH, "/vosk/model_small");
pub const VOSK_MODELS_PATH: &str = "resources/vosk";
pub const VOSK_MODEL_PATH: &str = "resources/vosk/model_small";
pub const VOSK_FETCH_PHRASE: &str = "джарвис";
pub const VOSK_MIN_RATIO: f64 = 70.0;
// 0.7 lenient, expect false positives
// 0.8 balanced
// 0.9 strict
// etc
pub const VOSK_WAKE_CONFIDENCE: f32 = 0.9;
pub const VOSK_SPEECH_RECOGNIZER_MAX_ALTERNATIVES: u16 = 3;
pub const VOSK_SPEECH_RECOGNIZER_WORDS: bool = false;
pub const VOSK_SPEECH_PARTIAL_WORDS: bool = false;
// IRE (intents recognition)
pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.75;
// embedding classifier
pub const EMBEDDING_MIN_CONFIDENCE: f64 = 0.70;
// AUDIO PROCESSING DEFAULTS
pub const DEFAULT_NOISE_SUPPRESSION: NoiseSuppressionBackend = NoiseSuppressionBackend::None;
pub const DEFAULT_GAIN_NORMALIZER: bool = false;
// VAD settings
pub const VAD_ENERGY_THRESHOLD: f32 = 100.0; // RMS threshold for energy-based VAD
pub const VAD_NNNOISELESS_THRESHOLD: f32 = 0.8; // probability threshold for nnnoiseless
pub const VAD_SILENCE_FRAMES: u32 = 15; // frames of silence before speech end (~480ms)
// gain normalizer settings
pub const GAIN_TARGET_RMS: f32 = 3000.0; // target RMS level
pub const GAIN_MIN: f32 = 0.5; // minimum gain multiplier
pub const GAIN_MAX: f32 = 3.0; // maximum gain multiplier
// nnnoiseless frame size (fixed by library)
pub const NNNOISELESS_FRAME_SIZE: usize = 480;
// LUA
pub const DEFAULT_LUA_SANDBOX: &str = "standard";
pub const DEFAULT_LUA_TIMEOUT: u64 = 10000; // ms
// ETC
pub const CMD_RATIO_THRESHOLD: f64 = 75f64;
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);
// pub const ASSISTANT_GREET_PHRASES: [&str; 3] = ["greet1", "greet2", "greet3"];
// pub const ASSISTANT_PHRASES_TBR: [&str; 17] = [
// "джарвис",
// "сэр",
// "слушаю сэр",
// "всегда к услугам",
// "произнеси",
// "ответь",
// "покажи",
// "скажи",
// "давай",
// "да сэр",
// "к вашим услугам сэр",
// "всегда к вашим услугам сэр",
// "запрос выполнен сэр",
// "выполнен сэр",
// "есть",
// "загружаю сэр",
// "очень тонкое замечание сэр",
// ];
pub fn get_wake_phrases(lang: &str) -> &'static [&'static str] {
match lang {
"ru" => &["джарвис", "джервис", "гарвис", "джарви", "гарви"],
"ua" => &["джарвіс", "джервіс"],
"en" => &["jarvis", "jervis"],
_ => &["jarvis"],
}
}
pub fn get_phrases_to_remove(lang: &str) -> &'static [&'static str] {
match lang {
"ru" => &[
"джарвис", "джервис", "гарвис", "джарви", "гарви",
"сэр", "слушаю сэр", "всегда к услугам",
"произнеси", "ответь", "покажи", "скажи", "давай",
"да сэр", "к вашим услугам сэр", "загружаю сэр",
],
"ua" => &[
"джарвіс", "джервіс", "сер", "слухаю сер", "завжди до послуг",
"скажи", "покажи", "відповідай", "давай",
"так сер", "до ваших послуг сер",
],
"en" => &[
"jarvis", "jervis", "sir", "yes sir", "at your service",
"please", "say", "show", "tell", "hey",
],
_ => &["jarvis"],
}
}
pub fn get_wake_grammar(lang: &str) -> &'static [&'static str] {
match lang {
"ru" => &[
"джарвис", "[unk]", "джон", "джони", "джей",
"джонстон", "привет", "давай",
],
"ua" => &[
"джарвіс", "[unk]", "джон", "джоні", "джей",
"привіт", "давай",
],
"en" => &[
"jarvis", "[unk]", "john", "johnny", "jay",
"hello", "hey", "hi",
],
_ => &["jarvis", "[unk]"],
}
}

View file

@ -0,0 +1,51 @@
use std::fmt;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum WakeWordEngine {
Rustpotter,
Vosk,
Porcupine,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
pub enum NoiseSuppressionBackend {
None,
Nnnoiseless,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum SpeechToTextEngine {
Vosk,
}
#[derive(PartialEq, Debug)]
pub enum RecorderType {
Cpal,
PvRecorder,
PortAudio,
}
#[derive(PartialEq, Debug)]
pub enum AudioType {
Rodio,
Kira,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for SpeechToTextEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl fmt::Display for NoiseSuppressionBackend {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}

View file

@ -0,0 +1,59 @@
pub mod structs;
pub mod manager;
use crate::{config, APP_CONFIG_DIR};
use log::info;
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
pub use manager::SettingsManager;
fn get_db_file_path() -> PathBuf {
PathBuf::from(format!(
"{}/{}",
APP_CONFIG_DIR.get().unwrap().display(),
config::DB_FILE_NAME
))
}
pub fn init_settings() -> structs::Settings {
let db_file_path = get_db_file_path();
info!(
"Loading settings db file located at: {}",
db_file_path.display()
);
if db_file_path.exists() {
if let Ok(db_file) = File::open(&db_file_path) {
let reader = BufReader::new(db_file);
if let Ok(settings) = serde_json::from_reader(reader) {
info!("Settings loaded.");
return settings;
}
}
}
warn!("No settings file found or there was an error parsing it. Creating default struct.");
structs::Settings::default()
}
/// init settings and return a SettingsManager ready to use
pub fn init() -> SettingsManager {
let settings = init_settings();
SettingsManager::new(settings)
}
pub fn save_settings(settings: &structs::Settings) -> Result<(), std::io::Error> {
let db_file_path = get_db_file_path();
std::fs::write(
&db_file_path,
serde_json::to_string_pretty(&settings).unwrap(),
)?;
info!("Settings saved to: {:#}", db_file_path.display());
Ok(())
}

View file

@ -0,0 +1,87 @@
use std::sync::Arc;
use parking_lot::RwLock;
use super::structs::Settings;
use super::save_settings;
// centralized settings manager.
// wraps Arc<RwLock<Settings>> and handles locking + auto-save
// can be used anywhere, ex. from GUI, tray, IPC, CLI, etc.
#[derive(Clone)]
pub struct SettingsManager {
inner: Arc<RwLock<Settings>>,
}
impl SettingsManager {
pub fn new(settings: Settings) -> Self {
Self {
inner: Arc::new(RwLock::new(settings)),
}
}
// wrap an existing Arc<RwLock<Settings>>
pub fn from_arc(arc: Arc<RwLock<Settings>>) -> Self {
Self { inner: arc }
}
// read a setting by key
pub fn read(&self, key: &str) -> Option<String> {
self.inner.read().get(key)
}
// write a setting by key, auto-saves to disk
pub fn write(&self, key: &str, val: &str) -> Result<(), String> {
let snapshot = {
let mut settings = self.inner.write();
settings.set(key, val)?;
settings.clone()
};
save_settings(&snapshot)
.map_err(|e| format!("failed to save settings: {}", e))?;
Ok(())
}
// write multiple settings at once, single save
pub fn write_many(&self, pairs: &[(&str, &str)]) -> Result<(), String> {
let snapshot = {
let mut settings = self.inner.write();
for (key, val) in pairs {
settings.set(key, val)?;
}
settings.clone()
};
save_settings(&snapshot)
.map_err(|e| format!("failed to save settings: {}", e))?;
Ok(())
}
// direct read access to the full Settings struct (for init code that
// needs to read multiple fields at once without key-based access)
pub fn lock(&self) -> parking_lot::RwLockReadGuard<'_, Settings> {
self.inner.read()
}
// direct write access (for bulk operations not covered by set())
pub fn lock_mut(&self) -> parking_lot::RwLockWriteGuard<'_, Settings> {
self.inner.write()
}
// get the underlying Arc
pub fn arc(&self) -> &Arc<RwLock<Settings>> {
&self.inner
}
// dump all settings as key-value pairs (for debugging)
pub fn dump(&self) -> Vec<(String, String)> {
let settings = self.inner.read();
Settings::keys().iter()
.filter_map(|&key| {
settings.get(key).map(|val| (key.to_string(), val))
})
.collect()
}
}

View file

@ -0,0 +1,184 @@
use crate::config;
use serde::{Deserialize, Serialize};
use crate::config::structs::SpeechToTextEngine;
use crate::config::structs::WakeWordEngine;
use crate::config::structs::NoiseSuppressionBackend;
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Settings {
pub microphone: i32,
pub voice: String,
pub wake_word_engine: WakeWordEngine,
// backend selections (string IDs matching model or code backend IDs)
#[serde(default = "default_intent_backend")]
pub intent_backend: String,
#[serde(default = "default_slots_backend")]
pub slots_backend: String,
#[serde(default = "default_vad_backend")]
pub vad_backend: String,
pub gliner_model: String,
pub speech_to_text_engine: SpeechToTextEngine,
pub vosk_model: String,
// audio processing
pub noise_suppression: NoiseSuppressionBackend,
pub gain_normalizer: bool,
#[serde(default = "default_language")]
pub language: String,
pub api_keys: ApiKeys,
}
fn default_intent_backend() -> String { config::DEFAULT_INTENT_BACKEND.to_string() }
fn default_slots_backend() -> String { config::DEFAULT_SLOTS_BACKEND.to_string() }
fn default_vad_backend() -> String { config::DEFAULT_VAD_BACKEND.to_string() }
fn default_language() -> String { crate::i18n::detect_system_language().to_string() }
// ### KEY-VALUE ACCESS
impl Settings {
/// read a setting by key. returns None for unknown keys.
pub fn get(&self, key: &str) -> Option<String> {
match key {
"selected_microphone" => Some(self.microphone.to_string()),
"assistant_voice" => Some(self.voice.clone()),
"selected_wake_word_engine" => Some(format!("{:?}", self.wake_word_engine)),
"intent_backend" => Some(self.intent_backend.clone()),
"slots_backend" => Some(self.slots_backend.clone()),
"vad_backend" => Some(self.vad_backend.clone()),
"selected_gliner_model" => Some(self.gliner_model.clone()),
"selected_vosk_model" => Some(self.vosk_model.clone()),
"speech_to_text_engine" => Some(format!("{:?}", self.speech_to_text_engine)),
"noise_suppression" => Some(format!("{:?}", self.noise_suppression)),
"gain_normalizer" => Some(self.gain_normalizer.to_string()),
"language" => Some(self.language.clone()),
"api_key__picovoice" => Some(self.api_keys.picovoice.clone()),
"api_key__openai" => Some(self.api_keys.openai.clone()),
_ => None,
}
}
/// write a setting by key. returns Err for unknown keys or invalid values.
pub fn set(&mut self, key: &str, val: &str) -> Result<(), String> {
match key {
"selected_microphone" => {
self.microphone = val.parse::<i32>()
.map_err(|_| format!("invalid integer: '{}'", val))?;
}
"assistant_voice" => {
self.voice = val.to_string();
}
"selected_wake_word_engine" => {
self.wake_word_engine = match val.to_lowercase().as_str() {
"rustpotter" => WakeWordEngine::Rustpotter,
"vosk" => WakeWordEngine::Vosk,
"porcupine" => WakeWordEngine::Porcupine,
_ => return Err(format!("unknown wake word engine: '{}'", val)),
};
}
"intent_backend" => {
self.intent_backend = val.to_string();
}
"slots_backend" => {
self.slots_backend = val.to_string();
}
"vad_backend" => {
self.vad_backend = val.to_string();
}
"selected_gliner_model" => {
self.gliner_model = val.to_string();
}
"selected_vosk_model" => {
self.vosk_model = val.to_string();
}
"noise_suppression" => {
self.noise_suppression = match val.to_lowercase().as_str() {
"none" => NoiseSuppressionBackend::None,
"nnnoiseless" => NoiseSuppressionBackend::Nnnoiseless,
_ => return Err(format!("unknown noise suppression backend: '{}'", val)),
};
}
"gain_normalizer" => {
self.gain_normalizer = match val.to_lowercase().as_str() {
"true" => true,
"false" => false,
_ => return Err(format!("expected 'true' or 'false', got: '{}'", val)),
};
}
"language" => {
self.language = val.to_string();
}
"api_key__picovoice" => {
self.api_keys.picovoice = val.to_string();
}
"api_key__openai" => {
self.api_keys.openai = val.to_string();
}
_ => return Err(format!("unknown setting: '{}'", key)),
}
Ok(())
}
/// all valid setting keys (for enumeration, debugging, etc.)
pub fn keys() -> &'static [&'static str] {
&[
"selected_microphone",
"assistant_voice",
"selected_wake_word_engine",
"intent_backend",
"slots_backend",
"vad_backend",
"selected_gliner_model",
"selected_vosk_model",
"speech_to_text_engine",
"noise_suppression",
"gain_normalizer",
"language",
"api_key__picovoice",
"api_key__openai",
]
}
}
// ### DEFAULT
impl Default for Settings {
fn default() -> Settings {
Settings {
microphone: -1,
voice: String::from(""),
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
intent_backend: config::DEFAULT_INTENT_BACKEND.to_string(),
slots_backend: config::DEFAULT_SLOTS_BACKEND.to_string(),
vad_backend: config::DEFAULT_VAD_BACKEND.to_string(),
gliner_model: String::new(),
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
vosk_model: String::from(""),
noise_suppression: config::DEFAULT_NOISE_SUPPRESSION,
gain_normalizer: config::DEFAULT_GAIN_NORMALIZER,
language: crate::i18n::detect_system_language().to_string(),
api_keys: ApiKeys {
picovoice: String::from(""),
openai: String::from(""),
},
}
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct ApiKeys {
pub picovoice: String,
pub openai: String,
}

View file

@ -0,0 +1,202 @@
use fluent_bundle::{FluentBundle, FluentResource, FluentArgs, FluentValue};
use fluent_bundle::concurrent::FluentBundle as ConcurrentFluentBundle;
use once_cell::sync::OnceCell;
use parking_lot::RwLock;
use std::collections::HashMap;
use unic_langid::LanguageIdentifier;
// locale files embedded at compile time
const LOCALE_RU: &str = include_str!("i18n/locales/ru.ftl");
const LOCALE_EN: &str = include_str!("i18n/locales/en.ftl");
const LOCALE_UA: &str = include_str!("i18n/locales/ua.ftl");
pub const SUPPORTED_LANGUAGES: &[&str] = &["ru", "en", "ua"];
pub const DEFAULT_LANGUAGE: &str = "en";
// detect the OS language and map it to a supported language.
// falls back to DEFAULT_LANGUAGE if not supported.
pub fn detect_system_language() -> &'static str {
if let Some(locale) = sys_locale::get_locale() {
// locale can be "en-US", "ru-RU", "uk-UA", etc.
let lang_code = locale.split(&['-', '_'][..]).next().unwrap_or("");
// map OS locale codes to our supported languages
let mapped = match lang_code {
"uk" => "ua", // ISO 639-1 "uk" (ukrainian) -> our "ua"
other => other,
};
if SUPPORTED_LANGUAGES.contains(&mapped) {
info!("Detected system language: {} (from locale '{}')", mapped, locale);
return SUPPORTED_LANGUAGES.iter()
.find(|&&l| l == mapped)
.unwrap();
}
info!("System locale '{}' not supported, using default '{}'", locale, DEFAULT_LANGUAGE);
}
DEFAULT_LANGUAGE
}
// use concurrent bundle (thread-safe)
type Bundle = ConcurrentFluentBundle<FluentResource>;
static BUNDLES: OnceCell<HashMap<String, Bundle>> = OnceCell::new();
static CURRENT_LANG: OnceCell<RwLock<String>> = OnceCell::new();
// Initialize i18n system
pub fn init(lang: &str) {
let bundles = create_bundles();
BUNDLES.set(bundles).ok();
let lang = if SUPPORTED_LANGUAGES.contains(&lang) { lang } else { DEFAULT_LANGUAGE };
CURRENT_LANG.set(RwLock::new(lang.to_string())).ok();
info!("i18n initialized with language: {}", lang);
}
fn create_bundles() -> HashMap<String, Bundle> {
let mut bundles = HashMap::new();
bundles.insert("ru".to_string(), create_bundle("ru", LOCALE_RU));
bundles.insert("en".to_string(), create_bundle("en", LOCALE_EN));
bundles.insert("ua".to_string(), create_bundle("ua", LOCALE_UA));
bundles
}
fn create_bundle(lang: &str, source: &str) -> Bundle {
let langid: LanguageIdentifier = lang.parse().expect("Invalid language identifier");
let mut bundle = ConcurrentFluentBundle::new_concurrent(vec![langid]);
let resource = FluentResource::try_new(source.to_string())
.expect("Failed to parse FTL resource");
bundle.add_resource(resource).expect("Failed to add resource");
bundle
}
// Set current language
pub fn set_language(lang: &str) {
if let Some(current) = CURRENT_LANG.get() {
let lang = if SUPPORTED_LANGUAGES.contains(&lang) { lang } else { DEFAULT_LANGUAGE };
*current.write() = lang.to_string();
info!("Language changed to: {}", lang);
}
}
// Get current language
pub fn get_language() -> String {
CURRENT_LANG.get()
.map(|l| l.read().clone())
.unwrap_or_else(|| DEFAULT_LANGUAGE.to_string())
}
// Translate a key
pub fn t(key: &str) -> String {
t_with_args(key, None)
}
// Translate a key with arguments
pub fn t_with_args(key: &str, args: Option<&FluentArgs>) -> String {
let lang = get_language();
let bundles = match BUNDLES.get() {
Some(b) => b,
None => return key.to_string(),
};
let bundle = match bundles.get(&lang) {
Some(b) => b,
None => bundles.get(DEFAULT_LANGUAGE).unwrap(),
};
let msg = match bundle.get_message(key) {
Some(m) => m,
None => return key.to_string(),
};
let pattern = match msg.value() {
Some(p) => p,
None => return key.to_string(),
};
let mut errors = vec![];
let result = bundle.format_pattern(pattern, args, &mut errors);
if !errors.is_empty() {
warn!("i18n errors for key '{}': {:?}", key, errors);
}
result.to_string()
}
// Translate with a single argument
pub fn t_arg(key: &str, arg_name: &str, arg_value: &str) -> String {
let mut args = FluentArgs::new();
args.set(arg_name, FluentValue::from(arg_value));
t_with_args(key, Some(&args))
}
// Translate with numeric argument
pub fn t_count(key: &str, count: i64) -> String {
let mut args = FluentArgs::new();
args.set("count", FluentValue::from(count));
t_with_args(key, Some(&args))
}
// Get all translations for current language (for frontend)
pub fn get_all_translations() -> HashMap<String, String> {
let lang = get_language();
get_translations_for(&lang)
}
// Get all translations for a specific language
pub fn get_translations_for(lang: &str) -> HashMap<String, String> {
let mut result = HashMap::new();
let bundles = match BUNDLES.get() {
Some(b) => b,
None => return result,
};
let bundle = match bundles.get(lang) {
Some(b) => b,
None => match bundles.get(DEFAULT_LANGUAGE) {
Some(b) => b,
None => return result,
},
};
// get source for this language and extract all keys
let source = match lang {
"ru" => LOCALE_RU,
"en" => LOCALE_EN,
"ua" => LOCALE_UA,
_ => LOCALE_RU,
};
// parse keys from FTL source (lines that have "=" and don't start with "#" or "-")
for line in source.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') || line.starts_with('-') {
continue;
}
if let Some(key) = line.split('=').next() {
let key = key.trim();
if !key.is_empty() && !key.contains(' ') {
if let Some(msg) = bundle.get_message(key) {
if let Some(pattern) = msg.value() {
let mut errors = vec![];
let value = bundle.format_pattern(pattern, None, &mut errors);
result.insert(key.to_string(), value.to_string());
}
}
}
}
}
result
}

View file

@ -0,0 +1,143 @@
# ### APP INFO
app-name = JARVIS
app-description = Voice Assistant
# ### TRAY MENU
tray-restart = Restart
tray-settings = Settings
tray-exit = Exit
tray-tooltip = JARVIS - Voice Assistant
tray-language = Language
tray-voice = Voice
tray-wake-word = Wake Word Engine
tray-noise-suppression = Noise Suppression
tray-vad = Voice Activity Detection
tray-gain-normalizer = Gain Normalizer
# ### HEADER
header-commands = COMMANDS
header-settings = SETTINGS
# ### SEARCH
search-placeholder = Enter a command manually or say «Jarvis» ...
# ### MAIN PAGE
assistant-not-running = ASSISTANT NOT RUNNING
assistant-offline-hint = You can configure it without starting.
btn-start = START
btn-starting = STARTING...
# ### STATUS
status-disconnected = Disconnected
status-standby = Standby
status-listening = Listening...
status-processing = Processing...
# ### STATS
stats-microphone = MICROPHONE
stats-neural-networks = NEURAL NETWORKS
stats-resources = RESOURCES
stats-system-default = System Default
stats-not-selected = Not selected
stats-loading = Loading...
# ### FOOTER
footer-author = Project author
footer-telegram = Our Telegram channel
footer-github = Github repository
footer-support = Support the project on
# ### SETTINGS
settings-title = Settings
settings-general = General
settings-devices = Devices
settings-neural-networks = Neural Networks
settings-audio = Audio
settings-recognition = Recognition
settings-about = About
settings-language = Language
settings-microphone = Microphone
settings-microphone-desc = The assistant will listen to this microphone.
settings-mic-default = Default (System)
settings-voice = Assistant voice
settings-voice-desc =
Not all commands work with all sound packs.
Click to listen the preview of sound.
settings-wake-word-engine = Wake word engine
settings-wake-word-desc = Choose the engine for wake word recognition.
settings-stt-engine = Speech recognition
settings-intent-engine = Intent recognition
settings-intent-engine-desc = Select neural network for command recognition.
settings-noise-suppression = Noise suppression
settings-noise-suppression-desc = Reduces background noise. May negatively affect recognition.
settings-vad = Voice detection (VAD)
settings-vad-desc = Skips silence, saves CPU resources.
settings-gain-normalizer = Gain normalizer
settings-gain-normalizer-desc = Automatically adjusts volume level.
settings-api-keys = API Keys
settings-save = Save
settings-cancel = Cancel
settings-back = Back
settings-enabled = Enabled
settings-disabled = Disabled
# settings - beta notice
settings-beta-title = BETA version!
settings-beta-desc = Some features may not work correctly.
settings-beta-feedback = Report all bugs to
settings-beta-bot = our Telegram bot
settings-open-logs = Open logs folder
# settings - picovoice
settings-attention = Attention!
settings-picovoice-warning = This neural network doesn't work for everyone!
settings-picovoice-waiting = We are waiting for an official patch from the developers.
settings-picovoice-key-desc = Enter your Picovoice key here. It is issued for free upon registration at
settings-picovoice-key = Picovoice Key
# settings - vosk
settings-auto-detect = Auto-detect
settings-vosk-model = Speech recognition model (Vosk)
settings-vosk-model-desc =
Select Vosk model for speech recognition.
You can download models here: https://alphacephei.com/vosk/models
settings-models-not-found = Models not found
settings-models-hint = Place Vosk models in resources/vosk folder
# settings - openai
settings-openai-key = OpenAI Key
settings-openai-not-supported = ChatGPT is not currently supported. It will be added in future updates.
# ### COMMANDS PAGE
commands-title = Commands
commands-search = Search commands...
commands-count = { $count } commands
commands-wip-title = [404] This section is under development!
commands-wip-desc = Here will be a list of commands + full-featured command editor.
commands-wip-follow = Follow updates in
commands-wip-channel = our Telegram channel
# ### ERRORS
error-generic = An error occurred
error-connection = Connection error
error-not-found = Not found
# ### NOTIFICATIONS
notification-saved = Settings saved!
notification-error = Error
notification-assistant-started = Assistant started
notification-assistant-stopped = Assistant stopped
# SLOTS EXTRACTION
settings-slot-engine = Slot extraction
settings-slot-engine-desc = Extract parameters from voice commands (e.g. city name, number).
settings-gliner-model = GLiNER ONNX model
settings-gliner-model-desc =
Select model variant.
Smaller quantized models (int8, uint8) are faster but less accurate.
settings-gliner-models-hint = No GLiNER models found.
# ETC
search-error-not-running = Assistant is not running
search-error-failed = Failed to execute command
settings-no-voices = No voices found

View file

@ -0,0 +1,143 @@
# APP INFO
app-name = JARVIS
app-description = Голосовой ассистент
# TRAY MENU
tray-restart = Перезапустить
tray-settings = Настройки
tray-exit = Выход
tray-tooltip = JARVIS - Голосовой ассистент
tray-language = Язык
tray-voice = Голос
tray-wake-word = Движок wake-word
tray-noise-suppression = Шумоподавление
tray-vad = Детекция голоса (VAD)
tray-gain-normalizer = Нормализация громкости
# HEADER
header-commands = КОМАНДЫ
header-settings = НАСТРОЙКИ
# SEARCH
search-placeholder = Введите команду вручную или произнесите «Джарвис» ...
# MAIN PAGE
assistant-not-running = АССИСТЕНТ НЕ ЗАПУЩЕН
assistant-offline-hint = Настроить его можно не запуская.
btn-start = ЗАПУСТИТЬ
btn-starting = ЗАПУСК...
# STATUS
status-disconnected = Отключен
status-standby = Ожидание
status-listening = Слушаю...
status-processing = Обработка...
# STATS
stats-microphone = МИКРОФОН
stats-neural-networks = НЕЙРОСЕТИ
stats-resources = РЕСУРСЫ
stats-system-default = Системный
stats-not-selected = Не выбран
stats-loading = Загрузка...
# FOOTER
footer-author = Автор проекта
footer-telegram = Наш телеграм канал
footer-github = Github репозиторий проекта
footer-support = Поддержать проект на
# SETTINGS
settings-title = Настройки
settings-general = Основные
settings-devices = Устройства
settings-neural-networks = Нейросети
settings-audio = Аудио
settings-recognition = Распознавание
settings-about = О программе
settings-language = Язык
settings-microphone = Микрофон
settings-microphone-desc = Его будет слушать ассистент.
settings-mic-default = По умолчанию (Система)
settings-voice = Голос ассистента
settings-voice-desc =
Не все команды работают со всеми звуковыми пакетами.
Кликните, чтобы прослушать как звучит голос.
settings-wake-word-engine = Движок активации
settings-wake-word-desc = Выберите нейросеть для распознавания активационной фразы.
settings-stt-engine = Распознавание речи
settings-intent-engine = Определение намерения
settings-intent-engine-desc = Выберите нейросеть для распознавания команд.
settings-noise-suppression = Шумоподавление
settings-noise-suppression-desc = Уменьшает фоновый шум. Может негативно влиять на распознавание.
settings-vad = Определение голоса (VAD)
settings-vad-desc = Пропускает тишину, экономит ресурсы CPU.
settings-gain-normalizer = Нормализация громкости
settings-gain-normalizer-desc = Автоматически регулирует уровень громкости.
settings-api-keys = API Ключи
settings-save = Сохранить
settings-cancel = Отмена
settings-back = Назад
settings-enabled = Включено
settings-disabled = Отключено
# settings - beta notice
settings-beta-title = БЕТА версия!
settings-beta-desc = Часть функций может работать некорректно.
settings-beta-feedback = Сообщайте обо всех найденных багах в
settings-beta-bot = наш телеграм бот
settings-open-logs = Открыть папку с логами
# settings - picovoice
settings-attention = Внимание!
settings-picovoice-warning = Эта нейросеть работает не у всех!
settings-picovoice-waiting = Мы ждем официального патча от разработчиков.
settings-picovoice-key-desc = Введите сюда свой ключ Picovoice. Он выдается бесплатно при регистрации в
settings-picovoice-key = Ключ Picovoice
# settings - vosk
settings-auto-detect = Авто-определение
settings-vosk-model = Модель распознавания речи (Vosk)
settings-vosk-model-desc =
Выберите модель Vosk для распознавания речи.
Вы можете скачать модели здесь: https://alphacephei.com/vosk/models
settings-models-not-found = Модели не найдены
settings-models-hint = Поместите модели Vosk в папку resources/vosk
# settings - openai
settings-openai-key = Ключ OpenAI
settings-openai-not-supported = В данный момент ChatGPT не поддерживается. Он будет добавлен в ближайших обновлениях.
# COMMANDS PAGE
commands-title = Команды
commands-search = Поиск команд...
commands-count = { $count } команд
commands-wip-title = [404] Этот раздел еще находится в разработке!
commands-wip-desc = Тут будет список команд + полноценный редактор команд.
commands-wip-follow = Следите за обновлениями в
commands-wip-channel = нашем телеграм канале
# ERRORS
error-generic = Произошла ошибка
error-connection = Ошибка подключения
error-not-found = Не найдено
# NOTIFICATIONS
notification-saved = Настройки сохранены!
notification-error = Ошибка
notification-assistant-started = Ассистент запущен
notification-assistant-stopped = Ассистент остановлен
# SLOTS EXTRACTION
settings-slot-engine = Извлечение параметров
settings-slot-engine-desc = Извлекает параметры из голосовых команд (напр. название города, число).
settings-gliner-model = Модель GLiNER ONNX
settings-gliner-model-desc =
Выберите вариант модели.
Квантизированные модели (int8, uint8) быстрее, но менее точны.
settings-gliner-models-hint = Модели GLiNER не найдены.
# ETC
search-error-not-running = Ассистент не запущен
search-error-failed = Не удалось выполнить команду
settings-no-voices = Голоса не найдены

View file

@ -0,0 +1,143 @@
# ### APP INFO
app-name = JARVIS
app-description = Голосовий асистент
# ### TRAY MENU
tray-restart = Перезапустити
tray-settings = Налаштування
tray-exit = Вихід
tray-tooltip = JARVIS - Голосовий асистент
tray-language = Мова
tray-voice = Голос
tray-wake-word = Рушій детекції
tray-noise-suppression = Шумозаглушення
tray-vad = Детекцiя голосу (VAD)
tray-gain-normalizer = Нормалізація гучності
# ### HEADER
header-commands = КОМАНДИ
header-settings = НАЛАШТУВАННЯ
# ### SEARCH
search-placeholder = Введіть команду вручну або скажіть «Джарвіс» ...
# ### MAIN PAGE
assistant-not-running = АСИСТЕНТ НЕ ЗАПУЩЕНО
assistant-offline-hint = Налаштувати його можна не запускаючи.
btn-start = ЗАПУСТИТИ
btn-starting = ЗАПУСК...
# ### STATUS
status-disconnected = Відключено
status-standby = Очікування
status-listening = Слухаю...
status-processing = Обробка...
# ### STATS
stats-microphone = МІКРОФОН
stats-neural-networks = НЕЙРОМЕРЕЖІ
stats-resources = РЕСУРСИ
stats-system-default = Системний
stats-not-selected = Не вибрано
stats-loading = Завантаження...
# ### FOOTER
footer-author = Автор проєкту
footer-telegram = Наш телеграм канал
footer-github = Github репозиторій проєкту
footer-support = Підтримати проєкт на
# ### SETTINGS
settings-title = Налаштування
settings-general = Основні
settings-devices = Пристрої
settings-neural-networks = Нейромережі
settings-audio = Аудіо
settings-recognition = Розпізнавання
settings-about = Про програму
settings-language = Мова
settings-microphone = Мікрофон
settings-microphone-desc = Його буде слухати асистент.
settings-mic-default = За замовчуванням (Система)
settings-voice = Голос асистента
settings-voice-desc =
Не всі команди працюють з усіма звуковими пакетами.
Натисніть, щоб прослухати як звучить голос.
settings-wake-word-engine = Рушій активації
settings-wake-word-desc = Виберіть нейромережу для розпізнавання активаційної фрази.
settings-stt-engine = Розпізнавання мовлення
settings-intent-engine = Визначення наміру
settings-intent-engine-desc = Виберіть нейромережу для розпізнавання команд.
settings-noise-suppression = Шумозаглушення
settings-noise-suppression-desc = Зменшує фоновий шум. Може негативно впливати на розпізнавання.
settings-vad = Визначення голосу (VAD)
settings-vad-desc = Пропускає тишу, економить ресурси CPU.
settings-gain-normalizer = Нормалізація гучності
settings-gain-normalizer-desc = Автоматично регулює рівень гучності.
settings-api-keys = API Ключі
settings-save = Зберегти
settings-cancel = Скасувати
settings-back = Назад
settings-enabled = Увімкнено
settings-disabled = Вимкнено
# settings - beta notice
settings-beta-title = БЕТА версія!
settings-beta-desc = Частина функцій може працювати некоректно.
settings-beta-feedback = Повідомляйте про всі знайдені баги в
settings-beta-bot = наш телеграм бот
settings-open-logs = Відкрити папку з логами
# settings - picovoice
settings-attention = Увага!
settings-picovoice-warning = Ця нейромережа працює не у всіх!
settings-picovoice-waiting = Ми чекаємо офіційного патча від розробників.
settings-picovoice-key-desc = Введіть сюди свій ключ Picovoice. Він видається безкоштовно при реєстрації в
settings-picovoice-key = Ключ Picovoice
# settings - vosk
settings-auto-detect = Авто-визначення
settings-vosk-model = Модель розпізнавання мовлення (Vosk)
settings-vosk-model-desc =
Виберіть модель Vosk для розпізнавання мовлення.
Ви можете завантажити моделі тут: https://alphacephei.com/vosk/models
settings-models-not-found = Моделі не знайдено
settings-models-hint = Помістіть моделі Vosk в папку resources/vosk
# settings - openai
settings-openai-key = Ключ OpenAI
settings-openai-not-supported = Наразі ChatGPT не підтримується. Він буде доданий у наступних оновленнях.
# ### COMMANDS PAGE
commands-title = Команди
commands-search = Пошук команд...
commands-count = { $count } команд
commands-wip-title = [404] Цей розділ ще в розробці!
commands-wip-desc = Тут буде список команд + повноцінний редактор команд.
commands-wip-follow = Слідкуйте за оновленнями в
commands-wip-channel = нашому телеграм каналі
# ### ERRORS
error-generic = Сталася помилка
error-connection = Помилка підключення
error-not-found = Не знайдено
# ### NOTIFICATIONS
notification-saved = Налаштування збережено!
notification-error = Помилка
notification-assistant-started = Асистент запущено
notification-assistant-stopped = Асистент зупинено
# SLOTS EXTRACTION
settings-slot-engine = Витяг параметрів
settings-slot-engine-desc = Витягує параметри з голосових команд (напр. назва міста, число).
settings-gliner-model = Модель GLiNER ONNX
settings-gliner-model-desc =
Оберіть варіант моделі.
Квантизовані моделі (int8, uint8) швидші, але менш точні.
settings-gliner-models-hint = Моделі GLiNER не знайдено.
# ETC
search-error-not-running = Асистент не запущено
search-error-failed = Не вдалося виконати команду
settings-no-voices = Голоси не знайдено

View file

@ -0,0 +1,89 @@
mod intentclassifier;
mod embeddingclassifier;
use std::path::PathBuf;
use crate::{commands::{self, JCommandsList, JCommand}, config, models};
use once_cell::sync::OnceCell;
use crate::DB;
static BACKEND: OnceCell<String> = OnceCell::new();
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
if BACKEND.get().is_some() {
return Ok(());
}
let backend = DB.get().unwrap().read().intent_backend.clone();
BACKEND.set(backend.clone()).map_err(|_| "Backend already set")?;
match backend.as_str() {
"none" => {
info!("Intent recognition disabled");
}
"intent-classifier" => {
info!("Initializing IntentClassifier backend.");
intentclassifier::init(&commands).await?;
info!("IntentClassifier backend initialized.");
}
// any other value is treated as a model ID for embedding classification
model_id => {
info!("Initializing EmbeddingClassifier with model '{}'.", model_id);
let model = models::embedding::load(models::registry(), model_id)?;
embeddingclassifier::init_with_model(model, &commands)?;
info!("EmbeddingClassifier backend initialized.");
}
}
Ok(())
}
pub async fn classify(text: &str) -> Option<(String, f64)> {
match BACKEND.get()?.as_str() {
"none" => None,
"intent-classifier" => {
match intentclassifier::classify(text).await {
Ok(prediction) => {
let confidence = prediction.confidence.value();
if confidence >= config::INTENT_CLASSIFIER_MIN_CONFIDENCE {
Some((prediction.intent.to_string(), confidence))
} else {
None
}
}
Err(e) => {
error!("Intent classification error: {}", e);
None
}
}
}
_ => {
match embeddingclassifier::classify(text) {
Ok((intent_id, confidence)) => {
if confidence >= config::EMBEDDING_MIN_CONFIDENCE {
Some((intent_id, confidence))
} else {
None
}
}
Err(e) => {
error!("Embedding classification error: {}", e);
None
}
}
}
}
}
// unified command lookup by intent ID - works for all backends
pub fn get_command_by_intent<'a>(
commands: &'a [JCommandsList],
intent_id: &str,
) -> Option<(&'a PathBuf, &'a JCommand)> {
if matches!(BACKEND.get().map(|s| s.as_str()), Some("none")) {
return None;
}
commands::get_command_by_id(commands, intent_id)
}

View file

@ -0,0 +1,195 @@
use std::path::PathBuf;
use std::sync::Arc;
use std::fs;
use once_cell::sync::OnceCell;
use crate::commands::JCommandsList;
use crate::i18n;
use crate::APP_CONFIG_DIR;
use crate::models::embedding::EmbeddingModel;
// no outer Mutex needed - state is immutable after init.
// the embedding model has its own internal Mutex.
static CLASSIFIER: OnceCell<EmbeddingClassifierState> = OnceCell::new();
struct IntentVector {
id: String,
vector: Vec<f32>,
}
struct EmbeddingClassifierState {
model: Arc<EmbeddingModel>,
intents: Vec<IntentVector>,
}
// model is Arc (Send+Sync), intents are read-only after init
unsafe impl Send for EmbeddingClassifierState {}
unsafe impl Sync for EmbeddingClassifierState {}
const CACHE_FILE: &str = "embedding_intents.json";
const HASH_FILE: &str = "embedding_hash.txt";
// init with a model loaded through the registry
pub fn init_with_model(model: Arc<EmbeddingModel>, commands: &[JCommandsList]) -> Result<(), String> {
if CLASSIFIER.get().is_some() {
return Ok(());
}
info!("Initializing embedding classifier...");
let current_hash = crate::commands::commands_hash(commands);
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
let hash_path = config_dir.join(HASH_FILE);
let cache_path = config_dir.join(CACHE_FILE);
// check if cached vectors are still valid
let should_retrain = if hash_path.exists() && cache_path.exists() {
let stored_hash = fs::read_to_string(&hash_path).unwrap_or_default();
stored_hash.trim() != current_hash
} else {
true
};
let intents = if should_retrain {
info!("Building intent vectors from commands...");
let intents = build_intent_vectors(&model, commands)?;
// cache to disk
if let Ok(json) = serde_json::to_string(&intents_to_cache(&intents)) {
let _ = fs::write(&cache_path, json);
let _ = fs::write(&hash_path, &current_hash);
info!("Intent vectors cached");
}
intents
} else {
info!("Loading cached intent vectors...");
load_cached_intents(&cache_path)?
};
info!("Embedding classifier ready with {} intents", intents.len());
CLASSIFIER.set(EmbeddingClassifierState { model, intents })
.map_err(|_| "Classifier already set".to_string())?;
Ok(())
}
fn build_intent_vectors(
model: &EmbeddingModel,
commands: &[JCommandsList],
) -> Result<Vec<IntentVector>, String> {
let lang = i18n::get_language();
let mut intents = Vec::new();
for cmd_list in commands {
for cmd in &cmd_list.commands {
let phrases = cmd.get_phrases(&lang);
if phrases.is_empty() {
continue;
}
let texts: Vec<&str> = phrases.iter().map(|s| s.as_str()).collect();
let embeddings = model.embedding.lock().embed(texts, None)
.map_err(|e| format!("Embedding failed for '{}': {}", cmd.id, e))?;
// average all phrase vectors into one intent vector
let dim = embeddings[0].len();
let mut avg = vec![0.0f32; dim];
for emb in &embeddings {
for (i, val) in emb.iter().enumerate() {
avg[i] += val;
}
}
let count = embeddings.len() as f32;
for val in &mut avg {
*val /= count;
}
// normalize
let norm: f32 = avg.iter().map(|v| v * v).sum::<f32>().sqrt();
if norm > 0.0 {
for val in &mut avg {
*val /= norm;
}
}
intents.push(IntentVector {
id: cmd.id.clone(),
vector: avg,
});
}
}
Ok(intents)
}
pub fn classify(text: &str) -> Result<(String, f64), String> {
let state = CLASSIFIER.get().ok_or("Classifier not initialized")?;
// only the embedding model needs locking, intents are read-only
let embeddings = state.model.embedding.lock().embed(vec![text], None)
.map_err(|e| format!("Failed to embed query: {}", e))?;
let mut query_vec = embeddings.into_iter().next()
.ok_or("Empty embedding result")?;
// normalize query
let norm: f32 = query_vec.iter().map(|v| v * v).sum::<f32>().sqrt();
if norm > 0.0 {
for val in &mut query_vec {
*val /= norm;
}
}
// cosine similarity - track index, clone only the winner
let mut best_idx: usize = 0;
let mut best_score: f64 = -1.0;
for (i, intent) in state.intents.iter().enumerate() {
let score: f64 = query_vec.iter()
.zip(intent.vector.iter())
.map(|(a, b)| (*a as f64) * (*b as f64))
.sum();
if score > best_score {
best_score = score;
best_idx = i;
}
}
let best_id = state.intents[best_idx].id.clone();
debug!("Embedding classify: '{}' -> '{}' ({:.2}%)", text, best_id, best_score * 100.0);
Ok((best_id, best_score))
}
#[derive(serde::Serialize, serde::Deserialize)]
struct CachedIntent {
id: String,
vector: Vec<f32>,
}
fn intents_to_cache(intents: &[IntentVector]) -> Vec<CachedIntent> {
intents.iter().map(|i| CachedIntent {
id: i.id.clone(),
vector: i.vector.clone(),
}).collect()
}
fn load_cached_intents(path: &PathBuf) -> Result<Vec<IntentVector>, String> {
let json = fs::read_to_string(path)
.map_err(|e| format!("Failed to read cache: {}", e))?;
let cached: Vec<CachedIntent> = serde_json::from_str(&json)
.map_err(|e| format!("Failed to parse cache: {}", e))?;
Ok(cached.into_iter().map(|c| IntentVector {
id: c.id,
vector: c.vector,
}).collect())
}

View file

@ -0,0 +1,96 @@
use intent_classifier::{
IntentPrediction, IntentError,
TrainingExample, TrainingSource, IntentId
};
use std::sync::Arc;
use std::fs;
use crate::commands::{self, JCommandsList};
use crate::models;
use crate::models::intent_classifier::IntentClassifierModel;
use crate::{APP_CONFIG_DIR, i18n};
use once_cell::sync::OnceCell;
static MODEL: OnceCell<Arc<IntentClassifierModel>> = OnceCell::new();
const TRAINING_CACHE_FILE: &str = "intent_training.json";
const COMMANDS_HASH_FILE: &str = "commands_hash.txt";
pub async fn init(commands: &[JCommandsList]) -> Result<(), String> {
let current_hash = commands::commands_hash(&commands);
let model = models::intent_classifier::load(models::registry(), "intent-classifier").await?;
// check if we can use cached training data
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
let hash_path = config_dir.join(COMMANDS_HASH_FILE);
let cache_path = config_dir.join(TRAINING_CACHE_FILE);
let should_retrain = if hash_path.exists() && cache_path.exists() {
let stored_hash = fs::read_to_string(&hash_path).unwrap_or_default();
stored_hash.trim() != current_hash
} else {
true
};
if should_retrain {
info!("Training intent classifier with {} commands...", commands.len());
train_classifier(&model.classifier, &commands).await?;
if let Ok(export) = model.classifier.export_training_data().await {
let _ = fs::write(&cache_path, export);
let _ = fs::write(&hash_path, &current_hash);
info!("Training data cached.");
}
} else {
info!("Loading cached training data...");
if let Ok(data) = fs::read_to_string(&cache_path) {
model.classifier.import_training_data(&data).await
.map_err(|e| format!("Failed to import training data: {}", e))?;
}
}
MODEL.set(model).map_err(|_| "Model already set")?;
Ok(())
}
pub async fn classify(text: &str) -> Result<IntentPrediction, IntentError> {
let model = MODEL.get().expect("IntentClassifier not initialized");
model.classifier.predict_intent(text).await
}
async fn train_classifier(
classifier: &intent_classifier::IntentClassifier,
commands: &[JCommandsList]
) -> Result<(), String> {
let lang = i18n::get_language();
info!("Training intent classifier for language: {}", lang);
let mut total_examples = 0;
for assistant_cmd in commands {
for cmd in &assistant_cmd.commands {
let phrases = cmd.get_phrases(&lang);
for phrase in phrases.iter() {
let example = TrainingExample {
text: phrase.clone(),
intent: IntentId::from(cmd.id.as_str()),
confidence: 1.0,
source: TrainingSource::Programmatic,
};
classifier.add_training_example(example).await
.map_err(|e| format!("Failed to add training example: {}", e))?;
total_examples += 1;
}
}
}
info!("Added {} training examples for language '{}'", total_examples, lang);
Ok(())
}

View file

@ -0,0 +1,5 @@
mod events;
mod server;
pub use events::{IpcAction, IpcEvent};
pub use server::{init, send, set_action_handler, start_server, has_clients, IPC_ADDR, IPC_PORT};

View file

@ -0,0 +1,56 @@
use serde::{Deserialize, Serialize};
// Events sent from jarvis-app to GUI
#[derive(Clone, Debug, Serialize)]
#[serde(tag = "event", rename_all = "snake_case")]
pub enum IpcEvent {
// Wake word detected, starting to listen
WakeWordDetected,
// Actively listening for command
Listening,
// Speech recognized
SpeechRecognized { text: String },
// Command was executed
CommandExecuted { id: String, success: bool },
// Returned to idle state
Idle,
// Error occurred
Error { message: String },
// App started
Started,
// App is shutting down
Stopping,
// Pong response
Pong,
// request GUI to reveal/focus window
RevealWindow,
}
// Actions sent from GUI to jarvis-app
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "action", rename_all = "snake_case")]
pub enum IpcAction {
// Request graceful shutdown
Stop,
// Reload commands from disk
ReloadCommands,
// Ping to check connection
Ping,
// Mute/unmute listening
SetMuted { muted: bool },
// Execute text command
TextCommand { text: String },
}

View file

@ -0,0 +1,198 @@
use std::net::SocketAddr;
use std::sync::Arc;
use futures_util::{SinkExt, StreamExt};
use once_cell::sync::OnceCell;
use parking_lot::RwLock;
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::broadcast;
use tokio_tungstenite::{accept_async, tungstenite::Message};
use super::events::{IpcAction, IpcEvent};
pub const IPC_PORT: u16 = 9712;
pub const IPC_ADDR: &str = "127.0.0.1";
static BROADCAST_TX: OnceCell<broadcast::Sender<IpcEvent>> = OnceCell::new();
static ACTION_HANDLER: OnceCell<Arc<RwLock<Option<Box<dyn Fn(IpcAction) + Send + Sync>>>>> = OnceCell::new();
// Initialize the IPC broadcast channel
pub fn init() -> broadcast::Sender<IpcEvent> {
if let Some(tx) = BROADCAST_TX.get() {
return tx.clone();
}
let (tx, _) = broadcast::channel::<IpcEvent>(32);
BROADCAST_TX.set(tx.clone()).ok();
ACTION_HANDLER.set(Arc::new(RwLock::new(None))).ok();
info!("IPC: Broadcast channel initialized");
tx
}
// Send event to all connected clients
pub fn send(event: IpcEvent) {
if let Some(tx) = BROADCAST_TX.get() {
match tx.send(event.clone()) {
Ok(n) => {
if n > 0 {
debug!("IPC: Sent {:?} to {} client(s)", event, n);
}
}
Err(_) => {
// no receivers, that's fine
}
}
}
}
// Register handler for incoming actions from GUI
pub fn set_action_handler<F>(handler: F)
where
F: Fn(IpcAction) + Send + Sync + 'static,
{
if let Some(h) = ACTION_HANDLER.get() {
*h.write() = Some(Box::new(handler));
}
}
fn handle_action(action: IpcAction) {
info!("IPC: Received action {:?}", action);
// handle ping internally
if matches!(action, IpcAction::Ping) {
send(IpcEvent::Pong);
return;
}
// forward to registered handler
if let Some(handler_lock) = ACTION_HANDLER.get() {
let handler = handler_lock.read();
if let Some(ref h) = *handler {
h(action);
}
}
}
// Start the WebSocket server (blocking)
pub async fn start_server() {
let addr = format!("{}:{}", IPC_ADDR, IPC_PORT);
let socket_addr: SocketAddr = addr.parse().expect("Invalid IPC address");
let listener = match TcpListener::bind(&socket_addr).await {
Ok(l) => {
info!("IPC: WebSocket server listening on ws://{}", addr);
l
}
Err(e) => {
error!("IPC: Failed to bind to {}: {}", addr, e);
return;
}
};
// notify that we're ready
send(IpcEvent::Started);
while let Ok((stream, peer_addr)) = listener.accept().await {
info!("IPC: Client connecting from {}", peer_addr);
let rx = BROADCAST_TX
.get()
.map(|tx| tx.subscribe())
.expect("IPC not initialized");
tokio::spawn(handle_client(stream, peer_addr, rx));
}
}
async fn handle_client(
stream: TcpStream,
peer_addr: SocketAddr,
mut event_rx: broadcast::Receiver<IpcEvent>,
) {
let ws_stream = match accept_async(stream).await {
Ok(ws) => {
info!("IPC: Client connected: {}", peer_addr);
ws
}
Err(e) => {
error!("IPC: WebSocket handshake failed for {}: {}", peer_addr, e);
return;
}
};
let (mut ws_tx, mut ws_rx) = ws_stream.split();
loop {
tokio::select! {
// forward events to client
event_result = event_rx.recv() => {
match event_result {
Ok(event) => {
let json = match serde_json::to_string(&event) {
Ok(j) => j,
Err(e) => {
error!("IPC: Failed to serialize event: {}", e);
continue;
}
};
if ws_tx.send(Message::Text(json.into())).await.is_err() {
info!("IPC: Client {} disconnected (send failed)", peer_addr);
break;
}
}
Err(broadcast::error::RecvError::Lagged(n)) => {
warn!("IPC: Client {} lagged {} events", peer_addr, n);
}
Err(broadcast::error::RecvError::Closed) => {
info!("IPC: Broadcast channel closed");
break;
}
}
}
// receive messages from client
msg_result = ws_rx.next() => {
match msg_result {
Some(Ok(Message::Text(text))) => {
match serde_json::from_str::<IpcAction>(&text) {
Ok(action) => handle_action(action),
Err(e) => {
warn!("IPC: Invalid action from {}: {} ({})", peer_addr, text, e);
}
}
}
Some(Ok(Message::Ping(data))) => {
if ws_tx.send(Message::Pong(data)).await.is_err() {
break;
}
}
Some(Ok(Message::Close(_))) => {
info!("IPC: Client {} sent close frame", peer_addr);
break;
}
Some(Err(e)) => {
error!("IPC: Error receiving from {}: {}", peer_addr, e);
break;
}
None => {
info!("IPC: Client {} stream ended", peer_addr);
break;
}
_ => {}
}
}
}
}
info!("IPC: Client disconnected: {}", peer_addr);
}
pub fn has_clients() -> bool {
if let Some(tx) = BROADCAST_TX.get() {
tx.receiver_count() > 0
} else {
false
}
}

View file

@ -0,0 +1,72 @@
use once_cell::sync::{Lazy, OnceCell};
use parking_lot::RwLock;
use std::{sync::Arc};
use platform_dirs::AppDirs;
use std::path::PathBuf;
#[macro_use]
extern crate log;
pub mod time;
pub mod audio;
pub mod commands;
pub mod config;
pub mod db;
pub mod i18n;
#[cfg(feature = "jarvis_app")]
pub mod listener;
pub mod recorder;
#[cfg(feature = "jarvis_app")]
pub mod stt;
#[cfg(feature = "intent")]
pub mod intent;
#[cfg(feature = "jarvis_app")]
pub mod slots;
pub mod models;
// re-exported from models/
pub use models::vosk_models;
pub use models::gliner_models;
#[cfg(feature = "jarvis_app")]
pub mod audio_processing;
#[cfg(feature = "jarvis_app")]
pub mod ipc;
pub mod voices;
pub mod audio_buffer;
#[cfg(feature = "lua")]
pub mod lua;
// shared statics
// pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| std::env::current_dir().unwrap());
pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| {
std::env::current_exe()
.ok()
.and_then(|p| p.parent().map(|p| p.to_path_buf()))
.unwrap_or_else(|| std::env::current_dir().unwrap())
});
pub static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| APP_DIR.clone().join(config::SOUND_PATH));
pub static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
pub static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
pub static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
pub static DB: OnceCell<Arc<RwLock<db::structs::Settings>>> = OnceCell::new();
pub static COMMANDS_LIST: OnceCell<Vec<JCommandsList>> = OnceCell::new();
// re-exports
pub use commands::JCommandsList;
pub use config::structs::*;
pub use db::structs::Settings;
pub use db::SettingsManager;
// use crate::commands::{JComandsList, JCommand};

View file

@ -0,0 +1,46 @@
mod rustpotter;
mod vosk;
use once_cell::sync::OnceCell;
use crate::config::structs::WakeWordEngine;
use crate::DB;
static WAKE_WORD_ENGINE: OnceCell<WakeWordEngine> = OnceCell::new();
pub fn init() -> Result<(), String> {
if WAKE_WORD_ENGINE.get().is_some() {
return Ok(());
}
let engine = DB.get().unwrap().read().wake_word_engine;
WAKE_WORD_ENGINE.set(engine)
.map_err(|_| "Wake word engine already set".to_string())?;
match engine {
WakeWordEngine::Porcupine => {
Err("Porcupine wake-word engine is not supported".to_string())
}
WakeWordEngine::Rustpotter => {
info!("Initializing Rustpotter wake-word engine.");
rustpotter::init()
.map_err(|_| "Failed to init Rustpotter".to_string())
}
WakeWordEngine::Vosk => {
info!("Initializing Vosk as wake-word engine.");
warn!("Using Vosk as wake-word engine is highly not recommended, because it's very slow for this task.");
vosk::init()
.map_err(|_| "Failed to init Vosk wake-word".to_string())
}
}
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
match WAKE_WORD_ENGINE.get()? {
WakeWordEngine::Porcupine => None,
WakeWordEngine::Rustpotter => rustpotter::data_callback(frame_buffer),
WakeWordEngine::Vosk => vosk::data_callback(frame_buffer),
}
}

View file

@ -0,0 +1,56 @@
use std::path::Path;
use once_cell::sync::OnceCell;
use porcupine::{Porcupine, PorcupineBuilder};
use crate::config;
use crate::DB;
// store porcupine instance
static PORCUPINE: OnceCell<Porcupine> = OnceCell::new();
pub fn init() -> Result<(), ()> {
let picovoice_api_key: String;
// retrieve picovoice api key
picovoice_api_key = DB.get().unwrap().api_keys.picovoice.clone();
if picovoice_api_key.trim().is_empty() {
warn!("Picovoice API key is not set.");
return Err(());
}
// create porcupine instance with the given API key
match PorcupineBuilder::new_with_keyword_paths(
picovoice_api_key,
&[Path::new(config::KEYWORDS_PATH).join(config::DEFAULT_KEYWORD)],
)
.sensitivities(&[config::DEFAULT_SENSITIVITY]) // set sensitivity
.init()
{
Ok(pinstance) => {
// success
info!("Porcupine successfully initialized with the given API key.");
// store
PORCUPINE.set(pinstance);
}
Err(msg) => {
error!("Porcupine failed to initialize, either API key is not valid or there is no internet connection.");
error!("Error details: {}", msg);
return Err(());
}
}
Ok(())
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) {
if keyword_index >= 0 {
return Some(keyword_index);
}
}
None
}

View file

@ -0,0 +1,69 @@
use std::sync::Mutex;
use once_cell::sync::OnceCell;
use rustpotter::Rustpotter;
use crate::config;
// store rustpotter instance
static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
pub fn init() -> Result<(), ()> {
let rustpotter_config = config::RUSTPOTTER_DEFAULT_CONFIG;
// create rustpotter instance
match Rustpotter::new(&rustpotter_config) {
Ok(mut rinstance) => {
// success
// wake word files list
// @TODO. Make it configurable via GUI for custom user voice.
let rustpotter_wake_word_files: [&str; 1] = [
"resources/rustpotter/jarvis-default.rpw",
// "rustpotter/jarvis-community-1.rpw",
// "rustpotter/jarvis-community-2.rpw",
// "rustpotter/jarvis-community-3.rpw",
// "rustpotter/jarvis-community-4.rpw",
// "rustpotter/jarvis-community-5.rpw",
];
// load wake word files
for rpw in rustpotter_wake_word_files {
// @TODO: Change wakeword key to something else?
if let Err(e) = rinstance.add_wakeword_from_file(rpw, rpw) {
error!("Failed to load wakeword file '{}': {}", rpw, e);
}
}
// store
let _ = RUSTPOTTER.set(Mutex::new(rinstance));
}
Err(msg) => {
error!("Rustpotter failed to initialize.\nError details: {}", msg);
return Err(());
}
}
Ok(())
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
let mut lock = RUSTPOTTER.get().unwrap().lock();
let rustpotter = lock.as_mut().unwrap();
// let detection = rustpotter.process_samples(frame_buffer.to_vec()); // @TODO. Temp crutch. Fix optimization issue, frame_buffer should not be copied to a new vector!
let detection = rustpotter.process_samples(frame_buffer);
// info!("Ruspotter data callback");
if let Some(detection) = detection {
if detection.score > config::RUSPOTTER_MIN_SCORE {
info!("Rustpotter detection info:\n{:?}", detection);
return Some(0);
} else {
info!("Rustpotter detection info:\n{:?}", detection)
}
}
None
}

View file

@ -0,0 +1,76 @@
use crate::{config, stt, i18n};
pub fn init() -> Result<(), ()> {
Ok(()) // nothing to init for Vosk
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
if let Some((recognized, _confidence)) = stt::recognize_wake_word(frame_buffer) {
let recognized = recognized.trim().to_lowercase();
// skip unknown/empty
if recognized.is_empty() || recognized == "[unk]" {
return None;
}
info!("Wake word candidate: '{}'", recognized);
// language-specific wake phrase
let lang = i18n::get_language();
let wake_phrases = config::get_wake_phrases(&lang);
// verify with seqdiff ratio
for word in recognized.split_whitespace() {
if word == "[unk]" {
continue;
}
let word_chars: Vec<char> = word.chars().collect();
for wake_phrase in wake_phrases {
let wake_chars: Vec<char> = wake_phrase.chars().collect();
let similarity = seqdiff::ratio(&wake_chars, &word_chars);
if similarity >= config::VOSK_MIN_RATIO {
info!("Wake word match: '{}' ~ '{}' ({:.1}%)", word, wake_phrase, similarity);
return Some(0);
}
}
}
// info!("Similarity: {:.1}% ('{}' vs '{}')", similarity, recognized, config::VOSK_FETCH_PHRASE);
}
None
}
// @TODO. Make it better somehow (more accurate or with higher sensitivity).
// pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
// // recognize & convert to sequence
// let recognized_phrase = stt::recognize(&frame_buffer, true).unwrap_or("".into());
// if !recognized_phrase.trim().is_empty() {
// info!("Vosk wake-word debug info:");
// info!("rec: {}", recognized_phrase);
// let recognized_phrases = recognized_phrase.split_whitespace();
// for phrase in recognized_phrases {
// let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
// // compare
// let compare_ratio = seqdiff::ratio(
// &config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(),
// &recognized_phrase_chars,
// );
// info!("og phrase: {:?}", &config::VOSK_FETCH_PHRASE);
// info!("recognized phrase: {:?}", &recognized_phrase_chars);
// info!("compare ratio: {}", compare_ratio);
// if compare_ratio >= config::VOSK_MIN_RATIO {
// info!("Phrase activated.");
// return Some(0);
// }
// }
// }
// None
// }

View file

@ -0,0 +1,28 @@
mod engine;
mod sandbox;
mod error;
mod api;
mod structs;
pub use structs::*;
pub use engine::LuaEngine;
pub use sandbox::SandboxLevel;
pub use error::LuaError;
use std::path::PathBuf;
use std::time::Duration;
#[cfg(test)]
mod tests;
// Execute a Lua command script
pub fn execute(
script_path: &PathBuf,
context: CommandContext,
sandbox: SandboxLevel,
timeout: Duration,
) -> Result<CommandResult, LuaError> {
let engine = LuaEngine::new(sandbox)?;
engine.execute(script_path, context, timeout)
}

View file

@ -0,0 +1,7 @@
pub mod core;
pub mod audio;
pub mod context;
pub mod http;
pub mod fs;
pub mod state;
pub mod system;

View file

@ -0,0 +1,37 @@
// QUICK HELP ON HOW TO ADD NEW LUA API MODULE
//
// # 1. DEFINE NEW API MODULE FILE
use mlua::{Lua, Table};
use crate::lua::error::LuaError;
pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> {
let mymodule = lua.create_table()?;
// add functions
let my_fn = lua.create_function(|_, arg: String| {
// implementation
Ok(format!("Result: {}", arg))
})?;
mymodule.set("my_function", my_fn)?;
jarvis.set("mymodule", mymodule)?;
Ok(())
}
// # 2. ADD NEW API MODULE TO mod.rs
pub mod mymodule;
// # 3. REGISTER NEW MODULE IN engine.rs
// in register_api()
api::mymodule::register(&self.lua, &jarvis)?;
// # 4. YOU CAN ALSO DEFINE ASYNC FUNCTIONS INSTEAD
let async_fn = lua.create_async_function(|_, url: String| async move {
let response = reqwest::get(&url).await
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
Ok(response.text().await.unwrap_or_default())
})?;

View file

@ -0,0 +1,77 @@
// Audio Lua API: something sound related, apparently :3
use mlua::{Lua, Table};
use crate::voices::{self, Reaction};
pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> {
let audio = lua.create_table()?;
// jarvis.audio.play(reaction)
// reactions: "ok", "reply", "greet", "not_found", "error", "goodbye", "thanks"
let play_fn = lua.create_function(|_, reaction: String| {
let reaction_enum = match reaction.to_lowercase().as_str() {
"ok" => Reaction::Ok,
"reply" => Reaction::Reply,
"greet" => Reaction::Greet,
"not_found" => Reaction::NotFound,
"error" => Reaction::Error,
"goodbye" => Reaction::Goodbye,
"thanks" => Reaction::Thanks,
// "joke" => Reaction::Joke, NO PUN INTENDED :3
_ => {
log::warn!("[Lua] Unknown reaction: {}", reaction);
return Ok(false);
}
};
voices::play(reaction_enum);
Ok(true)
})?;
audio.set("play", play_fn)?;
// jarvis.audio.play_ok()
let play_ok_fn = lua.create_function(|_, ()| {
voices::play_ok();
Ok(())
})?;
audio.set("play_ok", play_ok_fn)?;
// jarvis.audio.play_reply()
let play_reply_fn = lua.create_function(|_, ()| {
voices::play_reply();
Ok(())
})?;
audio.set("play_reply", play_reply_fn)?;
// jarvis.audio.play_error()
let play_error_fn = lua.create_function(|_, ()| {
voices::play_error();
Ok(())
})?;
audio.set("play_error", play_error_fn)?;
// jarvis.audio.play_not_found()
let play_not_found_fn = lua.create_function(|_, ()| {
voices::play_not_found();
Ok(())
})?;
audio.set("play_not_found", play_not_found_fn)?;
// jarvis.audio.play_greet()
let play_greet_fn = lua.create_function(|_, ()| {
voices::play_greet();
Ok(())
})?;
audio.set("play_greet", play_greet_fn)?;
// jarvis.audio.play_goodbye()
let play_goodbye_fn = lua.create_function(|_, ()| {
voices::play_goodbye();
Ok(())
})?;
audio.set("play_goodbye", play_goodbye_fn)?;
jarvis.set("audio", audio)?;
Ok(())
}

View file

@ -0,0 +1,45 @@
// Context Lua API: read-only command context
use mlua::{Lua, Table};
use crate::lua::{CommandContext};
use crate::commands::SlotValue;
pub fn register(lua: &Lua, jarvis: &Table, ctx: &CommandContext) -> mlua::Result<()> {
let context = lua.create_table()?;
// read-only context values
context.set("phrase", ctx.phrase.clone())?;
context.set("command_id", ctx.command_id.clone())?;
context.set("command_path", ctx.command_path.to_string_lossy().to_string())?;
context.set("language", ctx.language.clone())?;
// time info
let time = lua.create_table()?;
let now = chrono::Local::now();
time.set("year", now.format("%Y").to_string())?;
time.set("month", now.format("%m").to_string())?;
time.set("day", now.format("%d").to_string())?;
time.set("hour", now.format("%H").to_string())?;
time.set("minute", now.format("%M").to_string())?;
time.set("second", now.format("%S").to_string())?;
time.set("weekday", now.format("%A").to_string())?;
time.set("timestamp", now.timestamp())?;
context.set("time", time)?;
// slots
let slots_table = lua.create_table()?;
if let Some(ref slots) = ctx.slots {
for (name, value) in slots {
match value {
SlotValue::Text(t) => slots_table.set(name.as_str(), t.as_str())?,
SlotValue::Number(n) => slots_table.set(name.as_str(), *n)?,
}
}
}
context.set("slots", slots_table)?;
jarvis.set("context", context)?;
Ok(())
}

View file

@ -0,0 +1,50 @@
// Core Lua API: log, sleep, print, etc.
use mlua::{Lua, Table, MultiValue};
pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> {
// @ jarvis.log(level, message)
// log something
let log_fn = lua.create_function(|_, (level, message): (String, String)| {
match level.to_lowercase().as_str() {
"debug" => log::debug!("[Lua] {}", message),
"info" => log::info!("[Lua] {}", message),
"warn" => log::warn!("[Lua] {}", message),
"error" => log::error!("[Lua] {}", message),
_ => log::info!("[Lua] {}", message),
}
Ok(())
})?;
jarvis.set("log", log_fn)?;
// @ jarvis.print(...)
// simple print
let print_fn = lua.create_function(|_, args: MultiValue| {
let parts: Vec<String> = args.iter()
.map(|v| format!("{:?}", v))
.collect();
log::info!("[Lua] {}", parts.join(" "));
Ok(())
})?;
jarvis.set("print", print_fn)?;
// @ jarvis.sleep(ms)
// ..zZZ
let sleep_fn = lua.create_function(|_, ms: u64| {
std::thread::sleep(std::time::Duration::from_millis(ms));
Ok(())
})?;
jarvis.set("sleep", sleep_fn)?;
// @ jarvis.speak(text)
// @TODO: update when TTS will be implemented
let speak_fn = lua.create_function(|_, text: String| {
log::info!("[Lua] SPEAK: {}", text);
// pass
Ok(())
})?;
jarvis.set("speak", speak_fn)?;
Ok(())
}

View file

@ -0,0 +1,212 @@
// File System Lua API: read, write, list (sandboxed)
use mlua::{Lua, Table};
use std::path::{Path, PathBuf};
use std::fs;
use crate::lua::sandbox::SandboxLevel;
pub fn register(
lua: &Lua,
jarvis: &Table,
command_path: &PathBuf,
sandbox: SandboxLevel,
) -> mlua::Result<()> {
let fs_table = lua.create_table()?;
let cmd_path = command_path.clone();
let sandbox_level = sandbox;
// jarvis.fs.read(path)
let cmd_path_read = cmd_path.clone();
let read_fn = lua.create_function(move |_, path: String| {
let full_path = resolve_path(&cmd_path_read, &path, sandbox_level)?;
fs::read_to_string(&full_path)
.map_err(|e| mlua::Error::runtime(format!("Read error: {}", e)))
})?;
fs_table.set("read", read_fn)?;
// jarvis.fs.read_bytes(path)
let cmd_path_read_bytes = cmd_path.clone();
let read_bytes_fn = lua.create_function(move |lua, path: String| {
let full_path = resolve_path(&cmd_path_read_bytes, &path, sandbox_level)?;
let bytes = fs::read(&full_path)
.map_err(|e| mlua::Error::runtime(format!("Read error: {}", e)))?;
Ok(lua.create_string(&bytes)?)
})?;
fs_table.set("read_bytes", read_bytes_fn)?;
// jarvis.fs.write(path, content)
let cmd_path_write = cmd_path.clone();
let write_fn = lua.create_function(move |_, (path, content): (String, String)| {
if !sandbox_level.allows_fs_write() {
return Err(mlua::Error::runtime("Write not allowed in this sandbox"));
}
let full_path = resolve_path(&cmd_path_write, &path, sandbox_level)?;
// ensure parent directory exists
if let Some(parent) = full_path.parent() {
fs::create_dir_all(parent)
.map_err(|e| mlua::Error::runtime(format!("Create dir error: {}", e)))?;
}
fs::write(&full_path, content)
.map_err(|e| mlua::Error::runtime(format!("Write error: {}", e)))?;
Ok(true)
})?;
fs_table.set("write", write_fn)?;
// jarvis.fs.append(path, content)
let cmd_path_append = cmd_path.clone();
let append_fn = lua.create_function(move |_, (path, content): (String, String)| {
if !sandbox_level.allows_fs_write() {
return Err(mlua::Error::runtime("Write not allowed in this sandbox"));
}
let full_path = resolve_path(&cmd_path_append, &path, sandbox_level)?;
use std::io::Write;
let mut file = fs::OpenOptions::new()
.create(true)
.append(true)
.open(&full_path)
.map_err(|e| mlua::Error::runtime(format!("Open error: {}", e)))?;
file.write_all(content.as_bytes())
.map_err(|e| mlua::Error::runtime(format!("Write error: {}", e)))?;
Ok(true)
})?;
fs_table.set("append", append_fn)?;
// jarvis.fs.exists(path)
let cmd_path_exists = cmd_path.clone();
let exists_fn = lua.create_function(move |_, path: String| {
let full_path = resolve_path(&cmd_path_exists, &path, sandbox_level)?;
Ok(full_path.exists())
})?;
fs_table.set("exists", exists_fn)?;
// jarvis.fs.is_file(path)
let cmd_path_is_file = cmd_path.clone();
let is_file_fn = lua.create_function(move |_, path: String| {
let full_path = resolve_path(&cmd_path_is_file, &path, sandbox_level)?;
Ok(full_path.is_file())
})?;
fs_table.set("is_file", is_file_fn)?;
// jarvis.fs.is_dir(path)
let cmd_path_is_dir = cmd_path.clone();
let is_dir_fn = lua.create_function(move |_, path: String| {
let full_path = resolve_path(&cmd_path_is_dir, &path, sandbox_level)?;
Ok(full_path.is_dir())
})?;
fs_table.set("is_dir", is_dir_fn)?;
// jarvis.fs.list(path)
let cmd_path_list = cmd_path.clone();
let list_fn = lua.create_function(move |lua, path: Option<String>| {
let full_path = if let Some(p) = path {
resolve_path(&cmd_path_list, &p, sandbox_level)?
} else {
cmd_path_list.clone()
};
let result = lua.create_table()?;
let entries = fs::read_dir(&full_path)
.map_err(|e| mlua::Error::runtime(format!("List error: {}", e)))?;
let mut idx = 1;
for entry in entries {
if let Ok(entry) = entry {
let item = lua.create_table()?;
item.set("name", entry.file_name().to_string_lossy().to_string())?;
item.set("path", entry.path().to_string_lossy().to_string())?;
item.set("is_file", entry.path().is_file())?;
item.set("is_dir", entry.path().is_dir())?;
result.set(idx, item)?;
idx += 1;
}
}
Ok(result)
})?;
fs_table.set("list", list_fn)?;
// jarvis.fs.mkdir(path)
let cmd_path_mkdir = cmd_path.clone();
let mkdir_fn = lua.create_function(move |_, path: String| {
if !sandbox_level.allows_fs_write() {
return Err(mlua::Error::runtime("Write not allowed in this sandbox"));
}
let full_path = resolve_path(&cmd_path_mkdir, &path, sandbox_level)?;
fs::create_dir_all(&full_path)
.map_err(|e| mlua::Error::runtime(format!("Mkdir error: {}", e)))?;
Ok(true)
})?;
fs_table.set("mkdir", mkdir_fn)?;
// jarvis.fs.remove(path)
let cmd_path_remove = cmd_path.clone();
let remove_fn = lua.create_function(move |_, path: String| {
if !sandbox_level.allows_fs_write() {
return Err(mlua::Error::runtime("Write not allowed in this sandbox"));
}
let full_path = resolve_path(&cmd_path_remove, &path, sandbox_level)?;
if full_path.is_dir() {
fs::remove_dir_all(&full_path)
.map_err(|e| mlua::Error::runtime(format!("Remove error: {}", e)))?;
} else {
fs::remove_file(&full_path)
.map_err(|e| mlua::Error::runtime(format!("Remove error: {}", e)))?;
}
Ok(true)
})?;
fs_table.set("remove", remove_fn)?;
jarvis.set("fs", fs_table)?;
Ok(())
}
// Resolve path relative to command folder, with sandbox checks
fn resolve_path(command_path: &PathBuf, path: &str, sandbox: SandboxLevel) -> mlua::Result<PathBuf> {
let path = Path::new(path);
// if absolute path, check sandbox allows it
if path.is_absolute() {
if !sandbox.allows_expanded_paths() {
return Err(mlua::Error::runtime("Absolute paths not allowed in this sandbox"));
}
return Ok(path.to_path_buf());
}
// relative path - resolve against command folder
let resolved = command_path.join(path);
// canonicalize to resolve ../ etc and check it's still within command folder
let canonical = resolved.canonicalize()
.unwrap_or_else(|_| resolved.clone());
let cmd_canonical = command_path.canonicalize()
.unwrap_or_else(|_| command_path.clone());
if !sandbox.allows_expanded_paths() && !canonical.starts_with(&cmd_canonical) {
return Err(mlua::Error::runtime("Path escapes command folder"));
}
Ok(resolved)
}

View file

@ -0,0 +1,226 @@
// HTTP Lua API: GET, POST, JSON requests
use mlua::{Lua, Table, Value};
use std::collections::HashMap;
use std::time::Duration;
pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> {
let http = lua.create_table()?;
// jarvis.http.get(url, headers?)
let get_fn = lua.create_function(|lua, (url, headers): (String, Option<Table>)| {
http_request(lua, "GET", &url, None, headers)
})?;
http.set("get", get_fn)?;
// jarvis.http.post(url, body, headers?)
let post_fn = lua.create_function(|lua, (url, body, headers): (String, Option<String>, Option<Table>)| {
http_request(lua, "POST", &url, body, headers)
})?;
http.set("post", post_fn)?;
// jarvis.http.post_json(url, data, headers?)
let post_json_fn = lua.create_function(|lua, (url, data, headers): (String, Table, Option<Table>)| {
// convert Lua table to JSON string
let json_value = table_to_json(lua, data)?;
let body = serde_json::to_string(&json_value)
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
// add content-type header
let mut header_map: HashMap<String, String> = HashMap::new();
header_map.insert("Content-Type".to_string(), "application/json".to_string());
if let Some(h) = headers {
for pair in h.pairs::<String, String>() {
if let Ok((k, v)) = pair {
header_map.insert(k, v);
}
}
}
http_request_with_headers(lua, "POST", &url, Some(body), header_map)
})?;
http.set("post_json", post_json_fn)?;
// jarvis.http.json(url) - GET + parse JSON
let json_fn = lua.create_function(|lua, url: String| {
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(30))
.build()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
let response = client.get(&url)
.send()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
if response.status().is_success() {
let json: serde_json::Value = response.json()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
json_to_lua(lua, json)
} else {
Ok(Value::Nil)
}
})?;
http.set("json", json_fn)?;
jarvis.set("http", http)?;
Ok(())
}
fn http_request(
lua: &Lua,
method: &str,
url: &str,
body: Option<String>,
headers: Option<Table>,
) -> mlua::Result<Table> {
let header_map: HashMap<String, String> = if let Some(h) = headers {
h.pairs::<String, String>()
.filter_map(|r| r.ok())
.collect()
} else {
HashMap::new()
};
http_request_with_headers(lua, method, url, body, header_map)
}
fn http_request_with_headers(
lua: &Lua,
method: &str,
url: &str,
body: Option<String>,
headers: HashMap<String, String>,
) -> mlua::Result<Table> {
let client = reqwest::blocking::Client::builder()
.timeout(Duration::from_secs(30))
.build()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
let mut request = match method {
"POST" => client.post(url),
"PUT" => client.put(url),
"DELETE" => client.delete(url),
_ => client.get(url),
};
for (k, v) in headers {
request = request.header(&k, &v);
}
if let Some(b) = body {
request = request.body(b);
}
let result = lua.create_table()?;
match request.send() {
Ok(response) => {
result.set("ok", response.status().is_success())?;
result.set("status", response.status().as_u16())?;
// get headers
let headers_table = lua.create_table()?;
for (name, value) in response.headers() {
if let Ok(v) = value.to_str() {
headers_table.set(name.as_str(), v)?;
}
}
result.set("headers", headers_table)?;
// get body
match response.text() {
Ok(text) => result.set("body", text)?,
Err(e) => result.set("body", format!("Error reading body: {}", e))?,
}
}
Err(e) => {
result.set("ok", false)?;
result.set("status", 0)?;
result.set("error", e.to_string())?;
result.set("body", "")?;
}
}
Ok(result)
}
// Convert Lua table to serde_json::Value
fn table_to_json(lua: &Lua, table: Table) -> mlua::Result<serde_json::Value> {
use serde_json::{Value as JsonValue, Map};
// check if it's an array (sequential integer keys starting from 1)
let is_array = table.clone().pairs::<i64, Value>()
.filter_map(|r| r.ok())
.enumerate()
.all(|(i, (k, _))| k == (i + 1) as i64);
if is_array && table.len()? > 0 {
let arr: Vec<JsonValue> = table.sequence_values::<Value>()
.filter_map(|r| r.ok())
.map(|v| lua_to_json(lua, v))
.collect::<Result<Vec<_>, _>>()?;
Ok(JsonValue::Array(arr))
} else {
let mut map = Map::new();
for pair in table.pairs::<String, Value>() {
let (k, v) = pair?;
map.insert(k, lua_to_json(lua, v)?);
}
Ok(JsonValue::Object(map))
}
}
// Convert Lua Value to serde_json::Value
fn lua_to_json(lua: &Lua, value: Value) -> mlua::Result<serde_json::Value> {
use serde_json::{Value as JsonValue, Number};
match value {
Value::Nil => Ok(JsonValue::Null),
Value::Boolean(b) => Ok(JsonValue::Bool(b)),
Value::Integer(i) => Ok(JsonValue::Number(Number::from(i))),
Value::Number(n) => {
Number::from_f64(n)
.map(JsonValue::Number)
.ok_or_else(|| mlua::Error::runtime("Invalid float"))
}
Value::String(s) => Ok(JsonValue::String(s.to_str()?.to_string())),
Value::Table(t) => table_to_json(lua, t),
_ => Err(mlua::Error::runtime("Unsupported type for JSON")),
}
}
// Convert serde_json::Value to Lua Value
fn json_to_lua(lua: &Lua, json: serde_json::Value) -> mlua::Result<Value> {
use serde_json::Value as JsonValue;
match json {
JsonValue::Null => Ok(Value::Nil),
JsonValue::Bool(b) => Ok(Value::Boolean(b)),
JsonValue::Number(n) => {
if let Some(i) = n.as_i64() {
Ok(Value::Integer(i))
} else if let Some(f) = n.as_f64() {
Ok(Value::Number(f))
} else {
Ok(Value::Nil)
}
}
JsonValue::String(s) => Ok(Value::String(lua.create_string(&s)?)),
JsonValue::Array(arr) => {
let table = lua.create_table()?;
for (i, v) in arr.into_iter().enumerate() {
table.set(i + 1, json_to_lua(lua, v)?)?;
}
Ok(Value::Table(table))
}
JsonValue::Object(map) => {
let table = lua.create_table()?;
for (k, v) in map {
table.set(k, json_to_lua(lua, v)?)?;
}
Ok(Value::Table(table))
}
}
}

View file

@ -0,0 +1,184 @@
// State Lua API: persistent key-value storage per command
use mlua::{Lua, Table, Result, Value};
use std::path::PathBuf;
use std::fs;
use std::collections::HashMap;
const STATE_FILE: &str = ".state.json";
pub fn register(lua: &Lua, jarvis: &Table, command_path: &PathBuf) -> mlua::Result<()> {
let state = lua.create_table()?;
let state_path = command_path.join(STATE_FILE);
// jarvis.state.get(key)
let state_path_get = state_path.clone();
let get_fn = lua.create_function(move |lua, key: String| {
let data = load_state(&state_path_get);
if let Some(value) = data.get(&key) {
json_to_lua_value(lua, value.clone())
} else {
Ok(Value::Nil)
}
})?;
state.set("get", get_fn)?;
// jarvis.state.set(key, value)
let state_path_set = state_path.clone();
let set_fn = lua.create_function(move |_, (key, value): (String, Value)| {
let mut data = load_state(&state_path_set);
let json_value = lua_to_json_value(value)?;
data.insert(key, json_value);
save_state(&state_path_set, &data)?;
Ok(true)
})?;
state.set("set", set_fn)?;
// jarvis.state.delete(key)
let state_path_delete = state_path.clone();
let delete_fn = lua.create_function(move |_, key: String| {
let mut data = load_state(&state_path_delete);
let existed = data.remove(&key).is_some();
save_state(&state_path_delete, &data)?;
Ok(existed)
})?;
state.set("delete", delete_fn)?;
// jarvis.state.clear()
let state_path_clear = state_path.clone();
let clear_fn = lua.create_function(move |_, ()| {
let data: HashMap<String, serde_json::Value> = HashMap::new();
save_state(&state_path_clear, &data)?;
Ok(true)
})?;
state.set("clear", clear_fn)?;
// jarvis.state.keys()
let state_path_keys = state_path.clone();
let keys_fn = lua.create_function(move |lua, ()| {
let data = load_state(&state_path_keys);
let table = lua.create_table()?;
for (i, key) in data.keys().enumerate() {
table.set(i + 1, key.clone())?;
}
Ok(table)
})?;
state.set("keys", keys_fn)?;
// jarvis.state.all()
let state_path_all = state_path.clone();
let all_fn = lua.create_function(move |lua, ()| {
let data = load_state(&state_path_all);
let table = lua.create_table()?;
for (key, value) in data {
let lua_value = json_to_lua_value(lua, value)?;
table.set(key, lua_value)?;
}
Ok(table)
})?;
state.set("all", all_fn)?;
jarvis.set("state", state)?;
Ok(())
}
fn load_state(path: &PathBuf) -> HashMap<String, serde_json::Value> {
if !path.exists() {
return HashMap::new();
}
fs::read_to_string(path)
.ok()
.and_then(|s| serde_json::from_str(&s).ok())
.unwrap_or_default()
}
fn save_state(path: &PathBuf, data: &HashMap<String, serde_json::Value>) -> mlua::Result<()> {
let json = serde_json::to_string_pretty(data)
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
fs::write(path, json)
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
Ok(())
}
fn lua_to_json_value(value: Value) -> mlua::Result<serde_json::Value> {
use serde_json::Value as JsonValue;
match value {
Value::Nil => Ok(JsonValue::Null),
Value::Boolean(b) => Ok(JsonValue::Bool(b)),
Value::Integer(i) => Ok(JsonValue::Number(i.into())),
Value::Number(n) => {
serde_json::Number::from_f64(n)
.map(JsonValue::Number)
.ok_or_else(|| mlua::Error::runtime("Invalid float"))
}
Value::String(s) => Ok(JsonValue::String(s.to_str()?.to_string())),
Value::Table(t) => {
// check if array
let is_array = t.clone().pairs::<i64, Value>()
.filter_map(|r| r.ok())
.enumerate()
.all(|(i, (k, _))| k == (i + 1) as i64);
if is_array && t.len().unwrap_or(0) > 0 {
let arr: Vec<JsonValue> = t.sequence_values::<Value>()
.filter_map(|r| r.ok())
.map(lua_to_json_value)
.collect::<Result<Vec<_>>>()?;
Ok(JsonValue::Array(arr))
} else {
let mut map = serde_json::Map::new();
for pair in t.pairs::<String, Value>() {
let (k, v) = pair?;
map.insert(k, lua_to_json_value(v)?);
}
Ok(JsonValue::Object(map))
}
}
_ => Err(mlua::Error::runtime("Unsupported type for state")),
}
}
fn json_to_lua_value(lua: &Lua, json: serde_json::Value) -> mlua::Result<Value> {
use serde_json::Value as JsonValue;
match json {
JsonValue::Null => Ok(Value::Nil),
JsonValue::Bool(b) => Ok(Value::Boolean(b)),
JsonValue::Number(n) => {
if let Some(i) = n.as_i64() {
Ok(Value::Integer(i))
} else if let Some(f) = n.as_f64() {
Ok(Value::Number(f))
} else {
Ok(Value::Nil)
}
}
JsonValue::String(s) => Ok(Value::String(lua.create_string(&s)?)),
JsonValue::Array(arr) => {
let table = lua.create_table()?;
for (i, v) in arr.into_iter().enumerate() {
table.set(i + 1, json_to_lua_value(lua, v)?)?;
}
Ok(Value::Table(table))
}
JsonValue::Object(map) => {
let table = lua.create_table()?;
for (k, v) in map {
table.set(k, json_to_lua_value(lua, v)?)?;
}
Ok(Value::Table(table))
}
}
}

View file

@ -0,0 +1,240 @@
// System Lua API: exec, open, clipboard, notify
use mlua::{Lua, Table};
use std::process::Command;
use crate::lua::sandbox::SandboxLevel;
pub fn register(lua: &Lua, jarvis: &Table, sandbox: SandboxLevel) -> mlua::Result<()> {
let system = lua.create_table()?;
// jarvis.system.open(url_or_path) - always available
let open_fn = lua.create_function(|_, target: String| {
let result = if cfg!(target_os = "windows") {
Command::new("cmd")
.args(["/C", "start", "", &target])
.spawn()
} else if cfg!(target_os = "macos") {
Command::new("open")
.arg(&target)
.spawn()
} else {
Command::new("xdg-open")
.arg(&target)
.spawn()
};
match result {
Ok(_) => Ok(true),
Err(e) => {
log::warn!("[Lua] Failed to open {}: {}", target, e);
Ok(false)
}
}
})?;
system.set("open", open_fn)?;
// jarvis.system.exec(cmd, args?) - only in full sandbox
if sandbox.allows_exec() {
let exec_fn = lua.create_function(|lua, (cmd, args): (String, Option<Table>)| {
let mut command = if cfg!(target_os = "windows") {
let mut c = Command::new("cmd");
c.args(["/C", &cmd]);
c
} else {
let mut c = Command::new("sh");
c.args(["-c", &cmd]);
c
};
// add extra args if provided
if let Some(args_table) = args {
for pair in args_table.sequence_values::<String>() {
if let Ok(arg) = pair {
command.arg(arg);
}
}
}
let output = command.output()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
let result = lua.create_table()?;
result.set("success", output.status.success())?;
result.set("code", output.status.code().unwrap_or(-1))?;
result.set("stdout", String::from_utf8_lossy(&output.stdout).to_string())?;
result.set("stderr", String::from_utf8_lossy(&output.stderr).to_string())?;
Ok(result)
})?;
system.set("exec", exec_fn)?;
}
// jarvis.system.notify(title, message) - always available
let notify_fn = lua.create_function(|_, (title, message): (String, String)| {
log::info!("[Lua] NOTIFY: {} - {}", title, message);
// platform-specific notification
#[cfg(target_os = "windows")]
{
use winrt_notification::{Toast, Duration as ToastDuration};
if let Err(e) = Toast::new(Toast::POWERSHELL_APP_ID)
.title(&title)
.text1(&message)
.duration(ToastDuration::Short)
.show()
{
log::warn!("[Lua] Failed to show toast notification: {}", e);
// fallback to msg.exe
let _ = Command::new("msg")
.args(["*", "/time:10", &format!("{}: {}", title, message)])
.spawn();
}
}
#[cfg(target_os = "linux")]
{
let _ = Command::new("notify-send")
.args([&title, &message])
.spawn();
}
#[cfg(target_os = "macos")]
{
let script = format!(
r#"display notification "{}" with title "{}""#,
message.replace("\"", "\\\""),
title.replace("\"", "\\\"")
);
let _ = Command::new("osascript")
.args(["-e", &script])
.spawn();
}
Ok(true)
})?;
system.set("notify", notify_fn)?;
// jarvis.system.clipboard - subtable
let clipboard = lua.create_table()?;
// jarvis.system.clipboard.get() - always available
let clipboard_get_fn = lua.create_function(|_, ()| {
#[cfg(target_os = "windows")]
{
let output = Command::new("powershell")
.args(["-Command", "Get-Clipboard"])
.output()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
}
#[cfg(target_os = "linux")]
{
let output = Command::new("xclip")
.args(["-selection", "clipboard", "-o"])
.output()
.or_else(|_| {
Command::new("xsel")
.args(["--clipboard", "--output"])
.output()
})
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
Ok(String::from_utf8_lossy(&output.stdout).to_string())
}
#[cfg(target_os = "macos")]
{
let output = Command::new("pbpaste")
.output()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
Ok(String::from_utf8_lossy(&output.stdout).to_string())
}
#[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))]
{
Err(mlua::Error::runtime("Clipboard not supported on this platform"))
}
})?;
clipboard.set("get", clipboard_get_fn)?;
// jarvis.system.clipboard.set(text) - only in full sandbox
if sandbox.allows_clipboard_write() {
let clipboard_set_fn = lua.create_function(|_, text: String| {
#[cfg(target_os = "windows")]
{
let script = format!("Set-Clipboard -Value '{}'", text.replace("'", "''"));
Command::new("powershell")
.args(["-Command", &script])
.output()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
}
#[cfg(target_os = "linux")]
{
use std::io::Write;
let mut child = Command::new("xclip")
.args(["-selection", "clipboard"])
.stdin(std::process::Stdio::piped())
.spawn()
.or_else(|_| {
Command::new("xsel")
.args(["--clipboard", "--input"])
.stdin(std::process::Stdio::piped())
.spawn()
})
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
if let Some(stdin) = child.stdin.as_mut() {
stdin.write_all(text.as_bytes())
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
}
}
#[cfg(target_os = "macos")]
{
use std::io::Write;
let mut child = Command::new("pbcopy")
.stdin(std::process::Stdio::piped())
.spawn()
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
if let Some(stdin) = child.stdin.as_mut() {
stdin.write_all(text.as_bytes())
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
}
}
Ok(true)
})?;
clipboard.set("set", clipboard_set_fn)?;
}
system.set("clipboard", clipboard)?;
// jarvis.system.env(name) - get environment variable (always available)
let env_fn = lua.create_function(|_, name: String| {
Ok(std::env::var(&name).ok())
})?;
system.set("env", env_fn)?;
// jarvis.system.platform - read-only string
let platform = if cfg!(target_os = "windows") {
"windows"
} else if cfg!(target_os = "macos") {
"macos"
} else if cfg!(target_os = "linux") {
"linux"
} else {
"unknown"
};
system.set("platform", platform)?;
jarvis.set("system", system)?;
Ok(())
}

View file

@ -0,0 +1,169 @@
use mlua::{Lua, Value, StdLib};
use std::path::PathBuf;
use std::time::Duration;
use std::fs;
use super::sandbox::SandboxLevel;
use super::error::LuaError;
use super::{CommandContext, CommandResult};
use super::api;
pub struct LuaEngine {
lua: Lua,
sandbox: SandboxLevel,
}
impl LuaEngine {
pub fn new(sandbox: SandboxLevel) -> Result<Self, LuaError> {
// select which standard libraries to load based on sandbox access level
let std_libs = match sandbox {
SandboxLevel::Minimal => {
StdLib::TABLE | StdLib::STRING | StdLib::MATH
}
SandboxLevel::Standard => {
StdLib::TABLE | StdLib::STRING | StdLib::MATH | StdLib::UTF8
}
SandboxLevel::Full => {
StdLib::TABLE | StdLib::STRING | StdLib::MATH | StdLib::UTF8 | StdLib::OS
}
};
let lua = Lua::new_with(std_libs, mlua::LuaOptions::default())
.map_err(|e| LuaError::InitError(e.to_string()))?;
// remove dangerous globals regardless of sandbox
{
let globals = lua.globals();
// always remove these
let _ = globals.set("loadfile", Value::Nil);
let _ = globals.set("dofile", Value::Nil);
let _ = globals.set("load", Value::Nil);
let _ = globals.set("loadstring", Value::Nil);
// remove io unless full sandbox
if !matches!(sandbox, SandboxLevel::Full) {
let _ = globals.set("io", Value::Nil);
}
// remove os.execute, os.exit, os.setlocale even in full mode
// for SECURITY REASONS!!!
if matches!(sandbox, SandboxLevel::Full) {
if let Ok(os) = globals.get::<mlua::Table>("os") {
let _ = os.set("execute", Value::Nil);
let _ = os.set("exit", Value::Nil);
let _ = os.set("remove", Value::Nil);
let _ = os.set("rename", Value::Nil);
let _ = os.set("setlocale", Value::Nil);
}
}
}
Ok(Self { lua, sandbox })
}
// Register all jarvis APIs
fn register_api(&self, context: &CommandContext) -> Result<(), LuaError> {
let globals = self.lua.globals();
// main jarvis table
let jarvis = self.lua.create_table()
.map_err(|e| LuaError::InitError(e.to_string()))?;
// always register core APIs
api::core::register(&self.lua, &jarvis)?;
api::audio::register(&self.lua, &jarvis)?;
api::context::register(&self.lua, &jarvis, context)?;
// sandbox-controlled APIs
if self.sandbox.allows_http() {
api::http::register(&self.lua, &jarvis)?;
}
if self.sandbox.allows_state() {
api::state::register(&self.lua, &jarvis, &context.command_path)?;
}
if self.sandbox.allows_fs() {
api::fs::register(&self.lua, &jarvis, &context.command_path, self.sandbox)?;
}
api::system::register(&self.lua, &jarvis, self.sandbox)?;
globals.set("jarvis", jarvis)
.map_err(|e| LuaError::InitError(e.to_string()))?;
Ok(())
}
// Main LUA executor
pub fn execute(
&self,
script_path: &PathBuf,
context: CommandContext,
timeout: Duration,
) -> Result<CommandResult, LuaError> {
// register APIs
self.register_api(&context)?;
// load script
let script_content = fs::read_to_string(script_path)
.map_err(|e| LuaError::LoadError(format!("{}: {}", script_path.display(), e)))?;
let script_name = script_path.file_name()
.unwrap()
.to_string_lossy()
.to_string();
// set up timeout hook
let start = std::time::Instant::now();
self.lua.set_hook(mlua::HookTriggers {
every_nth_instruction: Some(1000),
..Default::default()
}, move |_lua, _debug| {
if start.elapsed() > timeout {
Err(mlua::Error::runtime("Script timeout"))
} else {
Ok(mlua::VmState::Continue)
}
}).map_err(|e| LuaError::InitError(e.to_string()))?;
// execute script
let result = self.lua.load(&script_content)
.set_name(&script_name)
.eval::<Value>();
// remove hook
let _ = self.lua.remove_hook();
// result
match result {
Ok(value) => Ok(self.parse_result(value)),
Err(e) => {
if e.to_string().contains("timeout") {
Err(LuaError::Timeout)
} else {
Err(LuaError::RuntimeError(e.to_string()))
}
}
}
}
// Parse Lua return value into CommandResult
fn parse_result(&self, value: Value) -> CommandResult {
match value {
// return { chain = false }
Value::Table(t) => {
let chain = t.get::<bool>("chain").unwrap_or(true);
CommandResult { chain }
}
// return false (shorthand for no chain)
Value::Boolean(chain) => CommandResult { chain },
// return nil or no return = chain
_ => CommandResult::default(),
}
}
}

View file

@ -0,0 +1,49 @@
use std::fmt;
#[derive(Debug)]
pub enum LuaError {
// Failed to create Lua VM
InitError(String),
// Failed to load script file
LoadError(String),
// Script execution error
RuntimeError(String),
// Script exceeded timeout
Timeout,
// Sandbox violation
SandboxViolation(String),
// IO error
IoError(std::io::Error),
}
impl fmt::Display for LuaError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
LuaError::InitError(msg) => write!(f, "Lua init error: {}", msg),
LuaError::LoadError(msg) => write!(f, "Script load error: {}", msg),
LuaError::RuntimeError(msg) => write!(f, "Runtime error: {}", msg),
LuaError::Timeout => write!(f, "Script timeout"),
LuaError::SandboxViolation(msg) => write!(f, "Sandbox violation: {}", msg),
LuaError::IoError(e) => write!(f, "IO error: {}", e),
}
}
}
impl std::error::Error for LuaError {}
impl From<mlua::Error> for LuaError {
fn from(e: mlua::Error) -> Self {
LuaError::RuntimeError(e.to_string())
}
}
impl From<std::io::Error> for LuaError {
fn from(e: std::io::Error) -> Self {
LuaError::IoError(e)
}
}

View file

@ -0,0 +1,65 @@
use serde::{Deserialize, Serialize};
// Sandbox level controlling what APIs are available
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum SandboxLevel {
// Minimal: only core APIs (log, speak, audio, context)
Minimal,
// Standard: + http, state, fs (command folder only)
Standard,
// Full: + system.exec, expanded fs access
Full,
}
impl SandboxLevel {
pub fn from_str(s: &str) -> Self {
match s.to_lowercase().as_str() {
"minimal" => SandboxLevel::Minimal,
"full" => SandboxLevel::Full,
_ => SandboxLevel::Standard,
}
}
// Can use HTTP API
pub fn allows_http(&self) -> bool {
matches!(self, SandboxLevel::Standard | SandboxLevel::Full)
}
// Can use persistent state API
pub fn allows_state(&self) -> bool {
matches!(self, SandboxLevel::Standard | SandboxLevel::Full)
}
// Can use file system API
pub fn allows_fs(&self) -> bool {
matches!(self, SandboxLevel::Standard | SandboxLevel::Full)
}
// Can write files
pub fn allows_fs_write(&self) -> bool {
matches!(self, SandboxLevel::Standard | SandboxLevel::Full)
}
// Can execute system commands
pub fn allows_exec(&self) -> bool {
matches!(self, SandboxLevel::Full)
}
// Can access clipboard write
pub fn allows_clipboard_write(&self) -> bool {
matches!(self, SandboxLevel::Full)
}
// Can access paths outside command folder
pub fn allows_expanded_paths(&self) -> bool {
matches!(self, SandboxLevel::Full)
}
}
impl Default for SandboxLevel {
fn default() -> Self {
SandboxLevel::Standard
}
}

View file

@ -0,0 +1,35 @@
use std::{collections::HashMap, path::PathBuf};
use crate::commands::SlotValue;
// Context passed to Lua scripts
#[derive(Debug, Clone)]
pub struct CommandContext {
// The phrase that triggered the command
pub phrase: String,
// Command ID
pub command_id: String,
// Path to command folder
pub command_path: PathBuf,
// Current language
pub language: String,
// Slots
pub slots: Option<HashMap<String, SlotValue>>
}
// Result returned from Lua script execution
#[derive(Debug, Clone)]
pub struct CommandResult {
// Whether to continue chaining commands
pub chain: bool,
}
impl Default for CommandResult {
fn default() -> Self {
Self { chain: true }
}
}

View file

@ -0,0 +1,112 @@
#[cfg(test)]
mod tests {
use crate::lua::{CommandContext, LuaError, SandboxLevel, execute};
use std::path::PathBuf;
use std::time::Duration;
use tempfile::tempdir;
use std::fs;
fn create_test_context(cmd_path: PathBuf) -> CommandContext {
CommandContext {
phrase: "test phrase".to_string(),
command_id: "test_cmd".to_string(),
command_path: cmd_path,
language: "en".to_string(),
slots: None,
}
}
#[test]
fn test_minimal_sandbox() {
let dir = tempdir().unwrap();
let script_path = dir.path().join("test.lua");
fs::write(&script_path, r#"
jarvis.log("info", "test log")
return { chain = false }
"#).unwrap();
let context = create_test_context(dir.path().to_path_buf());
let result = execute(
&script_path,
context,
SandboxLevel::Minimal,
Duration::from_secs(5),
);
assert!(result.is_ok());
assert_eq!(result.unwrap().chain, false);
}
#[test]
fn test_state_persistence() {
let dir = tempdir().unwrap();
let script_path = dir.path().join("test.lua");
// first run - set state
fs::write(&script_path, r#"
jarvis.state.set("key", "value")
return true
"#).unwrap();
let context = create_test_context(dir.path().to_path_buf());
execute(&script_path, context, SandboxLevel::Standard, Duration::from_secs(5)).unwrap();
// second run - read state
fs::write(&script_path, r#"
local val = jarvis.state.get("key")
if val == "value" then
return true
else
error("State not persisted")
end
"#).unwrap();
let context = create_test_context(dir.path().to_path_buf());
let result = execute(&script_path, context, SandboxLevel::Standard, Duration::from_secs(5));
assert!(result.is_ok());
}
#[test]
fn test_timeout() {
let dir = tempdir().unwrap();
let script_path = dir.path().join("test.lua");
fs::write(&script_path, r#"
while true do end
"#).unwrap();
let context = create_test_context(dir.path().to_path_buf());
let result = execute(
&script_path,
context,
SandboxLevel::Minimal,
Duration::from_millis(100),
);
assert!(matches!(result, Err(LuaError::Timeout)));
}
#[test]
fn test_sandbox_fs_escape() {
let dir = tempdir().unwrap();
let script_path = dir.path().join("test.lua");
fs::write(&script_path, r#"
local ok, err = pcall(function()
jarvis.fs.read("../../../etc/passwd")
end)
if ok then
error("Should have been blocked")
end
return true
"#).unwrap();
let context = create_test_context(dir.path().to_path_buf());
let result = execute(&script_path, context, SandboxLevel::Standard, Duration::from_secs(5));
assert!(result.is_ok());
}
}

View file

@ -0,0 +1,67 @@
mod registry;
mod catalog;
pub mod structs;
pub mod loaders;
pub mod vosk_models;
pub mod gliner_models;
// re-export loaders
#[cfg(feature = "jarvis_app")]
pub use loaders::embedding;
#[cfg(feature = "jarvis_app")]
pub use loaders::gliner;
#[cfg(feature = "jarvis_app")]
pub use loaders::ort_model;
#[cfg(feature = "jarvis_app")]
pub use loaders::intent_classifier;
#[cfg(feature = "vosk")]
pub use loaders::vosk;
#[cfg(feature = "nnnoiseless")]
pub use loaders::nnnoiseless;
pub use registry::ModelRegistry;
pub use structs::{Task, ModelDef, BackendOption};
use once_cell::sync::OnceCell;
use crate::APP_DIR;
pub const MODELS_PATH: &str = "resources/models";
static REGISTRY: OnceCell<ModelRegistry> = OnceCell::new();
pub fn init() -> Result<(), String> {
if REGISTRY.get().is_some() {
return Ok(());
}
let registry = ModelRegistry::new();
let models_dir = APP_DIR.join(MODELS_PATH);
let models = catalog::scan_models(&models_dir);
info!("Found {} model(s) in {:?}", models.len(), models_dir);
registry.set_catalog(models);
REGISTRY.set(registry)
.map_err(|_| "Models registry already initialized".to_string())?;
Ok(())
}
pub fn registry() -> &'static ModelRegistry {
REGISTRY.get().expect("Models registry not initialized - call models::init() first")
}
pub fn get_options(task: Task) -> Vec<BackendOption> {
registry().with_catalog(|models| catalog::get_options(task, models))
}
pub fn is_valid_backend(task: Task, backend_id: &str) -> bool {
registry().with_catalog(|models| catalog::is_valid_backend(task, backend_id, models))
}

View file

@ -0,0 +1,140 @@
use std::fs;
use std::path::Path;
use super::structs::{Task, ModelDef, BackendOption};
// scan the models directory for folders containing model.toml
pub fn scan_models(models_dir: &Path) -> Vec<ModelDef> {
let mut models = Vec::new();
if !models_dir.exists() {
warn!("Models directory not found: {:?}", models_dir);
return models;
}
let entries = match fs::read_dir(models_dir) {
Ok(e) => e,
Err(e) => {
warn!("Failed to read models dir: {}", e);
return models;
}
};
for entry in entries.flatten() {
let path = entry.path();
if !path.is_dir() {
continue;
}
let toml_path = path.join("model.toml");
if !toml_path.exists() {
continue;
}
match load_model_def(&toml_path, &path) {
Ok(def) => {
info!("Found model: {} ({}) - tasks: {:?}", def.name, def.id, def.tasks);
models.push(def);
}
Err(e) => warn!("Failed to load model from {:?}: {}", path, e),
}
}
models
}
fn load_model_def(toml_path: &Path, model_dir: &Path) -> Result<ModelDef, String> {
let content = fs::read_to_string(toml_path)
.map_err(|e| format!("read error: {}", e))?;
let parsed: ModelToml = toml::from_str(&content)
.map_err(|e| format!("parse error: {}", e))?;
let mut def = parsed.model;
def.path = model_dir.to_path_buf();
Ok(def)
}
#[derive(serde::Deserialize)]
struct ModelToml {
model: ModelDef,
}
// Code backends per task
pub fn code_backends(task: Task) -> Vec<BackendOption> {
match task {
Task::Intent => vec![
BackendOption {
id: "intent-classifier".into(),
name: "Intent Classifier".into(),
model_id: None,
},
],
Task::Slots => vec![],
Task::Vad => vec![
BackendOption {
id: "energy".into(),
name: "Energy-based".into(),
model_id: None,
},
BackendOption {
id: "nnnoiseless".into(),
name: "Nnnoiseless".into(),
model_id: None,
},
],
Task::NoiseSuppression => vec![
BackendOption {
id: "nnnoiseless".into(),
name: "Nnnoiseless".into(),
model_id: None,
},
],
Task::Stt => vec![
BackendOption {
id: "vosk".into(),
name: "Vosk".into(),
model_id: None,
},
],
}
}
// get all available options for a task:
// "none" first, then code backends, then AI models from catalog
pub fn get_options(task: Task, models: &[ModelDef]) -> Vec<BackendOption> {
let mut options = vec![
BackendOption {
id: "none".into(),
name: "Disabled".into(),
model_id: None,
},
];
options.extend(code_backends(task));
for model in models {
if model.tasks.contains(&task) {
options.push(BackendOption {
id: model.id.clone(),
name: model.name.clone(),
model_id: Some(model.id.clone()),
});
}
}
options
}
pub fn is_valid_backend(task: Task, backend_id: &str, models: &[ModelDef]) -> bool {
if backend_id == "none" {
return true;
}
if code_backends(task).iter().any(|b| b.id == backend_id) {
return true;
}
models.iter().any(|m| m.id == backend_id && m.tasks.contains(&task))
}

View file

@ -0,0 +1,130 @@
use std::collections::HashMap;
use std::fs;
use crate::APP_DIR;
const GLINER_DIRS: &[&str] = &["gliner_small-v2.1", "gliner_multi-v2.1"];
#[derive(Debug, Clone)]
pub struct GlinerModelVariant {
// type id stored in settings, e.g. "int8", "fp16", "full"
pub value: String,
// shown in dropdown, e.g. "int8 (174MB / 332MB)"
pub display_name: String,
}
// scan both model dirs and return a deduplicated list of model types
pub fn scan_gliner_variants() -> Vec<GlinerModelVariant> {
let base = APP_DIR.join("resources").join("models");
// collect: type -> { dir_name -> size_mb }
let mut types: HashMap<String, HashMap<String, u64>> = HashMap::new();
for dir_name in GLINER_DIRS {
let onnx_dir = base.join(dir_name).join("onnx");
if !onnx_dir.exists() { continue; }
let entries = match fs::read_dir(&onnx_dir) {
Ok(e) => e,
Err(_) => continue,
};
for entry in entries.flatten() {
let path = entry.path();
let file_name = match path.file_name().and_then(|n| n.to_str()) {
Some(n) if n.ends_with(".onnx") => n.to_string(),
_ => continue,
};
let variant_type = file_name_to_type(&file_name);
let size_mb = fs::metadata(&path).map(|m| m.len()).unwrap_or(0) / (1024 * 1024);
types.entry(variant_type)
.or_default()
.insert(dir_name.to_string(), size_mb);
}
}
let mut result: Vec<GlinerModelVariant> = types.into_iter().map(|(variant, sizes)| {
let size_str = build_size_string(&sizes);
let label = if variant == "full" { "Full".to_string() } else { variant.clone() };
GlinerModelVariant {
display_name: format!("{} ({})", label, size_str),
value: variant,
}
}).collect();
// sort: full first, then alphabetically
result.sort_by(|a, b| {
let a_full = a.value == "full";
let b_full = b.value == "full";
b_full.cmp(&a_full).then_with(|| a.value.cmp(&b.value))
});
result
}
// "model.onnx" -> "full", "model_int8.onnx" -> "int8"
fn file_name_to_type(name: &str) -> String {
let stem = name.strip_suffix(".onnx").unwrap_or(name);
if stem == "model" {
"full".to_string()
} else if let Some(variant) = stem.strip_prefix("model_") {
variant.to_string()
} else {
stem.to_string()
}
}
// build size display: "174MB" if only one dir, "small: 174MB / multi: 332MB" if both
fn build_size_string(sizes: &HashMap<String, u64>) -> String {
if sizes.len() == 1 {
let (dir, mb) = sizes.iter().next().unwrap();
let short = short_dir_name(dir);
return format!("{}: {}MB", short, mb);
}
let mut parts: Vec<String> = Vec::new();
for dir_name in GLINER_DIRS {
if let Some(mb) = sizes.get(*dir_name) {
parts.push(format!("{}: {}MB", short_dir_name(dir_name), mb));
}
}
parts.join(" / ")
}
fn short_dir_name(dir: &str) -> &str {
if dir.contains("small") { "small" }
else if dir.contains("multi") { "multi" }
else { dir }
}
// resolve variant type + language into actual file path
// returns (model_dir_path, onnx_file_name) or None
pub fn resolve_model(variant: &str, language: &str) -> Option<(std::path::PathBuf, String)> {
let base = APP_DIR.join("resources").join("models");
let file_name = type_to_file_name(variant);
// pick dir based on language
let preferred: &[&str] = match language {
"en" => &["gliner_small-v2.1", "gliner_multi-v2.1"],
_ => &["gliner_multi-v2.1", "gliner_small-v2.1"],
};
for dir_name in preferred {
let path = base.join(dir_name).join("onnx").join(&file_name);
if path.exists() {
return Some((base.join(dir_name), file_name));
}
}
None
}
// "full" -> "model.onnx", "int8" -> "model_int8.onnx"
fn type_to_file_name(variant: &str) -> String {
if variant == "full" || variant.is_empty() {
"model.onnx".to_string()
} else {
format!("model_{}.onnx", variant)
}
}

View file

@ -0,0 +1,47 @@
// fastembed embedding model (all-MiniLM-L6-v2, paraphrase-multilingual, etc.)
use std::sync::Arc;
use parking_lot::Mutex;
use fastembed::{TextEmbedding, UserDefinedEmbeddingModel, TokenizerFiles, Pooling, QuantizationMode, OutputKey};
use crate::models::registry::ModelRegistry;
pub struct EmbeddingModel {
pub embedding: Mutex<TextEmbedding>,
}
// fastembed uses ORT internally which is thread-safe
unsafe impl Send for EmbeddingModel {}
unsafe impl Sync for EmbeddingModel {}
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<EmbeddingModel>, String> {
registry.get_or_load::<EmbeddingModel>(model_id, |def| {
let model_dir = &def.path;
info!("Loading embedding model from: {}", model_dir.display());
let user_model = UserDefinedEmbeddingModel {
onnx_file: std::fs::read(model_dir.join("model.onnx"))
.map_err(|e| format!("Failed to read model.onnx: {}", e))?,
tokenizer_files: TokenizerFiles {
tokenizer_file: std::fs::read(model_dir.join("tokenizer.json"))
.map_err(|e| format!("Failed to read tokenizer.json: {}", e))?,
config_file: std::fs::read(model_dir.join("config.json"))
.map_err(|e| format!("Failed to read config.json: {}", e))?,
special_tokens_map_file: std::fs::read(model_dir.join("special_tokens_map.json"))
.map_err(|e| format!("Failed to read special_tokens_map.json: {}", e))?,
tokenizer_config_file: std::fs::read(model_dir.join("tokenizer_config.json"))
.map_err(|e| format!("Failed to read tokenizer_config.json: {}", e))?,
},
pooling: Some(Pooling::Mean),
quantization: QuantizationMode::None,
output_key: Some(OutputKey::ByName("last_hidden_state")),
};
let model = TextEmbedding::try_new_from_user_defined(user_model, Default::default())
.map_err(|e| format!("Failed to load embedding model: {}", e))?;
info!("Embedding model loaded: {}", def.name);
Ok(EmbeddingModel { embedding: Mutex::new(model) })
})
}

View file

@ -0,0 +1,51 @@
// GLiNER model for named entity recognition / slot extraction
use std::sync::Arc;
use parking_lot::Mutex;
use regex::Regex;
use tokenizers::Tokenizer;
use crate::models::registry::ModelRegistry;
const WORD_REGEX: &str = r"\w+(?:[-_]\w+)*|\S";
pub struct GlinerModel {
pub session: Mutex<ort::session::Session>,
pub tokenizer: Tokenizer,
pub splitter: Regex,
}
unsafe impl Send for GlinerModel {}
unsafe impl Sync for GlinerModel {}
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<GlinerModel>, String> {
registry.get_or_load::<GlinerModel>(model_id, |def| {
let model_dir = &def.path;
// GLiNER models keep onnx files in an "onnx" subfolder
let onnx_dir = model_dir.join("onnx");
let model_path = if onnx_dir.exists() {
onnx_dir.join("model.onnx")
} else {
model_dir.join("model.onnx")
};
let tokenizer_path = model_dir.join("tokenizer.json");
info!("Loading GLiNER model from: {}", model_dir.display());
let session = ort::session::Session::builder()
.map_err(|e| format!("Failed to create ORT session builder: {}", e))?
.commit_from_file(&model_path)
.map_err(|e| format!("Failed to load ONNX model: {}", e))?;
let tokenizer = Tokenizer::from_file(&tokenizer_path)
.map_err(|e| format!("Failed to load tokenizer: {}", e))?;
let splitter = Regex::new(WORD_REGEX)
.map_err(|e| format!("Failed to compile word regex: {}", e))?;
info!("GLiNER model loaded: {}", def.name);
Ok(GlinerModel { session: Mutex::new(session), tokenizer, splitter })
})
}

View file

@ -0,0 +1,30 @@
// intent-classifier crate wrapper
use std::sync::Arc;
use intent_classifier::IntentClassifier;
use crate::models::registry::ModelRegistry;
pub struct IntentClassifierModel {
pub classifier: IntentClassifier,
}
unsafe impl Send for IntentClassifierModel {}
unsafe impl Sync for IntentClassifierModel {}
// init is async (IntentClassifier::new().await), so we create it
// outside the registry and insert it after
pub async fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<IntentClassifierModel>, String> {
if let Some(existing) = registry.get::<IntentClassifierModel>(model_id) {
info!("IntentClassifier '{}' already loaded, reusing", model_id);
return Ok(existing);
}
info!("Initializing IntentClassifier...");
let classifier = IntentClassifier::new().await
.map_err(|e| format!("Failed to init IntentClassifier: {}", e))?;
info!("IntentClassifier initialized");
Ok(registry.insert(model_id, IntentClassifierModel { classifier }))
}

View file

@ -0,0 +1,12 @@
#[cfg(feature = "jarvis_app")]
pub mod embedding;
#[cfg(feature = "jarvis_app")]
pub mod gliner;
#[cfg(feature = "jarvis_app")]
pub mod ort_model;
#[cfg(feature = "jarvis_app")]
pub mod intent_classifier;
#[cfg(feature = "vosk")]
pub mod vosk;
#[cfg(feature = "nnnoiseless")]
pub mod nnnoiseless;

View file

@ -0,0 +1,110 @@
// nnnoiseless - used for both noise suppression and VAD.
// each consumer needs its own DenoiseState (stateful per-stream),
// so this doesn't go through the registry. just centralizes creation.
use nnnoiseless::DenoiseState;
use crate::config;
// noise suppression instance
pub struct NnnoiselessNS {
state: Box<DenoiseState<'static>>,
buffer: Vec<f32>,
}
impl NnnoiselessNS {
pub fn new() -> Self {
Self {
state: DenoiseState::new(),
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
}
}
pub fn process(&mut self, input: &[i16]) -> Vec<i16> {
self.buffer.extend(input.iter().map(|&s| s as f32));
let frame_size = config::NNNOISELESS_FRAME_SIZE;
let full_frames = self.buffer.len() / frame_size;
if full_frames == 0 {
return input.to_vec();
}
let mut output: Vec<i16> = Vec::with_capacity(full_frames * frame_size);
let mut input_frame = [0.0f32; 480];
let mut output_frame = [0.0f32; 480];
let consumed = full_frames * frame_size;
for i in 0..full_frames {
let offset = i * frame_size;
input_frame.copy_from_slice(&self.buffer[offset..offset + frame_size]);
let _ = self.state.process_frame(&mut output_frame, &input_frame);
for &sample in &output_frame {
let clamped = sample.clamp(i16::MIN as f32, i16::MAX as f32);
output.push(clamped as i16);
}
}
// keep leftover samples (single drain at the end)
self.buffer.drain(..consumed);
output
}
pub fn reset(&mut self) {
self.buffer.clear();
}
}
// VAD instance
pub struct NnnoiselessVAD {
state: Box<DenoiseState<'static>>,
buffer: Vec<f32>,
}
impl NnnoiselessVAD {
pub fn new() -> Self {
Self {
state: DenoiseState::new(),
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
}
}
pub fn detect(&mut self, input: &[i16]) -> (bool, f32) {
self.buffer.extend(input.iter().map(|&s| s as f32));
let frame_size = config::NNNOISELESS_FRAME_SIZE;
let full_frames = self.buffer.len() / frame_size;
if full_frames == 0 {
return (true, 0.5);
}
let mut total_vad = 0.0f32;
let mut input_frame = [0.0f32; 480];
let mut output_frame = [0.0f32; 480];
let consumed = full_frames * frame_size;
for i in 0..full_frames {
let offset = i * frame_size;
input_frame.copy_from_slice(&self.buffer[offset..offset + frame_size]);
let vad_prob = self.state.process_frame(&mut output_frame, &input_frame);
total_vad += vad_prob;
}
// single drain
self.buffer.drain(..consumed);
let avg_vad = total_vad / full_frames as f32;
let is_voice = avg_vad >= config::VAD_NNNOISELESS_THRESHOLD;
(is_voice, avg_vad)
}
pub fn reset(&mut self) {
self.state = DenoiseState::new();
self.buffer.clear();
}
}

View file

@ -0,0 +1,44 @@
// generic ORT model - session + optional tokenizer.
// for models like BERT (tiny, distil, mini) that can serve
// multiple tasks (intent, NER, text classification, etc.)
use std::sync::Arc;
use parking_lot::Mutex;
use tokenizers::Tokenizer;
use crate::models::registry::ModelRegistry;
pub struct OrtModel {
pub session: Mutex<ort::session::Session>,
pub tokenizer: Option<Tokenizer>,
}
unsafe impl Send for OrtModel {}
unsafe impl Sync for OrtModel {}
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<OrtModel>, String> {
registry.get_or_load::<OrtModel>(model_id, |def| {
let model_dir = &def.path;
let onnx_path = model_dir.join("model.onnx");
info!("Loading ORT model from: {}", model_dir.display());
let session = ort::session::Session::builder()
.map_err(|e| format!("ORT session builder error: {}", e))?
.commit_from_file(&onnx_path)
.map_err(|e| format!("Failed to load ONNX model '{}': {}", onnx_path.display(), e))?;
let tokenizer_path = model_dir.join("tokenizer.json");
let tokenizer = if tokenizer_path.exists() {
Some(
Tokenizer::from_file(&tokenizer_path)
.map_err(|e| format!("Failed to load tokenizer: {}", e))?
)
} else {
None
};
info!("ORT model loaded: {}", def.name);
Ok(OrtModel { session: Mutex::new(session), tokenizer })
})
}

View file

@ -0,0 +1,33 @@
// vosk speech recognition model
use std::sync::Arc;
use vosk::Model;
use crate::models::registry::ModelRegistry;
pub struct VoskModel {
pub model: Model,
}
unsafe impl Send for VoskModel {}
unsafe impl Sync for VoskModel {}
// load a vosk model by path through the registry.
// vosk models aren't in the catalog (they use their own directory structure),
// so we pass the path directly and use model_id for dedup.
// @ToDo: Consider moving to catalog
pub fn load(registry: &ModelRegistry, model_id: &str, model_path: &str) -> Result<Arc<VoskModel>, String> {
// check if already loaded
if let Some(existing) = registry.get::<VoskModel>(model_id) {
info!("Vosk model '{}' already loaded, reusing", model_id);
return Ok(existing);
}
info!("Loading Vosk model from: {}", model_path);
let model = Model::new(model_path)
.ok_or_else(|| format!("Failed to load Vosk model from: {}", model_path))?;
info!("Vosk model loaded: {}", model_id);
Ok(registry.insert(model_id, VoskModel { model }))
}

View file

@ -0,0 +1,108 @@
use std::any::Any;
use std::collections::HashMap;
use std::sync::Arc;
use parking_lot::{Mutex, RwLock};
use super::structs::ModelDef;
// central model registry. loads models once and shares them between components.
// completely type-agnostic
pub struct ModelRegistry {
loaded: Mutex<HashMap<String, Arc<dyn Any + Send + Sync>>>,
catalog: RwLock<Vec<ModelDef>>,
}
impl ModelRegistry {
pub fn new() -> Self {
Self {
loaded: Mutex::new(HashMap::new()),
catalog: RwLock::new(Vec::new()),
}
}
pub fn set_catalog(&self, defs: Vec<ModelDef>) {
*self.catalog.write() = defs;
}
// read access to catalog without cloning the whole vec
pub fn with_catalog<R>(&self, f: impl FnOnce(&[ModelDef]) -> R) -> R {
f(&self.catalog.read())
}
pub fn get_model_def(&self, id: &str) -> Option<ModelDef> {
self.catalog.read().iter().find(|m| m.id == id).cloned()
}
// get a loaded model, downcasted to the expected type
pub fn get<T: 'static + Send + Sync>(&self, id: &str) -> Option<Arc<T>> {
self.loaded.lock()
.get(id)?
.clone()
.downcast::<T>()
.ok()
}
// get or load a model. if two components request the same id,
// the model only loads once.
//
// the lock is released before calling the loader to avoid deadlocks
// if the loader tries to load a dependency through the registry.
pub fn get_or_load<T: 'static + Send + Sync>(
&self,
id: &str,
loader: impl FnOnce(&ModelDef) -> Result<T, String>,
) -> Result<Arc<T>, String> {
// fast path: already loaded
if let Some(existing) = self.get::<T>(id) {
info!("Model '{}' already loaded, reusing", id);
return Ok(existing);
}
// grab model def (releases catalog lock immediately)
let def = self.get_model_def(id)
.ok_or_else(|| format!("Model '{}' not found in catalog", id))?;
// run loader without holding any lock
info!("Loading model '{}' from {:?}...", id, def.path);
let model = loader(&def)?;
let arc = Arc::new(model);
// insert (check again in case another thread loaded it meanwhile)
let mut map = self.loaded.lock();
if let Some(existing) = map.get(id) {
if let Ok(typed) = existing.clone().downcast::<T>() {
info!("Model '{}' was loaded by another thread, reusing", id);
return Ok(typed);
}
}
map.insert(id.to_string(), arc.clone());
info!("Model '{}' loaded and registered", id);
Ok(arc)
}
// insert a model directly (for models not in the catalog,
// or loaded through non-standard means like async init)
pub fn insert<T: 'static + Send + Sync>(&self, id: &str, model: T) -> Arc<T> {
let arc = Arc::new(model);
self.loaded.lock().insert(id.to_string(), arc.clone());
arc
}
pub fn unload(&self, id: &str) -> bool {
let removed = self.loaded.lock().remove(id).is_some();
if removed {
info!("Model '{}' unloaded from registry", id);
}
removed
}
pub fn is_loaded(&self, id: &str) -> bool {
self.loaded.lock().contains_key(id)
}
pub fn loaded_ids(&self) -> Vec<String> {
self.loaded.lock().keys().cloned().collect()
}
}

View file

@ -0,0 +1,38 @@
use std::path::PathBuf;
use serde::{Serialize, Deserialize};
// tasks that components can request a backend for
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Task {
Intent,
Slots,
Vad,
NoiseSuppression,
Stt,
}
// metadata about a model, parsed from model.toml on disk
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelDef {
pub id: String,
pub name: String,
pub tasks: Vec<Task>,
#[serde(default)]
pub description: String,
// set at runtime after scanning the folder
#[serde(skip)]
pub path: PathBuf,
}
// a selectable option for a task (shown in UI / stored in settings)
#[derive(Debug, Clone, Serialize)]
pub struct BackendOption {
pub id: String,
pub name: String,
// if Some, this option uses a model from the registry.
// if None, it's a code-only backend (like energy VAD) or disabled.
pub model_id: Option<String>,
}

View file

@ -0,0 +1,113 @@
use std::fs;
use std::path::{Path, PathBuf};
use crate::{APP_DIR, config};
#[derive(Debug, Clone)]
pub struct VoskModelInfo {
pub name: String, // folder name: "vosk-model-small-ru-0.22"
pub path: PathBuf, // full path
pub language: String, // extracted from name: "ru"
pub size: String, // "small", "large", etc.
}
// Scan for available Vosk models
pub fn scan_vosk_models() -> Vec<VoskModelInfo> {
let models_dir = {
APP_DIR.join(config::VOSK_MODELS_PATH)
};
let mut models = Vec::new();
info!("VOSK MODELS DIR: {}", models_dir.display());
if !models_dir.exists() {
warn!("Vosk models directory not found: {}", models_dir.display());
return models;
}
let entries = match fs::read_dir(models_dir) {
Ok(e) => e,
Err(e) => {
warn!("Failed to read vosk models directory: {}", e);
return models;
}
};
for entry in entries {
let entry = match entry {
Ok(e) => e,
Err(_) => continue,
};
let path = entry.path();
// must be a directory
if !path.is_dir() {
continue;
}
// check if it looks like a vosk model (has am/conf/graph folders or similar)
if !is_vosk_model(&path) {
continue;
}
let name = path.file_name()
.and_then(|n| n.to_str())
.unwrap_or("")
.to_string();
let (language, size) = parse_model_name(&name);
models.push(VoskModelInfo {
name,
path,
language,
size,
});
}
models.sort_by(|a, b| a.name.cmp(&b.name));
models
}
// Check if directory looks like a Vosk model
fn is_vosk_model(path: &Path) -> bool {
// vosk models typically have these subdirectories
path.join("am").exists() ||
path.join("conf").exists() ||
path.join("graph").exists() ||
path.join("ivector").exists()
}
// Extract language and size from model name
// e.g., "vosk-model-small-ru-0.22" -> ("ru", "small")
fn parse_model_name(name: &str) -> (String, String) {
let parts: Vec<&str> = name.split('-').collect();
let mut language = String::from("unknown");
let mut size = String::from("unknown");
// look for common size indicators
for part in &parts {
match *part {
"small" | "big" | "large" | "lgraph" => size = part.to_string(),
// language codes are usually 2 letters
s if s.len() == 2 && s.chars().all(|c| c.is_alphabetic()) => {
language = s.to_string();
}
_ => {}
}
}
(language, size)
}
// Get model path by name
pub fn get_model_path(model_name: &str) -> Option<PathBuf> {
let path = Path::new(config::VOSK_MODELS_PATH).join(model_name);
if path.exists() && path.is_dir() {
Some(path)
} else {
None
}
}

View file

@ -0,0 +1,171 @@
mod pvrecorder;
// mod cpal;
// mod portaudio;
use once_cell::sync::OnceCell;
use crate::{config, config::structs::RecorderType, DB};
static RECORDER_TYPE: OnceCell<RecorderType> = OnceCell::new();
static FRAME_LENGTH: OnceCell<u32> = OnceCell::new();
pub fn init() -> Result<(), ()> {
// set default recorder type
// @TODO. Make it configurable?
RECORDER_TYPE.set(config::DEFAULT_RECORDER_TYPE).unwrap();
// some info
info!("Loading recorder ...");
info!("Available audio_devices are:\n{:?}", get_audio_devices());
// load given recorder
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
// Init Pv Recorder
info!("Initializing PvRecorder recording backend.");
FRAME_LENGTH.set(512u32).unwrap(); // pvrecorder requires frame buffer of 512
let selected_microphone = get_selected_microphone_index();
match pvrecorder::init_microphone(
selected_microphone,
FRAME_LENGTH.get().unwrap().to_owned(),
) {
false => {
error!("Recorder initialization failed.");
return Err(());
}
_ => {
info!(
"Recorder initialization success. Listening to microphone ({}): {}",
selected_microphone,
get_audio_device_name(selected_microphone)
);
}
}
}
RecorderType::PortAudio => {
// Init PortAudio
info!("Initializing PortAudio recording backend");
todo!();
// match portaudio::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
// false => {
// // Switch to PortAudio recorder
// error!("PortAudio audio backend failed.");
// },
// _ => ()
// }
}
RecorderType::Cpal => {
// Init CPAL
info!("Initializing CPAL recording backend");
todo!();
// match cpal::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
// false => {
// // Switch to CPAL recorder
// error!("CPAL audio backend failed.");
// },
// _ => ()
// }
}
}
Ok(())
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
pvrecorder::read_microphone(frame_buffer);
}
RecorderType::PortAudio => {
todo!();
// portaudio::read_microphone(frame_buffer);
}
RecorderType::Cpal => {
// cpal::read_microphone(frame_buffer);
panic!("Cpal should be used via callback assignment");
}
}
}
pub fn start_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
return pvrecorder::start_recording(
get_selected_microphone_index(),
FRAME_LENGTH.get().unwrap().to_owned(),
);
}
RecorderType::PortAudio => {
todo!();
// portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
}
RecorderType::Cpal => {
todo!();
// cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
}
}
}
pub fn stop_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => pvrecorder::stop_recording(),
RecorderType::PortAudio => {
todo!();
// portaudio::stop_recording();
}
RecorderType::Cpal => {
todo!();
// cpal::stop_recording();
}
}
}
pub fn get_selected_microphone_index() -> i32 {
let idx = DB.get().unwrap().read().microphone;
if idx > 0 {
// validate that this microphone is actually in the list
let devices = get_audio_devices();
if (idx as usize) >= devices.len() {
warn!("Microphone index {} not found ({} available), falling back to default",
idx, devices.len());
return -1;
}
}
idx
}
pub fn get_audio_devices() -> Vec<String> {
match RECORDER_TYPE.get() {
Some(RecorderType::PvRecorder) => pvrecorder::list_audio_devices(),
Some(RecorderType::PortAudio) => {
todo!();
}
Some(RecorderType::Cpal) => {
todo!();
}
None => {
// not initialized yet, default to pvrecorder
pvrecorder::list_audio_devices()
}
}
}
pub fn get_audio_device_name(idx: i32) -> String {
match RECORDER_TYPE.get() {
Some(RecorderType::PvRecorder) => pvrecorder::get_audio_device_name(idx),
Some(RecorderType::PortAudio) => {
todo!();
}
Some(RecorderType::Cpal) => {
todo!();
}
None => {
// not initialized yet, default to pvrecorder
pvrecorder::get_audio_device_name(idx)
}
}
}

View file

@ -0,0 +1,191 @@
/*
Abandoned temporary.
Problems with frame size.
*/
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{BufferSize, StreamConfig, SampleRate, Host, Device, Stream, SampleFormat};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::Arc;
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
use crate::tauri_commands::cpal_data_callback;
static HOST: OnceCell<Host> = OnceCell::new();
thread_local!(static RECORDER: OnceCell<ArcSwap<Stream>> = OnceCell::new());
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
// init host & frame buffer for the callback
if HOST.get().is_none() {
HOST.set(cpal::default_host());
// FRAME_BUFFER.set(Mutex::new(vec![0; FRAME_LENGTH.load(Ordering::SeqCst) as usize]));
}
// init microphone
RECORDER.with(|recorder| {
match recorder.get().is_none() {
true => {
if let Some(device) = get_device(device_index as usize) {
// store
recorder.set(ArcSwap::from_pointee(create_stream(device, frame_length)));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
} else {
false
}
},
false => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
stop_recording();
}
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
if let Some(device) = get_device(device_index as usize) {
// store
recorder.get().unwrap().store(Arc::new(create_stream(device, frame_length)));
// success
return true
} else {
return false
}
}
// success
true
}
}
})
}
fn create_stream(device: Device, frame_length: u32) -> Stream {
// get default input stream config
// let default_config = device.default_input_config().unwrap();
// create config for the stream
// let config: StreamConfig = StreamConfig {
// channels: default_config.channels(),
// sample_rate: SampleRate(16000),
// buffer_size: BufferSize::Fixed(frame_length)
// };
let config = device
.default_input_config()
.expect("Failed to load default input config");
let channels = config.channels();
let err_fn = move |err| {
eprintln!("an error occurred on stream: {}", err);
};
match config.sample_format() {
SampleFormat::F32 => device.build_input_stream(
&config.into(),
move |data: &[f32], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
SampleFormat::U16 => device.build_input_stream(
&config.into(),
move |data: &[u16], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
SampleFormat::I16 => device.build_input_stream(
&config.into(),
move |data: &[i16], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
_ => todo!()
}.unwrap()
}
pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
let mut result = Vec::with_capacity(input_data.len() / 2);
result.extend(
input_data
.chunks_exact(2)
.map(|chunk| chunk[0] / 2 + chunk[1] / 2),
);
result
}
fn get_device(device_index: usize) -> Option<Device> {
if let Some(device) = HOST.get().unwrap().input_devices().expect("Get devices error ...").nth(device_index) {
Some(device)
} else {
if let Some(default) = HOST.get().unwrap().default_input_device() {
Some(default)
} else {
error!("No default input device ...");
None
}
}
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.with(|recorder| {
match recorder.get().unwrap().load().play() {
Err(msg) => {
error!("[CPAL] Audio stream PLAY error ... {:?}", msg);
},
_ => ()
};
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
});
}
pub fn stop_recording() {
// ensure microphone is initialized
RECORDER.with(|recorder| {
if recorder.get().is_some() && IS_RECORDING.load(Ordering::SeqCst) {
// pause instead of stop
match recorder.get().unwrap().load().pause() {
Err(msg) => {
error!("[CPAL] Audio stream PAUSE error ... {:?}", msg);
},
_ => ()
};
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
});
}

View file

@ -0,0 +1,205 @@
/*
Abandoned temporary.
*/
use portaudio as pa;
use pa::{DeviceIndex, Stream};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::{Arc, Mutex};
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
thread_local!(static RECORDER: OnceCell<ArcSwap<Mutex<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>>>> = OnceCell::new());
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
const CHANNELS: i32 = 1;
const SAMPLE_RATE: f64 = 16_000.0;
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
RECORDER.with(|r| {
match r.get().is_none() {
true => {
match create_stream(device_index, frame_length) {
Ok(stream) => {
// store
r.set(ArcSwap::from_pointee(Mutex::new(stream)));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
},
Err(msg) => {
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
// fail
false
}
}
},
_ => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
// RECORDER.get().unwrap().load().stop().expect("Failed to start audio recording!");
stop_recording();
}
// store
match create_stream(device_index, frame_length) {
Ok(stream) => {
// store new stream
r.get().unwrap().store(Arc::new(Mutex::new(stream)));
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
return true
},
Err(msg) => {
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
// fail
return false
}
}
}
// success
true
}
}
})
}
fn create_stream(device_index: i32, frame_length: u32) -> Result<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>, pa::Error> {
let pa_recorder: Result<pa::PortAudio, pa::Error> = pa::PortAudio::new();
match pa_recorder {
Ok(pa) => {
let input_settings = match get_input_settings(DeviceIndex(device_index as u32), &pa, SAMPLE_RATE, frame_length, CHANNELS) {
Ok(settings) => settings,
Err(error) => panic!("{}", String::from(error))
};
// Construct a stream with input and output sample types of i16
match pa.open_blocking_stream(input_settings) {
Ok(strm) => Ok(strm),
Err(error) => panic!("{}", error.to_string()),
}
},
Err(msg) => Err(msg)
}
}
fn get_input_latency(audio_port: &pa::PortAudio, input_index: pa::DeviceIndex) -> Result<f64, String>
{
let input_device_information = audio_port.device_info(input_index).or_else(|error| Err(String::from(format!("{}", error))));
Ok(input_device_information.unwrap().default_low_input_latency)
}
fn get_input_stream_parameters(input_index: pa::DeviceIndex, latency: f64, channels: i32) -> Result<pa::StreamParameters<i16>, String>
{
const INTERLEAVED: bool = true;
Ok(pa::StreamParameters::<i16>::new(input_index, channels, INTERLEAVED, latency))
}
fn get_input_settings(input_index: pa::DeviceIndex, audio_port: &pa::PortAudio, sample_rate: f64, frames: u32, channels: i32) -> Result<pa::InputStreamSettings<i16>, String>
{
Ok(
pa::InputStreamSettings::new(
(get_input_stream_parameters(
input_index,
(get_input_latency(
&audio_port,
input_index,
))?,
channels
))?,
sample_rate,
frames,
)
)
}
// We'll use this function to wait for read/write availability.
fn wait_for_stream<F>(f: F, name: &str) -> u32
where
F: Fn() -> Result<pa::StreamAvailable, pa::error::Error>,
{
loop {
match f() {
Ok(available) => match available {
pa::StreamAvailable::Frames(frames) => return frames as u32,
pa::StreamAvailable::InputOverflowed => println!("Input stream has overflowed"),
pa::StreamAvailable::OutputUnderflowed => {
println!("Output stream has underflowed")
}
},
Err(err) => panic!(
"An error occurred while waiting for the {} stream: {}",
name, err
),
}
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
RECORDER.with(|r| {
if r.get().is_some() {
let cell = r.get().unwrap().load();
let mut lock = cell.lock();
let stream = lock.as_mut().unwrap();
// read to frame buffer
let in_frames = wait_for_stream(|| stream.read_available(), "Read");
if in_frames > 0 {
// let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
// println!("Read {:?} frames from the input stream.", in_frames);
let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
println!("Read: {} (required {})", input_samples.len(), frame_buffer.len());
frame_buffer.copy_from_slice(input_samples.chunks(frame_buffer.len()).last().unwrap());
}
// r.get().unwrap().load().read(frame_buffer).expect("Failed to read audio frame");
}
});
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.with(|r| {
r.get().unwrap().load().lock().unwrap().start().expect("Failed to start audio recording!");
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
});
}
pub fn stop_recording() {
RECORDER.with(|r| {
if r.get().is_some() && IS_RECORDING.load(Ordering::SeqCst) {
// stop recording
let pa = r.get().unwrap().load();
r.get().unwrap().load().lock().unwrap().stop().expect("Failed to stop audio recording!");
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
});
}

View file

@ -0,0 +1,136 @@
use once_cell::sync::OnceCell;
use pv_recorder::{PvRecorder, PvRecorderBuilder};
use std::sync::atomic::{AtomicBool, Ordering};
static RECORDER: OnceCell<PvRecorder> = OnceCell::new();
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
if RECORDER.get().is_some() {
return true; // already initialized
}
// initialize
let pv_recorder = PvRecorderBuilder::new(frame_length as i32)
.device_index(device_index)
// .frame_length(frame_length as i32)
.init();
match pv_recorder {
Ok(pv) => {
// store
let _ = RECORDER.set(pv);
// success
true
}
Err(msg) => {
error!("Failed to initialize pvrecorder.\nError details: {:?}", msg);
// fail
false
}
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
if RECORDER.get().is_some() {
// read to frame buffer
let frame = RECORDER.get().unwrap().read();
match frame {
Ok(f) => {
frame_buffer.copy_from_slice(f.as_slice());
}
Err(msg) => {
// @TODO: Fix? PvRecorder always wait for PCM buffer size of 512.
error!("Failed to read audio frame. {:?}", msg);
}
}
}
}
pub fn start_recording(device_index: i32, frame_length: u32) -> Result<(), ()> {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
match RECORDER.get().unwrap().start() {
Ok(_) => {
info!("START recording from microphone ...");
// change recording state
IS_RECORDING.store(true, Ordering::SeqCst);
// success
Ok(())
}
Err(msg) => {
error!("Failed to START audio recording: {}", msg);
// fail
Err(())
}
}
}
pub fn stop_recording() -> Result<(), ()> {
// ensure microphone is initialized & recording is in process
if RECORDER.get().is_some() && IS_RECORDING.load(Ordering::SeqCst) {
// stop recording
match RECORDER.get().unwrap().stop() {
Ok(_) => {
info!("STOP recording from microphone ...");
// change recording state
IS_RECORDING.store(false, Ordering::SeqCst);
// success
return Ok(());
}
Err(msg) => {
error!("Failed to STOP audio recording: {}", msg);
// fail
return Err(());
}
}
}
Ok(()) // if already stopped or not yet initialized
}
pub fn list_audio_devices() -> Vec<String> {
let audio_devices = PvRecorderBuilder::default().get_available_devices();
match audio_devices {
Ok(audio_devices) => audio_devices,
Err(err) => {
error!("Failed to get audio devices: {}", err);
Vec::new()
},
}
}
pub fn get_audio_device_name(idx: i32) -> String {
if idx == -1 {
return String::from("System Default");
}
let audio_devices = list_audio_devices();
let mut first_device: String = String::new();
for (_idx, device) in audio_devices.iter().enumerate() {
if idx as usize == _idx {
return device.to_string();
}
if _idx == 0 {
first_device = device.to_string()
}
}
// return first device as default, if none were matched
first_device
}

View file

@ -0,0 +1,58 @@
mod gliner;
use std::collections::HashMap;
use once_cell::sync::OnceCell;
use crate::commands::{SlotDefinition, SlotValue};
use crate::{models, DB};
static BACKEND: OnceCell<String> = OnceCell::new();
pub fn init() -> Result<(), String> {
if BACKEND.get().is_some() {
return Ok(());
}
let backend = DB.get()
.map(|db| db.read().slots_backend.clone())
.unwrap_or_else(|| "none".to_string());
BACKEND.set(backend.clone()).map_err(|_| "Slot backend already set")?;
match backend.as_str() {
"none" => {
info!("Slot extraction disabled");
}
// any model ID is treated as a GLiNER model for now
model_id => {
info!("Initializing GLiNER slot extraction with model '{}'.", model_id);
let model = models::gliner::load(models::registry(), model_id)?;
gliner::init_with_model(model)?;
info!("GLiNER slot extraction initialized.");
}
}
Ok(())
}
pub fn extract(
text: &str,
slots: &HashMap<String, SlotDefinition>,
) -> HashMap<String, SlotValue> {
if slots.is_empty() {
return HashMap::new();
}
match BACKEND.get().map(|s| s.as_str()).unwrap_or("none") {
"none" => HashMap::new(),
_ => {
match gliner::extract(text, slots) {
Ok(result) => result,
Err(e) => {
error!("GLiNER slot extraction failed: {}", e);
HashMap::new()
}
}
}
}
}

View file

@ -0,0 +1,395 @@
// BASED ON: gline-rs crate source code
// https://github.com/fbilhaut/gline-rs
use std::collections::HashMap;
use std::sync::Arc;
use once_cell::sync::OnceCell;
use ndarray::Array;
use ort::value::Tensor;
use crate::commands::{SlotDefinition, SlotValue};
use crate::models::gliner::GlinerModel;
static MODEL: OnceCell<Arc<GlinerModel>> = OnceCell::new();
// GLiNER defaults
const THRESHOLD: f32 = 0.3;
const MAX_WIDTH: usize = 12;
const MAX_LENGTH: usize = 512;
const MIN_CONFIDENCE: f32 = 0.4;
pub fn init_with_model(model: Arc<GlinerModel>) -> Result<(), String> {
MODEL.set(model).map_err(|_| "GLiNER model already initialized".to_string())?;
info!("GLiNER slot extraction ready");
Ok(())
}
// word splitting
struct WordToken<'a> {
start: usize,
end: usize,
text: &'a str,
}
fn split_words<'a>(text: &'a str, model: &GlinerModel, limit: Option<usize>) -> Vec<WordToken<'a>> {
let mut tokens = Vec::new();
for m in model.splitter.find_iter(text) {
tokens.push(WordToken {
start: m.start(),
end: m.end(),
text: m.as_str(),
});
if let Some(lim) = limit {
if tokens.len() >= lim { break; }
}
}
tokens
}
// prompt construction
//
// GLiNER prompt format:
// [<<ENT>>, label1_w1, label1_w2, <<ENT>>, label2_w1, ..., <<SEP>>, word1, word2, ..., wordN]
fn build_prompt(entities: &[&str], words: &[WordToken]) -> (Vec<String>, usize) {
let mut prompt = Vec::with_capacity(entities.len() * 2 + 1 + words.len());
for entity in entities {
prompt.push("<<ENT>>".to_string());
prompt.push(entity.to_string());
}
prompt.push("<<SEP>>".to_string());
let entities_len = prompt.len();
for w in words {
prompt.push(w.text.to_string());
}
(prompt, entities_len)
}
// encoding
struct EncodedBatch {
input_ids: ndarray::Array2<i64>,
attention_masks: ndarray::Array2<i64>,
word_masks: ndarray::Array2<i64>,
text_lengths: ndarray::Array2<i64>,
num_words: usize,
}
fn encode_single(
model: &GlinerModel,
entities: &[&str],
words: &[WordToken],
) -> Result<EncodedBatch, String> {
let (prompt, ent_len) = build_prompt(entities, words);
let text_word_count = words.len();
let mut word_encodings: Vec<Vec<u32>> = Vec::with_capacity(prompt.len());
let mut total_tokens: usize = 2; // BOS + EOS
let mut entity_tokens: usize = 0;
for (pos, word) in prompt.iter().enumerate() {
let encoding = model.tokenizer.encode(word.as_str(), false)
.map_err(|e| format!("Tokenizer encode error: {}", e))?;
let ids = encoding.get_ids().to_vec();
total_tokens += ids.len();
if pos < ent_len {
entity_tokens += ids.len();
}
word_encodings.push(ids);
}
let text_offset = entity_tokens + 1;
if log::log_enabled!(log::Level::Debug) {
debug!("GLiNER prompt ({} total, ent_len={}, text_offset={}):", prompt.len(), ent_len, text_offset);
for (i, (word, enc)) in prompt.iter().zip(word_encodings.iter()).enumerate() {
debug!(" [{}]{} '{}' -> {:?}", i, if i < ent_len { " ENT" } else { " TXT" }, word, enc);
}
}
let mut input_ids = Array::zeros((1, total_tokens));
let mut attention_masks = Array::zeros((1, total_tokens));
let mut word_masks = Array::zeros((1, total_tokens));
let mut idx: usize = 0;
let mut word_id: i64 = 0;
// BOS
input_ids[[0, idx]] = 1;
attention_masks[[0, idx]] = 1;
idx += 1;
for word_enc in word_encodings.iter() {
for (token_idx, &token_id) in word_enc.iter().enumerate() {
input_ids[[0, idx]] = token_id as i64;
attention_masks[[0, idx]] = 1;
if idx >= text_offset && token_idx == 0 {
word_masks[[0, idx]] = word_id;
}
idx += 1;
}
if idx >= text_offset {
word_id += 1;
}
}
// EOS
input_ids[[0, idx]] = 2;
attention_masks[[0, idx]] = 1;
let mut text_lengths = Array::zeros((1, 1));
text_lengths[[0, 0]] = (text_word_count + 1) as i64;
if log::log_enabled!(log::Level::Debug) {
debug!("GLiNER input_ids: {:?}", input_ids.as_slice().unwrap());
debug!("GLiNER word_masks: {:?}", word_masks.as_slice().unwrap());
debug!("GLiNER text_lengths: {}", text_word_count);
}
Ok(EncodedBatch {
input_ids,
attention_masks,
word_masks,
text_lengths,
num_words: text_word_count + 1,
})
}
// span tensors
fn make_span_tensors(num_words: usize, max_width: usize) -> (ndarray::Array3<i64>, ndarray::Array2<bool>) {
let num_spans = num_words * max_width;
let mut span_idx = Array::zeros((1, num_spans, 2));
let mut span_mask = Array::from_elem((1, num_spans), false);
for start in 0..num_words {
let remaining = num_words - start;
let actual_max = max_width.min(remaining);
for width in 0..actual_max {
let dim = start * max_width + width;
span_idx[[0, dim, 0]] = start as i64;
span_idx[[0, dim, 1]] = (start + width) as i64;
span_mask[[0, dim]] = true;
}
}
(span_idx, span_mask)
}
// decode + greedy search
fn sigmoid(x: f32) -> f32 {
1.0 / (1.0 + (-x).exp())
}
struct Entity {
text: String,
label: String,
prob: f32,
start: usize,
end: usize,
}
fn decode_and_search(
logits_data: &[f32],
logits_shape: &[usize],
words: &[WordToken],
text: &str,
entities: &[&str],
max_width: usize,
threshold: f32,
) -> Vec<Entity> {
let num_tokens = words.len();
let dim_mw = logits_shape.get(2).copied().unwrap_or(0);
let dim_e = logits_shape.get(3).copied().unwrap_or(0);
let mut spans: Vec<Entity> = Vec::new();
for start in 1..=num_tokens {
let max_end = (start + max_width).min(num_tokens + 1);
for end in start..max_end {
let width = end - start;
for (class_idx, &entity_label) in entities.iter().enumerate() {
let flat_idx = start * dim_mw * dim_e + width * dim_e + class_idx;
if flat_idx >= logits_data.len() { continue; }
let raw_score = logits_data[flat_idx];
let prob = sigmoid(raw_score);
if prob >= threshold {
let w_start = start - 1;
let w_end = end - 1;
let start_offset = words[w_start].start;
let end_offset = words[w_end].end;
let span_text = text[start_offset..end_offset].to_string();
spans.push(Entity {
text: span_text,
label: entity_label.to_string(),
prob,
start: start_offset,
end: end_offset,
});
}
}
}
}
spans.sort_unstable_by(|a, b| (a.start, a.end).cmp(&(b.start, b.end)));
greedy_flat(spans)
}
// takes ownership, filters in place - no cloning
fn greedy_flat(mut spans: Vec<Entity>) -> Vec<Entity> {
if spans.len() <= 1 {
return spans;
}
let mut keep = vec![false; spans.len()];
let mut prev = 0usize;
for next in 1..spans.len() {
let no_overlap = spans[next].start >= spans[prev].end
|| spans[prev].start >= spans[next].end;
if no_overlap {
keep[prev] = true;
prev = next;
} else if spans[prev].prob < spans[next].prob {
prev = next;
}
}
keep[prev] = true;
let mut idx = 0;
spans.retain(|_| { let k = keep[idx]; idx += 1; k });
spans
}
// public extract API
pub fn extract(
text: &str,
slots: &HashMap<String, SlotDefinition>,
) -> Result<HashMap<String, SlotValue>, String> {
let model = MODEL.get().ok_or("GLiNER not initialized")?;
let mut label_to_slots: HashMap<&str, Vec<&str>> = HashMap::new();
for (slot_name, def) in slots {
if !def.entity.is_empty() {
label_to_slots
.entry(def.entity.as_str())
.or_default()
.push(slot_name.as_str());
}
}
if label_to_slots.is_empty() {
return Ok(HashMap::new());
}
let labels: Vec<&str> = label_to_slots.keys().copied().collect();
debug!("GLiNER extract: text='{}', labels={:?}", text, labels);
let words = split_words(text, &model, Some(MAX_LENGTH));
if words.is_empty() {
return Ok(HashMap::new());
}
let encoded = encode_single(&model, &labels, &words)?;
let (span_idx, span_mask) = make_span_tensors(encoded.num_words, MAX_WIDTH);
let t_input_ids = Tensor::from_array(encoded.input_ids).map_err(|e| format!("tensor: {}", e))?;
let t_attn = Tensor::from_array(encoded.attention_masks).map_err(|e| format!("tensor: {}", e))?;
let t_words = Tensor::from_array(encoded.word_masks).map_err(|e| format!("tensor: {}", e))?;
let t_lengths = Tensor::from_array(encoded.text_lengths).map_err(|e| format!("tensor: {}", e))?;
let t_span_idx = Tensor::from_array(span_idx).map_err(|e| format!("tensor: {}", e))?;
let t_span_mask = Tensor::from_array(span_mask).map_err(|e| format!("tensor: {}", e))?;
let mut session = model.session.lock();
let outputs = session.run(
ort::inputs! {
"input_ids" => t_input_ids,
"attention_mask" => t_attn,
"words_mask" => t_words,
"text_lengths" => t_lengths,
"span_idx" => t_span_idx,
"span_mask" => t_span_mask,
}
).map_err(|e| format!("ort inference error: {}", e))?;
let (shape, logits_data) = outputs["logits"]
.try_extract_tensor::<f32>()
.map_err(|e| format!("Failed to extract logits: {}", e))?;
let logits_shape: Vec<usize> = shape.iter().map(|&d| d as usize).collect();
// debug dump - gated so sigmoid/loop don't run in release
if log::log_enabled!(log::Level::Debug) {
debug!("GLiNER logits shape: {:?}, data len: {}", logits_shape, logits_data.len());
let max_logit = logits_data.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
debug!("GLiNER max logit: {:.4}, sigmoid: {:.4}", max_logit, sigmoid(max_logit));
let num_words = logits_shape.get(1).copied().unwrap_or(0);
let dim_mw = logits_shape.get(2).copied().unwrap_or(0);
let dim_e = logits_shape.get(3).copied().unwrap_or(0);
for start in 0..num_words {
for width in 0..dim_mw.min(num_words - start) {
for class_idx in 0..dim_e {
let flat_idx = start * dim_mw * dim_e + width * dim_e + class_idx;
if flat_idx < logits_data.len() {
let score = logits_data[flat_idx];
let prob = sigmoid(score);
if prob > 0.05 {
let end = start + width;
let w_start = if start < words.len() { words[start].text } else { "?" };
let w_end = if end < words.len() { words[end].text } else { "?" };
debug!(" span[{}..{}] '{}'->'{}' label={} score={:.3} prob={:.3}",
start, end, w_start, w_end, labels.get(class_idx).unwrap_or(&"?"), score, prob);
}
}
}
}
}
}
let entities = decode_and_search(
logits_data, &logits_shape, &words, text, &labels, MAX_WIDTH, THRESHOLD,
);
let mut result = HashMap::new();
for entity in &entities {
if entity.prob < MIN_CONFIDENCE {
continue;
}
debug!("GLiNER entity: '{}' -> '{}' ({:.1}%)",
entity.text, entity.label, entity.prob * 100.0);
if let Some(slot_names) = label_to_slots.get(entity.label.as_str()) {
for &slot_name in slot_names {
if !result.contains_key(slot_name) {
let value = parse_slot_value(&entity.text);
result.insert(slot_name.to_string(), value);
}
}
}
}
Ok(result)
}
fn parse_slot_value(text: &str) -> SlotValue {
if let Ok(n) = text.parse::<f64>() {
return SlotValue::Number(n);
}
SlotValue::Text(text.to_string())
}

View file

@ -0,0 +1,41 @@
#[cfg(feature = "vosk")]
mod vosk;
use crate::config;
use once_cell::sync::OnceCell;
use crate::config::structs::SpeechToTextEngine;
pub use self::vosk::init_vosk;
pub use self::vosk::recognize_wake_word;
pub use self::vosk::recognize_speech;
pub use self::vosk::reset_speech_recognizer;
pub use self::vosk::reset_wake_recognizer;
static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new();
pub fn init() -> Result<(), String> {
if STT_TYPE.get().is_some() {
return Ok(());
}
STT_TYPE.set(config::DEFAULT_SPEECH_TO_TEXT_ENGINE)
.map_err(|_| "STT type already set".to_string())?;
match STT_TYPE.get().unwrap() {
SpeechToTextEngine::Vosk => {
info!("Initializing Vosk STT backend.");
vosk::init_vosk()?;
info!("STT backend initialized.");
}
}
Ok(())
}
pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
if include_partial {
vosk::recognize_wake_word(data).map(|(text, _)| text)
} else {
vosk::recognize_speech(data)
}
}

View file

@ -0,0 +1,146 @@
use once_cell::sync::OnceCell;
use vosk::{DecodingState, Recognizer};
use std::sync::Arc;
use parking_lot::Mutex;
use crate::{vosk_models, i18n, config, models};
use crate::models::vosk::VoskModel;
use crate::DB;
// the model Arc keeps the vosk::Model alive for the recognizers
static VOSK_MODEL: OnceCell<Arc<VoskModel>> = OnceCell::new();
static WAKE_RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
static SPEECH_RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
pub fn init_vosk() -> Result<(), String> {
if VOSK_MODEL.get().is_some() {
return Ok(());
}
let model_path = get_configured_model_path()?;
let model_id = format!("vosk:{}", model_path.display());
// load through registry (shared if anything else needs the same model)
let vosk = models::vosk::load(
models::registry(),
&model_id,
model_path.to_str().unwrap(),
)?;
// language-specific wake grammar
let lang = i18n::get_language();
let wake_grammar = config::get_wake_grammar(&lang);
info!("Wake grammar for '{}': {:?}", lang, wake_grammar);
let mut wake_recognizer = Recognizer::new_with_grammar(&vosk.model, 16000.0, wake_grammar)
.ok_or("Failed to create wake word recognizer")?;
wake_recognizer.set_max_alternatives(1);
let mut speech_recognizer = Recognizer::new(&vosk.model, 16000.0)
.ok_or("Failed to create speech recognizer")?;
speech_recognizer.set_max_alternatives(config::VOSK_SPEECH_RECOGNIZER_MAX_ALTERNATIVES);
speech_recognizer.set_words(config::VOSK_SPEECH_RECOGNIZER_WORDS);
speech_recognizer.set_partial_words(config::VOSK_SPEECH_PARTIAL_WORDS);
VOSK_MODEL.set(vosk).map_err(|_| "Model already set")?;
WAKE_RECOGNIZER.set(Mutex::new(wake_recognizer)).map_err(|_| "Wake recognizer already set")?;
SPEECH_RECOGNIZER.set(Mutex::new(speech_recognizer)).map_err(|_| "Speech recognizer already set")?;
Ok(())
}
pub fn recognize_wake_word(data: &[i16]) -> Option<(String, f32)> {
let mut recognizer = WAKE_RECOGNIZER.get()?.lock();
match recognizer.accept_waveform(data) {
Ok(DecodingState::Running) => {
None
}
Ok(DecodingState::Finalized) => {
let result = recognizer.result();
if let Some(alternatives) = result.multiple() {
if let Some(best) = alternatives.alternatives.first() {
if !best.text.is_empty() {
return Some((best.text.to_string(), best.confidence));
}
}
}
None
}
_ => None,
}
}
pub fn recognize_speech(data: &[i16]) -> Option<String> {
let mut recognizer = SPEECH_RECOGNIZER.get()?.lock();
match recognizer.accept_waveform(data) {
Ok(DecodingState::Finalized) => {
recognizer.result()
.multiple()
.and_then(|m| m.alternatives.first().map(|a| a.text.to_string()))
}
_ => None,
}
}
pub fn reset_speech_recognizer() {
if let Some(recognizer) = SPEECH_RECOGNIZER.get() {
recognizer.lock().reset();
}
}
pub fn reset_wake_recognizer() {
if let Some(recognizer) = WAKE_RECOGNIZER.get() {
recognizer.lock().reset();
}
}
fn get_configured_model_path() -> Result<std::path::PathBuf, String> {
// try to get from settings
if let Some(db) = DB.get() {
let settings = db.read();
if !settings.vosk_model.is_empty() {
if let Some(path) = vosk_models::get_model_path(&settings.vosk_model) {
return Ok(path);
}
warn!("Configured Vosk model '{}' not found, falling back to auto-detect", settings.vosk_model);
}
}
// auto-detect: prefer model matching current language
let available = vosk_models::scan_vosk_models();
let language = i18n::get_language();
let lang_code = match language.as_str() {
"ru" => "ru",
"en" => "us",
"ua" => "uk",
other => other,
};
if let Some(matched) = available.iter().find(|m| m.language == lang_code) {
info!("Auto-detected Vosk model for '{}': {}", language, matched.name);
return Ok(matched.path.clone());
}
if let Some(first) = available.first() {
info!("Auto-detected Vosk model (no language match): {}", first.name);
return Ok(first.path.clone());
}
// fallback to legacy path
let legacy_path = std::path::Path::new(config::VOSK_MODEL_PATH);
if legacy_path.exists() {
return Ok(legacy_path.to_path_buf());
}
Err("No Vosk models found".into())
}

View file

@ -0,0 +1,2 @@
pub mod structs;
pub use structs::*;

Some files were not shown because too many files have changed in this diff Show more