Compare commits
93 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1b055fa08c | ||
|
|
9a1c047282 | ||
|
|
d23c7fbd12 | ||
|
|
acebb02a8e | ||
|
|
a3389f8bf7 | ||
|
|
37c8371f3f | ||
|
|
3f19366ad0 | ||
|
|
ff70e6019f | ||
|
|
3429d934e3 | ||
|
|
8a2b3f6206 | ||
|
|
e5aaa7e064 | ||
|
|
176f405310 | ||
|
|
1e2c883e68 | ||
|
|
40f2f8b5f9 | ||
|
|
b2045cf950 | ||
|
|
23c57b855b | ||
|
|
5b7a2d15d4 | ||
|
|
3d9a95a2db | ||
|
|
02f5829aaa | ||
|
|
d6792fc038 | ||
|
|
3b4835935f | ||
|
|
a58b832b44 | ||
|
|
cb140b1c64 | ||
|
|
fb5c096327 | ||
|
|
893a83d197 | ||
|
|
08245d7de6 | ||
|
|
3c81525ab6 | ||
|
|
54f90bb7c8 | ||
|
|
8883492715 | ||
|
|
61009fd871 | ||
|
|
520b98143f | ||
|
|
a8ff3442ff | ||
|
|
b9d5f41bbd | ||
|
|
8e830334e8 | ||
|
|
4815c7f9bb | ||
|
|
13d6686dd9 | ||
|
|
acc806d403 | ||
|
|
b10b5f9ab1 | ||
|
|
baf84e0d8f | ||
|
|
7440baa1b2 | ||
|
|
11c2500d9c | ||
|
|
e2370dc046 | ||
|
|
47b7e7a65d | ||
|
|
412acb7e2d | ||
|
|
adec595cfa | ||
|
|
eb0d40bae6 | ||
|
|
a640e6caea | ||
|
|
cab53abcbe | ||
|
|
36acacf259 | ||
|
|
03da13ed73 | ||
|
|
5f1f9ce321 | ||
|
|
0c28304840 | ||
|
|
091a41ac33 | ||
|
|
61b7a79455 | ||
|
|
88d73eeb10 | ||
|
|
cf018240c5 | ||
|
|
09f72622bc | ||
|
|
3ffbe876f3 | ||
|
|
0fbb5ac5aa | ||
|
|
76aca1e34a | ||
|
|
1a146e9505 | ||
|
|
840e132180 | ||
|
|
b3fd107707 | ||
|
|
530682ed05 | ||
|
|
7e6aabe30b | ||
|
|
80e0cb8baf | ||
|
|
17a4033948 | ||
|
|
b4c11261b5 | ||
|
|
5a550087c4 | ||
|
|
3c7df5fc6e | ||
|
|
9e42085ff3 | ||
|
|
1b835e92b5 | ||
|
|
15eacbd20f | ||
|
|
4e860d63c3 | ||
|
|
4f3d572b26 | ||
|
|
d4fcb0ea9c | ||
|
|
38dd734352 | ||
|
|
8dc6efd81a | ||
|
|
488f5c0786 | ||
|
|
fc764c0c85 | ||
|
|
1033840325 | ||
|
|
511f2eaa5c | ||
|
|
2255479ec3 | ||
|
|
1a7b108091 | ||
|
|
4de585fb0a | ||
|
|
a988f3edc3 | ||
|
|
8168475643 | ||
|
|
5f35beb5cf | ||
|
|
4e1413d400 | ||
|
|
791bd204ee | ||
|
|
9e6aa9458a | ||
|
|
d9c0ac77c9 | ||
|
|
2c2f0e71fd |
485 changed files with 674605 additions and 12638 deletions
44
.gitignore
vendored
44
.gitignore
vendored
|
|
@ -7,12 +7,8 @@ yarn-error.log*
|
||||||
pnpm-debug.log*
|
pnpm-debug.log*
|
||||||
lerna-debug.log*
|
lerna-debug.log*
|
||||||
|
|
||||||
node_modules
|
|
||||||
dist
|
|
||||||
dist-ssr
|
|
||||||
*.local
|
|
||||||
|
|
||||||
# Editor directories and files
|
# Editor directories and files
|
||||||
|
.vscode
|
||||||
.vscode/*
|
.vscode/*
|
||||||
!.vscode/extensions.json
|
!.vscode/extensions.json
|
||||||
.idea
|
.idea
|
||||||
|
|
@ -24,5 +20,39 @@ dist-ssr
|
||||||
*.sw?
|
*.sw?
|
||||||
|
|
||||||
# Etc
|
# Etc
|
||||||
.routify
|
__other
|
||||||
build
|
psd
|
||||||
|
list.py
|
||||||
|
tree.txt
|
||||||
|
|
||||||
|
# Rust / Cargo 🔧
|
||||||
|
# Ignore Cargo build output, generated files, and common Rust artifacts
|
||||||
|
/target/
|
||||||
|
**/target/
|
||||||
|
/target/doc/
|
||||||
|
|
||||||
|
# Binaries and dynamic libraries
|
||||||
|
*.exe
|
||||||
|
# *.dll
|
||||||
|
*.so
|
||||||
|
*.dylib
|
||||||
|
|
||||||
|
# Static libraries
|
||||||
|
*.a
|
||||||
|
# *.lib
|
||||||
|
|
||||||
|
# Rust compilation artifacts
|
||||||
|
*.rlib
|
||||||
|
*.rmeta
|
||||||
|
|
||||||
|
# Backup files and editor temp files
|
||||||
|
*.rs.bk
|
||||||
|
|
||||||
|
# Ignore packaged crates
|
||||||
|
*.crate
|
||||||
|
|
||||||
|
# Ignore AI models
|
||||||
|
/resources/models/*
|
||||||
|
|
||||||
|
# Tauri-generated platform schemas (regenerated on every build)
|
||||||
|
crates/jarvis-gui/gen/schemas/*-schema.json
|
||||||
|
|
|
||||||
7
.vscode/extensions.json
vendored
7
.vscode/extensions.json
vendored
|
|
@ -1,7 +0,0 @@
|
||||||
{
|
|
||||||
"recommendations": [
|
|
||||||
"svelte.svelte-vscode",
|
|
||||||
"tauri-apps.tauri-vscode",
|
|
||||||
"rust-lang.rust-analyzer"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
9593
Cargo.lock
generated
Normal file
9593
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load diff
57
Cargo.toml
Normal file
57
Cargo.toml
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
[workspace]
|
||||||
|
members = [
|
||||||
|
"crates/jarvis-core",
|
||||||
|
"crates/jarvis-app",
|
||||||
|
"crates/jarvis-gui",
|
||||||
|
"crates/jarvis-cli"
|
||||||
|
]
|
||||||
|
resolver = "2"
|
||||||
|
|
||||||
|
[workspace.package]
|
||||||
|
version = "0.1.0"
|
||||||
|
authors = ["Abraham Tugalov (original)", "Bossiara13 (fork)"]
|
||||||
|
license = "GPL-3.0-only"
|
||||||
|
repository = "https://github.com/Priler/jarvis"
|
||||||
|
edition = "2021"
|
||||||
|
|
||||||
|
[workspace.dependencies]
|
||||||
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
|
serde_json = "1.0"
|
||||||
|
serde_yaml = "0.9"
|
||||||
|
once_cell = "1.19"
|
||||||
|
log = "0.4"
|
||||||
|
rand = "0.8"
|
||||||
|
seqdiff = "0.3"
|
||||||
|
hound = "3.5"
|
||||||
|
platform-dirs = "0.3"
|
||||||
|
rodio = "0.21"
|
||||||
|
kira = "0.11"
|
||||||
|
pv_recorder = { git = "https://github.com/Priler/pvrecorder" }
|
||||||
|
vosk = "0.3"
|
||||||
|
rustpotter = { git = "https://github.com/Priler/rustpotter" }
|
||||||
|
image = "0.25"
|
||||||
|
parking_lot = "0.12.5"
|
||||||
|
toml = "0.9.8"
|
||||||
|
sha2 = "0.10"
|
||||||
|
nnnoiseless = "0.5"
|
||||||
|
sysinfo = "0.37.2"
|
||||||
|
tokio-tungstenite = "0.28.0"
|
||||||
|
futures-util = "0.3"
|
||||||
|
fluent = "0.17.0"
|
||||||
|
fluent-bundle = "0.16.0"
|
||||||
|
unic-langid = "0.9"
|
||||||
|
chrono = "0.4"
|
||||||
|
mlua = { version = "0.11.5", features = ["lua55", "vendored", "async", "serde"] }
|
||||||
|
reqwest = { version = "0.13.1", features = ["blocking", "json"] }
|
||||||
|
thiserror = "2"
|
||||||
|
tempfile = "^3.24"
|
||||||
|
winrt-notification = "0.5"
|
||||||
|
|
||||||
|
fastembed = { version = "^5.8.1", default-features = false, features = ["ort-download-binaries"] }
|
||||||
|
ort = { version = "=2.0.0-rc.11" }
|
||||||
|
ndarray = "0.17"
|
||||||
|
tokenizers = { version = "0.22", default-features = false }
|
||||||
|
regex = "1"
|
||||||
|
sys-locale = "0.3"
|
||||||
|
webrtc-vad = "0.4"
|
||||||
|
|
||||||
437
LICENSE.txt
Normal file
437
LICENSE.txt
Normal file
|
|
@ -0,0 +1,437 @@
|
||||||
|
Attribution-NonCommercial-ShareAlike 4.0 International
|
||||||
|
|
||||||
|
=======================================================================
|
||||||
|
|
||||||
|
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
||||||
|
does not provide legal services or legal advice. Distribution of
|
||||||
|
Creative Commons public licenses does not create a lawyer-client or
|
||||||
|
other relationship. Creative Commons makes its licenses and related
|
||||||
|
information available on an "as-is" basis. Creative Commons gives no
|
||||||
|
warranties regarding its licenses, any material licensed under their
|
||||||
|
terms and conditions, or any related information. Creative Commons
|
||||||
|
disclaims all liability for damages resulting from their use to the
|
||||||
|
fullest extent possible.
|
||||||
|
|
||||||
|
Using Creative Commons Public Licenses
|
||||||
|
|
||||||
|
Creative Commons public licenses provide a standard set of terms and
|
||||||
|
conditions that creators and other rights holders may use to share
|
||||||
|
original works of authorship and other material subject to copyright
|
||||||
|
and certain other rights specified in the public license below. The
|
||||||
|
following considerations are for informational purposes only, are not
|
||||||
|
exhaustive, and do not form part of our licenses.
|
||||||
|
|
||||||
|
Considerations for licensors: Our public licenses are
|
||||||
|
intended for use by those authorized to give the public
|
||||||
|
permission to use material in ways otherwise restricted by
|
||||||
|
copyright and certain other rights. Our licenses are
|
||||||
|
irrevocable. Licensors should read and understand the terms
|
||||||
|
and conditions of the license they choose before applying it.
|
||||||
|
Licensors should also secure all rights necessary before
|
||||||
|
applying our licenses so that the public can reuse the
|
||||||
|
material as expected. Licensors should clearly mark any
|
||||||
|
material not subject to the license. This includes other CC-
|
||||||
|
licensed material, or material used under an exception or
|
||||||
|
limitation to copyright. More considerations for licensors:
|
||||||
|
wiki.creativecommons.org/Considerations_for_licensors
|
||||||
|
|
||||||
|
Considerations for the public: By using one of our public
|
||||||
|
licenses, a licensor grants the public permission to use the
|
||||||
|
licensed material under specified terms and conditions. If
|
||||||
|
the licensor's permission is not necessary for any reason--for
|
||||||
|
example, because of any applicable exception or limitation to
|
||||||
|
copyright--then that use is not regulated by the license. Our
|
||||||
|
licenses grant only permissions under copyright and certain
|
||||||
|
other rights that a licensor has authority to grant. Use of
|
||||||
|
the licensed material may still be restricted for other
|
||||||
|
reasons, including because others have copyright or other
|
||||||
|
rights in the material. A licensor may make special requests,
|
||||||
|
such as asking that all changes be marked or described.
|
||||||
|
Although not required by our licenses, you are encouraged to
|
||||||
|
respect those requests where reasonable. More_considerations
|
||||||
|
for the public:
|
||||||
|
wiki.creativecommons.org/Considerations_for_licensees
|
||||||
|
|
||||||
|
=======================================================================
|
||||||
|
|
||||||
|
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
||||||
|
Public License
|
||||||
|
|
||||||
|
By exercising the Licensed Rights (defined below), You accept and agree
|
||||||
|
to be bound by the terms and conditions of this Creative Commons
|
||||||
|
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
||||||
|
("Public License"). To the extent this Public License may be
|
||||||
|
interpreted as a contract, You are granted the Licensed Rights in
|
||||||
|
consideration of Your acceptance of these terms and conditions, and the
|
||||||
|
Licensor grants You such rights in consideration of benefits the
|
||||||
|
Licensor receives from making the Licensed Material available under
|
||||||
|
these terms and conditions.
|
||||||
|
|
||||||
|
|
||||||
|
Section 1 -- Definitions.
|
||||||
|
|
||||||
|
a. Adapted Material means material subject to Copyright and Similar
|
||||||
|
Rights that is derived from or based upon the Licensed Material
|
||||||
|
and in which the Licensed Material is translated, altered,
|
||||||
|
arranged, transformed, or otherwise modified in a manner requiring
|
||||||
|
permission under the Copyright and Similar Rights held by the
|
||||||
|
Licensor. For purposes of this Public License, where the Licensed
|
||||||
|
Material is a musical work, performance, or sound recording,
|
||||||
|
Adapted Material is always produced where the Licensed Material is
|
||||||
|
synched in timed relation with a moving image.
|
||||||
|
|
||||||
|
b. Adapter's License means the license You apply to Your Copyright
|
||||||
|
and Similar Rights in Your contributions to Adapted Material in
|
||||||
|
accordance with the terms and conditions of this Public License.
|
||||||
|
|
||||||
|
c. BY-NC-SA Compatible License means a license listed at
|
||||||
|
creativecommons.org/compatiblelicenses, approved by Creative
|
||||||
|
Commons as essentially the equivalent of this Public License.
|
||||||
|
|
||||||
|
d. Copyright and Similar Rights means copyright and/or similar rights
|
||||||
|
closely related to copyright including, without limitation,
|
||||||
|
performance, broadcast, sound recording, and Sui Generis Database
|
||||||
|
Rights, without regard to how the rights are labeled or
|
||||||
|
categorized. For purposes of this Public License, the rights
|
||||||
|
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
||||||
|
Rights.
|
||||||
|
|
||||||
|
e. Effective Technological Measures means those measures that, in the
|
||||||
|
absence of proper authority, may not be circumvented under laws
|
||||||
|
fulfilling obligations under Article 11 of the WIPO Copyright
|
||||||
|
Treaty adopted on December 20, 1996, and/or similar international
|
||||||
|
agreements.
|
||||||
|
|
||||||
|
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
||||||
|
any other exception or limitation to Copyright and Similar Rights
|
||||||
|
that applies to Your use of the Licensed Material.
|
||||||
|
|
||||||
|
g. License Elements means the license attributes listed in the name
|
||||||
|
of a Creative Commons Public License. The License Elements of this
|
||||||
|
Public License are Attribution, NonCommercial, and ShareAlike.
|
||||||
|
|
||||||
|
h. Licensed Material means the artistic or literary work, database,
|
||||||
|
or other material to which the Licensor applied this Public
|
||||||
|
License.
|
||||||
|
|
||||||
|
i. Licensed Rights means the rights granted to You subject to the
|
||||||
|
terms and conditions of this Public License, which are limited to
|
||||||
|
all Copyright and Similar Rights that apply to Your use of the
|
||||||
|
Licensed Material and that the Licensor has authority to license.
|
||||||
|
|
||||||
|
j. Licensor means the individual(s) or entity(ies) granting rights
|
||||||
|
under this Public License.
|
||||||
|
|
||||||
|
k. NonCommercial means not primarily intended for or directed towards
|
||||||
|
commercial advantage or monetary compensation. For purposes of
|
||||||
|
this Public License, the exchange of the Licensed Material for
|
||||||
|
other material subject to Copyright and Similar Rights by digital
|
||||||
|
file-sharing or similar means is NonCommercial provided there is
|
||||||
|
no payment of monetary compensation in connection with the
|
||||||
|
exchange.
|
||||||
|
|
||||||
|
l. Share means to provide material to the public by any means or
|
||||||
|
process that requires permission under the Licensed Rights, such
|
||||||
|
as reproduction, public display, public performance, distribution,
|
||||||
|
dissemination, communication, or importation, and to make material
|
||||||
|
available to the public including in ways that members of the
|
||||||
|
public may access the material from a place and at a time
|
||||||
|
individually chosen by them.
|
||||||
|
|
||||||
|
m. Sui Generis Database Rights means rights other than copyright
|
||||||
|
resulting from Directive 96/9/EC of the European Parliament and of
|
||||||
|
the Council of 11 March 1996 on the legal protection of databases,
|
||||||
|
as amended and/or succeeded, as well as other essentially
|
||||||
|
equivalent rights anywhere in the world.
|
||||||
|
|
||||||
|
n. You means the individual or entity exercising the Licensed Rights
|
||||||
|
under this Public License. Your has a corresponding meaning.
|
||||||
|
|
||||||
|
|
||||||
|
Section 2 -- Scope.
|
||||||
|
|
||||||
|
a. License grant.
|
||||||
|
|
||||||
|
1. Subject to the terms and conditions of this Public License,
|
||||||
|
the Licensor hereby grants You a worldwide, royalty-free,
|
||||||
|
non-sublicensable, non-exclusive, irrevocable license to
|
||||||
|
exercise the Licensed Rights in the Licensed Material to:
|
||||||
|
|
||||||
|
a. reproduce and Share the Licensed Material, in whole or
|
||||||
|
in part, for NonCommercial purposes only; and
|
||||||
|
|
||||||
|
b. produce, reproduce, and Share Adapted Material for
|
||||||
|
NonCommercial purposes only.
|
||||||
|
|
||||||
|
2. Exceptions and Limitations. For the avoidance of doubt, where
|
||||||
|
Exceptions and Limitations apply to Your use, this Public
|
||||||
|
License does not apply, and You do not need to comply with
|
||||||
|
its terms and conditions.
|
||||||
|
|
||||||
|
3. Term. The term of this Public License is specified in Section
|
||||||
|
6(a).
|
||||||
|
|
||||||
|
4. Media and formats; technical modifications allowed. The
|
||||||
|
Licensor authorizes You to exercise the Licensed Rights in
|
||||||
|
all media and formats whether now known or hereafter created,
|
||||||
|
and to make technical modifications necessary to do so. The
|
||||||
|
Licensor waives and/or agrees not to assert any right or
|
||||||
|
authority to forbid You from making technical modifications
|
||||||
|
necessary to exercise the Licensed Rights, including
|
||||||
|
technical modifications necessary to circumvent Effective
|
||||||
|
Technological Measures. For purposes of this Public License,
|
||||||
|
simply making modifications authorized by this Section 2(a)
|
||||||
|
(4) never produces Adapted Material.
|
||||||
|
|
||||||
|
5. Downstream recipients.
|
||||||
|
|
||||||
|
a. Offer from the Licensor -- Licensed Material. Every
|
||||||
|
recipient of the Licensed Material automatically
|
||||||
|
receives an offer from the Licensor to exercise the
|
||||||
|
Licensed Rights under the terms and conditions of this
|
||||||
|
Public License.
|
||||||
|
|
||||||
|
b. Additional offer from the Licensor -- Adapted Material.
|
||||||
|
Every recipient of Adapted Material from You
|
||||||
|
automatically receives an offer from the Licensor to
|
||||||
|
exercise the Licensed Rights in the Adapted Material
|
||||||
|
under the conditions of the Adapter's License You apply.
|
||||||
|
|
||||||
|
c. No downstream restrictions. You may not offer or impose
|
||||||
|
any additional or different terms or conditions on, or
|
||||||
|
apply any Effective Technological Measures to, the
|
||||||
|
Licensed Material if doing so restricts exercise of the
|
||||||
|
Licensed Rights by any recipient of the Licensed
|
||||||
|
Material.
|
||||||
|
|
||||||
|
6. No endorsement. Nothing in this Public License constitutes or
|
||||||
|
may be construed as permission to assert or imply that You
|
||||||
|
are, or that Your use of the Licensed Material is, connected
|
||||||
|
with, or sponsored, endorsed, or granted official status by,
|
||||||
|
the Licensor or others designated to receive attribution as
|
||||||
|
provided in Section 3(a)(1)(A)(i).
|
||||||
|
|
||||||
|
b. Other rights.
|
||||||
|
|
||||||
|
1. Moral rights, such as the right of integrity, are not
|
||||||
|
licensed under this Public License, nor are publicity,
|
||||||
|
privacy, and/or other similar personality rights; however, to
|
||||||
|
the extent possible, the Licensor waives and/or agrees not to
|
||||||
|
assert any such rights held by the Licensor to the limited
|
||||||
|
extent necessary to allow You to exercise the Licensed
|
||||||
|
Rights, but not otherwise.
|
||||||
|
|
||||||
|
2. Patent and trademark rights are not licensed under this
|
||||||
|
Public License.
|
||||||
|
|
||||||
|
3. To the extent possible, the Licensor waives any right to
|
||||||
|
collect royalties from You for the exercise of the Licensed
|
||||||
|
Rights, whether directly or through a collecting society
|
||||||
|
under any voluntary or waivable statutory or compulsory
|
||||||
|
licensing scheme. In all other cases the Licensor expressly
|
||||||
|
reserves any right to collect such royalties, including when
|
||||||
|
the Licensed Material is used other than for NonCommercial
|
||||||
|
purposes.
|
||||||
|
|
||||||
|
|
||||||
|
Section 3 -- License Conditions.
|
||||||
|
|
||||||
|
Your exercise of the Licensed Rights is expressly made subject to the
|
||||||
|
following conditions.
|
||||||
|
|
||||||
|
a. Attribution.
|
||||||
|
|
||||||
|
1. If You Share the Licensed Material (including in modified
|
||||||
|
form), You must:
|
||||||
|
|
||||||
|
a. retain the following if it is supplied by the Licensor
|
||||||
|
with the Licensed Material:
|
||||||
|
|
||||||
|
i. identification of the creator(s) of the Licensed
|
||||||
|
Material and any others designated to receive
|
||||||
|
attribution, in any reasonable manner requested by
|
||||||
|
the Licensor (including by pseudonym if
|
||||||
|
designated);
|
||||||
|
|
||||||
|
ii. a copyright notice;
|
||||||
|
|
||||||
|
iii. a notice that refers to this Public License;
|
||||||
|
|
||||||
|
iv. a notice that refers to the disclaimer of
|
||||||
|
warranties;
|
||||||
|
|
||||||
|
v. a URI or hyperlink to the Licensed Material to the
|
||||||
|
extent reasonably practicable;
|
||||||
|
|
||||||
|
b. indicate if You modified the Licensed Material and
|
||||||
|
retain an indication of any previous modifications; and
|
||||||
|
|
||||||
|
c. indicate the Licensed Material is licensed under this
|
||||||
|
Public License, and include the text of, or the URI or
|
||||||
|
hyperlink to, this Public License.
|
||||||
|
|
||||||
|
2. You may satisfy the conditions in Section 3(a)(1) in any
|
||||||
|
reasonable manner based on the medium, means, and context in
|
||||||
|
which You Share the Licensed Material. For example, it may be
|
||||||
|
reasonable to satisfy the conditions by providing a URI or
|
||||||
|
hyperlink to a resource that includes the required
|
||||||
|
information.
|
||||||
|
3. If requested by the Licensor, You must remove any of the
|
||||||
|
information required by Section 3(a)(1)(A) to the extent
|
||||||
|
reasonably practicable.
|
||||||
|
|
||||||
|
b. ShareAlike.
|
||||||
|
|
||||||
|
In addition to the conditions in Section 3(a), if You Share
|
||||||
|
Adapted Material You produce, the following conditions also apply.
|
||||||
|
|
||||||
|
1. The Adapter's License You apply must be a Creative Commons
|
||||||
|
license with the same License Elements, this version or
|
||||||
|
later, or a BY-NC-SA Compatible License.
|
||||||
|
|
||||||
|
2. You must include the text of, or the URI or hyperlink to, the
|
||||||
|
Adapter's License You apply. You may satisfy this condition
|
||||||
|
in any reasonable manner based on the medium, means, and
|
||||||
|
context in which You Share Adapted Material.
|
||||||
|
|
||||||
|
3. You may not offer or impose any additional or different terms
|
||||||
|
or conditions on, or apply any Effective Technological
|
||||||
|
Measures to, Adapted Material that restrict exercise of the
|
||||||
|
rights granted under the Adapter's License You apply.
|
||||||
|
|
||||||
|
|
||||||
|
Section 4 -- Sui Generis Database Rights.
|
||||||
|
|
||||||
|
Where the Licensed Rights include Sui Generis Database Rights that
|
||||||
|
apply to Your use of the Licensed Material:
|
||||||
|
|
||||||
|
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
||||||
|
to extract, reuse, reproduce, and Share all or a substantial
|
||||||
|
portion of the contents of the database for NonCommercial purposes
|
||||||
|
only;
|
||||||
|
|
||||||
|
b. if You include all or a substantial portion of the database
|
||||||
|
contents in a database in which You have Sui Generis Database
|
||||||
|
Rights, then the database in which You have Sui Generis Database
|
||||||
|
Rights (but not its individual contents) is Adapted Material,
|
||||||
|
including for purposes of Section 3(b); and
|
||||||
|
|
||||||
|
c. You must comply with the conditions in Section 3(a) if You Share
|
||||||
|
all or a substantial portion of the contents of the database.
|
||||||
|
|
||||||
|
For the avoidance of doubt, this Section 4 supplements and does not
|
||||||
|
replace Your obligations under this Public License where the Licensed
|
||||||
|
Rights include other Copyright and Similar Rights.
|
||||||
|
|
||||||
|
|
||||||
|
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
||||||
|
|
||||||
|
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
||||||
|
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
||||||
|
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
||||||
|
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
||||||
|
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
||||||
|
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
||||||
|
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
||||||
|
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
||||||
|
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
||||||
|
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
||||||
|
|
||||||
|
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
||||||
|
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
||||||
|
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
||||||
|
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
||||||
|
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
||||||
|
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
||||||
|
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
||||||
|
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
||||||
|
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
||||||
|
|
||||||
|
c. The disclaimer of warranties and limitation of liability provided
|
||||||
|
above shall be interpreted in a manner that, to the extent
|
||||||
|
possible, most closely approximates an absolute disclaimer and
|
||||||
|
waiver of all liability.
|
||||||
|
|
||||||
|
|
||||||
|
Section 6 -- Term and Termination.
|
||||||
|
|
||||||
|
a. This Public License applies for the term of the Copyright and
|
||||||
|
Similar Rights licensed here. However, if You fail to comply with
|
||||||
|
this Public License, then Your rights under this Public License
|
||||||
|
terminate automatically.
|
||||||
|
|
||||||
|
b. Where Your right to use the Licensed Material has terminated under
|
||||||
|
Section 6(a), it reinstates:
|
||||||
|
|
||||||
|
1. automatically as of the date the violation is cured, provided
|
||||||
|
it is cured within 30 days of Your discovery of the
|
||||||
|
violation; or
|
||||||
|
|
||||||
|
2. upon express reinstatement by the Licensor.
|
||||||
|
|
||||||
|
For the avoidance of doubt, this Section 6(b) does not affect any
|
||||||
|
right the Licensor may have to seek remedies for Your violations
|
||||||
|
of this Public License.
|
||||||
|
|
||||||
|
c. For the avoidance of doubt, the Licensor may also offer the
|
||||||
|
Licensed Material under separate terms or conditions or stop
|
||||||
|
distributing the Licensed Material at any time; however, doing so
|
||||||
|
will not terminate this Public License.
|
||||||
|
|
||||||
|
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
||||||
|
License.
|
||||||
|
|
||||||
|
|
||||||
|
Section 7 -- Other Terms and Conditions.
|
||||||
|
|
||||||
|
a. The Licensor shall not be bound by any additional or different
|
||||||
|
terms or conditions communicated by You unless expressly agreed.
|
||||||
|
|
||||||
|
b. Any arrangements, understandings, or agreements regarding the
|
||||||
|
Licensed Material not stated herein are separate from and
|
||||||
|
independent of the terms and conditions of this Public License.
|
||||||
|
|
||||||
|
|
||||||
|
Section 8 -- Interpretation.
|
||||||
|
|
||||||
|
a. For the avoidance of doubt, this Public License does not, and
|
||||||
|
shall not be interpreted to, reduce, limit, restrict, or impose
|
||||||
|
conditions on any use of the Licensed Material that could lawfully
|
||||||
|
be made without permission under this Public License.
|
||||||
|
|
||||||
|
b. To the extent possible, if any provision of this Public License is
|
||||||
|
deemed unenforceable, it shall be automatically reformed to the
|
||||||
|
minimum extent necessary to make it enforceable. If the provision
|
||||||
|
cannot be reformed, it shall be severed from this Public License
|
||||||
|
without affecting the enforceability of the remaining terms and
|
||||||
|
conditions.
|
||||||
|
|
||||||
|
c. No term or condition of this Public License will be waived and no
|
||||||
|
failure to comply consented to unless expressly agreed to by the
|
||||||
|
Licensor.
|
||||||
|
|
||||||
|
d. Nothing in this Public License constitutes or may be interpreted
|
||||||
|
as a limitation upon, or waiver of, any privileges and immunities
|
||||||
|
that apply to the Licensor or You, including from the legal
|
||||||
|
processes of any jurisdiction or authority.
|
||||||
|
|
||||||
|
=======================================================================
|
||||||
|
|
||||||
|
Creative Commons is not a party to its public
|
||||||
|
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
||||||
|
its public licenses to material it publishes and in those instances
|
||||||
|
will be considered the “Licensor.” The text of the Creative Commons
|
||||||
|
public licenses is dedicated to the public domain under the CC0 Public
|
||||||
|
Domain Dedication. Except for the limited purpose of indicating that
|
||||||
|
material is shared under a Creative Commons public license or as
|
||||||
|
otherwise permitted by the Creative Commons policies published at
|
||||||
|
creativecommons.org/policies, Creative Commons does not authorize the
|
||||||
|
use of the trademark "Creative Commons" or any other trademark or logo
|
||||||
|
of Creative Commons without its prior written consent including,
|
||||||
|
without limitation, in connection with any unauthorized modifications
|
||||||
|
to any of its public licenses or any other arrangements,
|
||||||
|
understandings, or agreements concerning use of licensed material. For
|
||||||
|
the avoidance of doubt, this paragraph does not form part of the
|
||||||
|
public licenses.
|
||||||
|
|
||||||
|
Creative Commons may be contacted at creativecommons.org.
|
||||||
177
README.md
Normal file
177
README.md
Normal file
|
|
@ -0,0 +1,177 @@
|
||||||
|
# J.A.R.V.I.S (Rust) — форк Bossiara13
|
||||||
|
|
||||||
|
Форк Rust-переписки голосового ассистента [Priler/jarvis](https://github.com/Priler/jarvis).
|
||||||
|
Текущий репозиторий: <https://github.com/DmitryBykov-ISPO/J.A.R.V.I.S-rust>.
|
||||||
|
|
||||||
|
## Что это
|
||||||
|
|
||||||
|
Голосовой ассистент, написанный на Rust, работающий локально (без облака).
|
||||||
|
Текущий стек:
|
||||||
|
|
||||||
|
- Vosk — Speech-to-Text (через `vosk-rs`).
|
||||||
|
- fastembed + ort — локальные эмбеддинги для intent-классификации (MiniLM L6/L12 ONNX).
|
||||||
|
- Picovoice Porcupine / Rustpotter / Vosk — три опциональных движка wake-word.
|
||||||
|
- mlua (Lua 5.5, vendored) — скрипты пользовательских команд.
|
||||||
|
- Tauri + Vite/Svelte — GUI-оболочка (фронтенд в отдельной папке `frontend/`).
|
||||||
|
- nnnoiseless — подавление шума.
|
||||||
|
- fluent / unic-langid — i18n (`ru`, `ua`, `en`).
|
||||||
|
|
||||||
|
**LLM-клиент (Groq / OpenAI-совместимый) добавлен в `jarvis-core::llm` и подключён к голосовому циклу.** Если фраза начинается с триггера («скажи», «ответь», «произнеси»), она уходит в Groq и ответ возвращается через IPC-событие `LlmReply`. Без триггера всё работает как раньше — wake-word + intent + Lua. Это следующий шаг после CLI-only LLM из v0.2.0.
|
||||||
|
|
||||||
|
## Это форк
|
||||||
|
|
||||||
|
Оригинальный автор — Abraham Tugalov (Priler).
|
||||||
|
Апстрим: <https://github.com/Priler/jarvis>.
|
||||||
|
Лицензия сохранена: **CC BY-NC-SA 4.0** (см. `LICENSE.txt`).
|
||||||
|
Атрибуция в `Cargo.toml` и `voice.toml` пакетов озвучки не изменена.
|
||||||
|
|
||||||
|
## Что отличается от апстрима
|
||||||
|
|
||||||
|
- Обновлён список авторов в `Cargo.toml` (добавлен `Bossiara13 (fork)`, оригинал сохранён).
|
||||||
|
- README переписан и отражает фактическую архитектуру (апстримный README называет проект "Tauri+Svelte", что давно не соответствует действительности — это workspace из 4-х крейтов).
|
||||||
|
- Отсутствующие в апстриме ONNX-модели (`all-MiniLM-L6-v2`, `paraphrase-multilingual-MiniLM-L12-v2-onnx-Q`) подтянуты через Git LFS из HuggingFace (Qdrant) и запушены в форк.
|
||||||
|
|
||||||
|
## Структура репозитория
|
||||||
|
|
||||||
|
Cargo workspace из четырёх крейтов:
|
||||||
|
|
||||||
|
| Крейт | Назначение |
|
||||||
|
|----------------|----------------------------------------------------------------------------|
|
||||||
|
| `jarvis-core` | Библиотека: конфиг, intent, STT, wake-word, аудио, Lua-бэкенд, i18n. |
|
||||||
|
| `jarvis-app` | Бинарь-«демон»: собирает всё вместе, tray, IPC. |
|
||||||
|
| `jarvis-gui` | Tauri-приложение (использует `frontend/dist/client`). |
|
||||||
|
| `jarvis-cli` | CLI для отладки: классификация intent, список команд, dump конфига. |
|
||||||
|
|
||||||
|
Прочее:
|
||||||
|
|
||||||
|
- `frontend/` — Vite + Svelte UI для `jarvis-gui`. Собирается отдельно.
|
||||||
|
- `lib/windows/amd64/` — нативные DLL/LIB для Vosk, Porcupine, PvRecorder.
|
||||||
|
- `resources/` — голоса, модели, конфиги по умолчанию. ONNX-модели хранятся в Git LFS.
|
||||||
|
- `post_build.py` — постпроцессинг артефактов сборки (Python 3).
|
||||||
|
|
||||||
|
## Сборка
|
||||||
|
|
||||||
|
Требования:
|
||||||
|
|
||||||
|
- Rust 1.93+ (собирается на stable MSVC).
|
||||||
|
- Node 24+ и npm — для фронтенда.
|
||||||
|
- Python 3 — для `post_build.py`.
|
||||||
|
- MSVC build tools (Windows, x64).
|
||||||
|
- Установленные `libvosk.lib`, `libpv_porcupine.dll`, `libpv_recorder.dll` в `lib/windows/amd64/` (уже в репозитории).
|
||||||
|
|
||||||
|
Перед сборкой `jarvis-gui` нужно собрать фронтенд:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd frontend
|
||||||
|
npm install
|
||||||
|
npm run build
|
||||||
|
cd ..
|
||||||
|
```
|
||||||
|
|
||||||
|
Затем workspace:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build --workspace
|
||||||
|
```
|
||||||
|
|
||||||
|
Холодная сборка занимает около 10 минут (ONNX runtime, aws-lc-rs, tauri).
|
||||||
|
|
||||||
|
## Статус сборки в этом форке
|
||||||
|
|
||||||
|
На моей машине (`cargo build --workspace`, stable MSVC) итог:
|
||||||
|
|
||||||
|
- `jarvis-core` — собрался (1 warning, unused import).
|
||||||
|
- `jarvis-app` — собрался, бинарник `target/debug/jarvis-app.exe` создан.
|
||||||
|
- `jarvis-cli` — **падает на линковке**: `LNK1181: cannot open input file "libvosk.lib"`.
|
||||||
|
Причина: у `jarvis-cli` нет своего `build.rs`, а `.cargo/config.toml` с `rustc-link-search` лежит только внутри `crates/jarvis-app/` и не подтягивается для `jarvis-cli`. Лечится либо добавлением такого же `build.rs` в `crates/jarvis-cli/`, либо вынесением `config.toml` в корень. Сознательно не трогал — фикс выходит за рамки рефакторинга (v0.0.1-import фиксирует поведение апстрима как есть).
|
||||||
|
- `jarvis-gui` — падает в `tauri::generate_context!()`: `frontendDist = "../../frontend/dist/client"` не существует. Это ожидаемо, если не запустить `npm run build` в `frontend/` заранее (см. секцию «Сборка»).
|
||||||
|
|
||||||
|
Запуск уже собранного:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./target/debug/jarvis-app.exe
|
||||||
|
```
|
||||||
|
|
||||||
|
Для CLI (`jarvis-cli --help`, команды `classify`, `execute`, `list`, `phrases`) нужно сначала починить линковку Vosk (см. выше).
|
||||||
|
|
||||||
|
## LLM (Groq)
|
||||||
|
|
||||||
|
В `jarvis-core` есть модуль `llm` — блокирующий клиент для OpenAI-совместимого эндпоинта chat completions. По умолчанию настроен на Groq. Используется через фиче-флаг `llm` (включён в дефолтный набор `jarvis_app`, также подтянут в `jarvis-cli`).
|
||||||
|
|
||||||
|
Переменные окружения:
|
||||||
|
|
||||||
|
| Переменная | Обязательна | Значение по умолчанию |
|
||||||
|
|-----------------|-------------|----------------------------------------|
|
||||||
|
| `GROQ_TOKEN` | да | — |
|
||||||
|
| `GROQ_BASE_URL` | нет | `https://api.groq.com/openai/v1` |
|
||||||
|
| `GROQ_MODEL` | нет | `llama-3.3-70b-versatile` |
|
||||||
|
|
||||||
|
Быстрая проверка через CLI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
set GROQ_TOKEN=gsk_...
|
||||||
|
jarvis-cli ask "скажи привет одной фразой"
|
||||||
|
```
|
||||||
|
|
||||||
|
Ответ печатается в stdout. Без `GROQ_TOKEN` команда завершится с кодом 2 и сообщением об ошибке. При ошибке API — код 1 и тело ответа.
|
||||||
|
|
||||||
|
Программное использование из Rust:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use jarvis_core::llm::{LlmClient, ChatMessage};
|
||||||
|
|
||||||
|
let client = LlmClient::from_env()?;
|
||||||
|
let reply = client.complete(&[ChatMessage::user("привет")], 256)?;
|
||||||
|
println!("{}", reply);
|
||||||
|
```
|
||||||
|
|
||||||
|
### Подключение к голосовому циклу
|
||||||
|
|
||||||
|
Помимо CLI, LLM подключён напрямую в `jarvis-app`. Логика в `crates/jarvis-app/src/llm_fallback.rs`:
|
||||||
|
|
||||||
|
- При старте `jarvis-app` пытается прочитать `GROQ_TOKEN`. Если переменной нет — фоллбэк отключается, в лог пишется warning, голосовые команды продолжают работать как раньше.
|
||||||
|
- Распознанная фраза (как из микрофона, так и из текстовой панели GUI) проверяется на префиксы-триггеры из `config::get_llm_trigger_phrases` (для `ru`/`ua`: `скажи`, `ответь`, `произнеси`; для `en`: `say`, `tell`, `answer`).
|
||||||
|
- Если триггер найден — остаток фразы уходит в `LlmClient::complete()`, ответ публикуется в IPC как `IpcEvent::LlmReply { text }` (UI/GUI слушает этот ивент и проговаривает текст уже на своей стороне), а звук-«ок» проигрывается из текущего голосового пресета.
|
||||||
|
- История разговора хранится в `ConversationHistory` с потолком `LLM_DEFAULT_MAX_HISTORY = 8` ходов; system-prompt всегда сохраняется при вытеснении старых ходов.
|
||||||
|
- При сетевой/API-ошибке последний user-turn убирается из истории, в IPC уходит `LlmReply` с короткой русской фразой («Не могу связаться с сервером, сэр.»), играется звук-«error». Голосовой цикл не падает.
|
||||||
|
|
||||||
|
Системный промпт (русский) описывает J.A.R.V.I.S. как британского дворецкого Тони Старка — короткие реплики (1–3 предложения), обращение «сэр», без излишней цензуры. Меняется в `config::LLM_SYSTEM_PROMPT_RU`.
|
||||||
|
|
||||||
|
## VAD: умное завершение команды по тишине
|
||||||
|
|
||||||
|
После срабатывания wake-word `jarvis-app` слушает команду. Раньше окно закрывалось либо по таймауту Vosk, либо по жёсткому пределу из `config::CMS_WAIT_DELAY` — короткие команды («стоп», «громче») всё равно ждали несколько секунд тишины.
|
||||||
|
|
||||||
|
Теперь окно закрывается, как только пользователь замолчал. Реализовано через `webrtc-vad` (чистый Rust, тот же алгоритм, что в Python-версии). Это **паритет с Python v0.2.0**, где использовался `webrtcvad` с теми же параметрами.
|
||||||
|
|
||||||
|
Логика (state-машина в `crates/jarvis-core/src/audio_processing/vad/listen_window.rs`):
|
||||||
|
|
||||||
|
- Кадр VAD: 30 мс при 16 кГц = 480 сэмплов = 960 байт (mono i16). Кадры с микрофона (по 512 сэмплов от `pv_recorder`) аккумулируются и режутся на VAD-кадры адаптером `WebRtcVad::push_samples`.
|
||||||
|
- Каждый VAD-кадр учитывается в счётчиках `speech_ms` / `silence_ms`.
|
||||||
|
- Окно закрывается, когда `silence_ms >= VAD_COMMAND_END_SILENCE_MS` И `speech_ms >= VAD_COMMAND_MIN_SPEECH_MS`.
|
||||||
|
- До истечения `VAD_COMMAND_MIN_LISTEN_MS` окно не закрывается — пользователю даётся время начать говорить.
|
||||||
|
- При достижении `VAD_COMMAND_MAX_LISTEN_MS` срабатывает hard-cap и управление возвращается в режим ожидания wake-word.
|
||||||
|
- На событии «закрыть окно» вызывается `stt::finalize_speech()` — Vosk форсированно отдаёт финальный результат, не дожидаясь собственного таймаута.
|
||||||
|
|
||||||
|
Параметры (`crates/jarvis-core/src/config.rs`):
|
||||||
|
|
||||||
|
| Константа | Значение | Назначение |
|
||||||
|
|---------------------------------|----------|-------------------------------------------------------------|
|
||||||
|
| `VAD_AGGRESSIVENESS` | `2` | Уровень WebRTC VAD: 0 — Quality, 3 — VeryAggressive. |
|
||||||
|
| `VAD_COMMAND_END_SILENCE_MS` | `1200` | Сколько тишины подряд считать «команда закончилась». |
|
||||||
|
| `VAD_COMMAND_MIN_SPEECH_MS` | `500` | Минимум речи в окне — иначе закрытие игнорируется. |
|
||||||
|
| `VAD_COMMAND_MIN_LISTEN_MS` | `1000` | Минимальная длительность окна (страховка от ранних закрытий)|
|
||||||
|
| `VAD_COMMAND_MAX_LISTEN_MS` | `15000` | Жёсткий потолок окна. |
|
||||||
|
|
||||||
|
На короткие команды отзыв стал заметно быстрее (мы выходим на распознавание сразу после паузы пользователя, а не по 5-секундному порогу Vosk).
|
||||||
|
|
||||||
|
## Лицензия
|
||||||
|
|
||||||
|
Creative Commons **Attribution-NonCommercial-ShareAlike 4.0 International** (CC BY-NC-SA 4.0).
|
||||||
|
Полный текст — в `LICENSE.txt`. Атрибуция оригинального автора (Abraham Tugalov) сохранена.
|
||||||
|
|
||||||
|
В `Cargo.toml` декларирован `license = "GPL-3.0-only"` — это несоответствие унаследовано от апстрима и не правилось, чтобы не расходиться с upstream-конфигом. Приоритет имеет `LICENSE.txt`.
|
||||||
|
|
||||||
|
## Python-версия
|
||||||
|
|
||||||
|
Старая версия ассистента была на Python.
|
||||||
|
Последний коммит с Python-кодом в апстриме — [943efbf](https://github.com/Priler/jarvis/tree/943efbfbdb8aeb5889fa5e2dc7348ca4ea0b81df).
|
||||||
16
crates/jarvis-app/.cargo/config.toml
Normal file
16
crates/jarvis-app/.cargo/config.toml
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Cargo configuration file
|
||||||
|
|
||||||
|
[build]
|
||||||
|
jobs = 4
|
||||||
|
rustflags = [
|
||||||
|
# "-Ctarget-feature=+fp16,+fhm"
|
||||||
|
]
|
||||||
|
|
||||||
|
[profile.dev]
|
||||||
|
opt-level = 0
|
||||||
|
debug = true
|
||||||
|
|
||||||
|
[profile.release]
|
||||||
|
opt-level = 3
|
||||||
|
debug = false
|
||||||
|
lto = true
|
||||||
32
crates/jarvis-app/Cargo.toml
Normal file
32
crates/jarvis-app/Cargo.toml
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
[package]
|
||||||
|
name = "jarvis-app"
|
||||||
|
version.workspace = true
|
||||||
|
authors.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
repository.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
jarvis-core = { path = "../jarvis-core", features = ["intent"] }
|
||||||
|
once_cell.workspace = true
|
||||||
|
log.workspace = true
|
||||||
|
simple-log = "2.4"
|
||||||
|
tray-icon = "0.21"
|
||||||
|
winit = "0.30"
|
||||||
|
image.workspace = true
|
||||||
|
platform-dirs.workspace = true
|
||||||
|
rand.workspace = true
|
||||||
|
parking_lot.workspace = true
|
||||||
|
|
||||||
|
tokio = { version = "1", features = ["rt-multi-thread"] }
|
||||||
|
|
||||||
|
[target.'cfg(windows)'.dependencies.winit]
|
||||||
|
version = "0.30"
|
||||||
|
features = []
|
||||||
|
|
||||||
|
[target.'cfg(target_os = "windows")'.dependencies]
|
||||||
|
winapi = { version = "0.3", features = ["winuser"] }
|
||||||
|
|
||||||
|
[target.'cfg(target_os = "linux")'.dependencies]
|
||||||
|
gtk = "0.18"
|
||||||
|
glib = "0.21.5"
|
||||||
|
|
@ -25,21 +25,21 @@ command = "cargo"
|
||||||
args = ["test"]
|
args = ["test"]
|
||||||
# dependencies = ["clean"]
|
# dependencies = ["clean"]
|
||||||
|
|
||||||
[tasks.vosk]
|
[tasks.post_build]
|
||||||
script_runner = "python"
|
script_runner = "python"
|
||||||
script_extension = "py"
|
script_extension = "py"
|
||||||
script = { file = "vosk_build.py" }
|
script = { file = "post_build.py" }
|
||||||
|
|
||||||
[tasks.debug]
|
[tasks.debug]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"format",
|
"format",
|
||||||
"build_debug",
|
"build_debug",
|
||||||
"vosk"
|
"post_build"
|
||||||
]
|
]
|
||||||
|
|
||||||
[tasks.release]
|
[tasks.release]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"format",
|
"format",
|
||||||
"build_release",
|
"build_release",
|
||||||
"vosk"
|
"post_build"
|
||||||
]
|
]
|
||||||
10
crates/jarvis-app/build.rs
Normal file
10
crates/jarvis-app/build.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
fn main() {
|
||||||
|
// link to Vosk lib
|
||||||
|
// println!("cargo:rustc-link-lib=libvosk.dll");
|
||||||
|
|
||||||
|
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||||
|
let lib_path = std::path::Path::new(&manifest_dir)
|
||||||
|
.join("..\\..\\lib\\windows\\amd64");
|
||||||
|
|
||||||
|
println!("cargo:rustc-link-search=native={}", lib_path.display());
|
||||||
|
}
|
||||||
337
crates/jarvis-app/src/_app.rs
Normal file
337
crates/jarvis-app/src/_app.rs
Normal file
|
|
@ -0,0 +1,337 @@
|
||||||
|
use std::sync::mpsc::Receiver;
|
||||||
|
use std::time::SystemTime;
|
||||||
|
|
||||||
|
use jarvis_core::{audio_buffer::AudioRingBuffer, audio, audio_processing, commands, config, listener, recorder, stt, COMMANDS_LIST, intent, voices, ipc::{self, IpcEvent}};
|
||||||
|
use rand::prelude::*;
|
||||||
|
|
||||||
|
use crate::should_stop;
|
||||||
|
|
||||||
|
// VAD state machine
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||||
|
enum VadState {
|
||||||
|
WaitingForVoice,
|
||||||
|
VoiceActive,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start(text_cmd_rx: Receiver<String>) -> Result<(), ()> {
|
||||||
|
// start the loop
|
||||||
|
main_loop(text_cmd_rx)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main_loop(text_cmd_rx: Receiver<String>) -> Result<(), ()> {
|
||||||
|
let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime");
|
||||||
|
let mut start: SystemTime;
|
||||||
|
// let sounds_directory = audio::get_sound_directory().unwrap();
|
||||||
|
let frame_length: usize = 512; // default for every wake-word engine
|
||||||
|
let sample_rate: usize = 16000;
|
||||||
|
let mut frame_buffer: Vec<i16> = vec![0; frame_length];
|
||||||
|
|
||||||
|
// ring buffer: keep last 2 seconds of audio
|
||||||
|
let mut audio_buffer = AudioRingBuffer::new(2.0, frame_length, sample_rate);
|
||||||
|
|
||||||
|
// VAD state
|
||||||
|
let mut vad_state = VadState::WaitingForVoice;
|
||||||
|
let mut silence_frames: u32 = 0;
|
||||||
|
|
||||||
|
// how many frames of silence before we consider speech ended
|
||||||
|
// 1.5 seconds = 1.5 * (16000 / 512) ≈ 47 frames
|
||||||
|
// @TODO: Put this to config
|
||||||
|
let silence_threshold: u32 = ((1.5 * sample_rate as f32) / frame_length as f32) as u32;
|
||||||
|
|
||||||
|
// play some startup phrase
|
||||||
|
// audio::play_sound(&sounds_directory.join("run.wav"));
|
||||||
|
voices::play_greet();
|
||||||
|
|
||||||
|
// start recording
|
||||||
|
match recorder::start_recording() {
|
||||||
|
Ok(_) => info!("Recording started."),
|
||||||
|
Err(_) => {
|
||||||
|
error!("Cannot start recording.");
|
||||||
|
return Err(()); // quit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// notify GUI we're ready
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
|
||||||
|
// DEBUG counter
|
||||||
|
let mut frame_count: u32 = 0;
|
||||||
|
|
||||||
|
// the loop
|
||||||
|
'wake_word: loop {
|
||||||
|
// check for stop signal
|
||||||
|
if should_stop() {
|
||||||
|
info!("Stop signal received, shutting down...");
|
||||||
|
voices::play_goodbye();
|
||||||
|
ipc::send(IpcEvent::Stopping);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// check for text commands
|
||||||
|
if let Ok(text) = text_cmd_rx.try_recv() {
|
||||||
|
process_text_command(&text, &rt);
|
||||||
|
continue 'wake_word;
|
||||||
|
}
|
||||||
|
|
||||||
|
// read from microphone
|
||||||
|
recorder::read_microphone(&mut frame_buffer);
|
||||||
|
|
||||||
|
// DEBUG: check raw audio
|
||||||
|
frame_count += 1;
|
||||||
|
let raw_rms = calculate_rms(&frame_buffer);
|
||||||
|
|
||||||
|
if frame_count % 100 == 0 {
|
||||||
|
info!("DEBUG [{}]: raw_rms={:.0}", frame_count, raw_rms);
|
||||||
|
}
|
||||||
|
|
||||||
|
// check if we're getting any audio at all
|
||||||
|
if frame_count == 100 && raw_rms < 10.0 {
|
||||||
|
warn!("WARNING: Microphone appears to be silent! RMS={:.0}", raw_rms);
|
||||||
|
}
|
||||||
|
|
||||||
|
// process audio (gain -> noise suppression -> VAD)
|
||||||
|
let processed = audio_processing::process(&frame_buffer);
|
||||||
|
|
||||||
|
if frame_count % 100 == 0 {
|
||||||
|
info!("DEBUG [{}]: is_voice={}, vad_conf={:.2}, processed_rms={:.0}",
|
||||||
|
frame_count,
|
||||||
|
processed.is_voice,
|
||||||
|
processed.vad_confidence,
|
||||||
|
calculate_rms(&processed.samples)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// skip if no voice detected (vad)
|
||||||
|
if !processed.is_voice {
|
||||||
|
continue 'wake_word;
|
||||||
|
}
|
||||||
|
|
||||||
|
// DEBUG: we passed VAD
|
||||||
|
if frame_count % 50 == 0 {
|
||||||
|
info!("DEBUG: Voice detected, checking wake word...");
|
||||||
|
}
|
||||||
|
|
||||||
|
// recognize wake-word
|
||||||
|
match listener::data_callback(&frame_buffer) {
|
||||||
|
Some(_keyword_index) => {
|
||||||
|
// notify GUI
|
||||||
|
ipc::send(IpcEvent::WakeWordDetected);
|
||||||
|
|
||||||
|
// reset some things
|
||||||
|
stt::reset_wake_recognizer();
|
||||||
|
stt::reset_speech_recognizer();
|
||||||
|
audio_processing::reset();
|
||||||
|
|
||||||
|
// wake-word activated, process further commands
|
||||||
|
// capture current time
|
||||||
|
start = SystemTime::now();
|
||||||
|
silence_frames = 0;
|
||||||
|
|
||||||
|
// play some reply phrase
|
||||||
|
// @TODO. Make it via commands or upcoming events system.
|
||||||
|
voices::play_reply();
|
||||||
|
|
||||||
|
|
||||||
|
// notify GUI we're listening
|
||||||
|
ipc::send(IpcEvent::Listening);
|
||||||
|
|
||||||
|
// wait for voice commands
|
||||||
|
'voice_recognition: loop {
|
||||||
|
// check for stop
|
||||||
|
if should_stop() {
|
||||||
|
break 'wake_word;
|
||||||
|
}
|
||||||
|
|
||||||
|
// read from microphone
|
||||||
|
recorder::read_microphone(&mut frame_buffer);
|
||||||
|
|
||||||
|
// process first
|
||||||
|
let processed = audio_processing::process(&frame_buffer);
|
||||||
|
|
||||||
|
// detect silence, return to wake-word if silence
|
||||||
|
if processed.is_voice {
|
||||||
|
silence_frames = 0;
|
||||||
|
} else {
|
||||||
|
silence_frames += 1;
|
||||||
|
if silence_frames > config::VAD_SILENCE_FRAMES * 2 {
|
||||||
|
info!("Long silence detected, returning to wake word mode.");
|
||||||
|
break 'voice_recognition;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stt part (without partials)
|
||||||
|
if let Some(mut recognized_voice) = stt::recognize(&frame_buffer, false) {
|
||||||
|
// something was recognized
|
||||||
|
info!("Recognized voice: {}", recognized_voice);
|
||||||
|
|
||||||
|
// notify GUI
|
||||||
|
ipc::send(IpcEvent::SpeechRecognized {
|
||||||
|
text: recognized_voice.clone(),
|
||||||
|
});
|
||||||
|
|
||||||
|
// filter recognized voice
|
||||||
|
// @TODO. Better recognized voice filtration.
|
||||||
|
recognized_voice = recognized_voice.to_lowercase();
|
||||||
|
|
||||||
|
// answer again if it's activation phrase repeated
|
||||||
|
if recognized_voice.contains(config::VOSK_FETCH_PHRASE) {
|
||||||
|
info!("Wake word detected during chaining, reactivating...");
|
||||||
|
|
||||||
|
// play greet sound
|
||||||
|
// audio::play_sound(&sounds_directory.join(format!(
|
||||||
|
// "{}.wav",
|
||||||
|
// config::ASSISTANT_GREET_PHRASES
|
||||||
|
// .choose(&mut rand::thread_rng())
|
||||||
|
// .unwrap()
|
||||||
|
// )));
|
||||||
|
voices::play_reply();
|
||||||
|
|
||||||
|
// reset timer and continue listening
|
||||||
|
start = SystemTime::now();
|
||||||
|
silence_frames = 0;
|
||||||
|
stt::reset_speech_recognizer();
|
||||||
|
|
||||||
|
ipc::send(IpcEvent::Listening);
|
||||||
|
continue 'voice_recognition;
|
||||||
|
}
|
||||||
|
|
||||||
|
// filter out activation phrase from command
|
||||||
|
for tbr in config::ASSISTANT_PHRASES_TBR {
|
||||||
|
recognized_voice = recognized_voice.replace(tbr, "");
|
||||||
|
}
|
||||||
|
recognized_voice = recognized_voice.trim().into();
|
||||||
|
|
||||||
|
// skip if nothing left after filtering (*evil laugh*)
|
||||||
|
if recognized_voice.is_empty() {
|
||||||
|
continue 'voice_recognition;
|
||||||
|
}
|
||||||
|
|
||||||
|
// execute command (shared executor)
|
||||||
|
execute_command(&recognized_voice, &rt);
|
||||||
|
|
||||||
|
// return to wake-word listening after command execution (no matter successful or not)
|
||||||
|
break 'voice_recognition;
|
||||||
|
}
|
||||||
|
|
||||||
|
// only recognize voice for a certain period of time
|
||||||
|
match start.elapsed() {
|
||||||
|
Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
|
||||||
|
// return to wake-word listening after N seconds
|
||||||
|
break 'voice_recognition;
|
||||||
|
}
|
||||||
|
_ => (),
|
||||||
|
}
|
||||||
|
|
||||||
|
// reset things
|
||||||
|
stt::reset_wake_recognizer();
|
||||||
|
audio_processing::reset();
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => (),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanup
|
||||||
|
recorder::stop_recording().ok();
|
||||||
|
ipc::send(IpcEvent::Stopping);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// process text command from GUI
|
||||||
|
fn process_text_command(text: &str, rt: &tokio::runtime::Runtime) {
|
||||||
|
info!("Processing text command: {}", text);
|
||||||
|
|
||||||
|
ipc::send(IpcEvent::SpeechRecognized { text: text.to_string() });
|
||||||
|
|
||||||
|
// filter text same as voice
|
||||||
|
let mut filtered = text.to_lowercase();
|
||||||
|
for tbr in config::ASSISTANT_PHRASES_TBR {
|
||||||
|
filtered = filtered.replace(tbr, "");
|
||||||
|
}
|
||||||
|
let filtered = filtered.trim();
|
||||||
|
|
||||||
|
if filtered.is_empty() {
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
execute_command(filtered, rt);
|
||||||
|
}
|
||||||
|
|
||||||
|
// shared command execution logic (manual & voice)
|
||||||
|
fn execute_command(text: &str, rt: &tokio::runtime::Runtime) {
|
||||||
|
let commands_list = match COMMANDS_LIST.get() {
|
||||||
|
Some(c) => c,
|
||||||
|
None => {
|
||||||
|
ipc::send(IpcEvent::Error { message: "Commands not loaded".to_string() });
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// let sounds_directory = audio::get_sound_directory().unwrap();
|
||||||
|
|
||||||
|
// try intent recognition first, fallback to levenshtein
|
||||||
|
let cmd_result = if let Some((intent_id, confidence)) =
|
||||||
|
rt.block_on(intent::classify(text))
|
||||||
|
{
|
||||||
|
info!("Intent recognized: {} (confidence: {:.2})", intent_id, confidence);
|
||||||
|
intent::get_command_by_intent(commands_list, &intent_id)
|
||||||
|
} else {
|
||||||
|
info!("Intent not recognized, trying levenshtein fallback...");
|
||||||
|
commands::fetch_command(text, commands_list)
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some((cmd_path, cmd_config)) = cmd_result {
|
||||||
|
info!("Command found: {:?}", cmd_path);
|
||||||
|
|
||||||
|
match commands::execute_command(&cmd_path, &cmd_config) {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Command executed successfully");
|
||||||
|
voices::play_ok(); // command executed sound
|
||||||
|
ipc::send(IpcEvent::CommandExecuted {
|
||||||
|
id: cmd_config.id.clone(),
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Error executing command: {}", msg);
|
||||||
|
voices::play_error();
|
||||||
|
ipc::send(IpcEvent::CommandExecuted {
|
||||||
|
id: cmd_config.id.clone(),
|
||||||
|
success: false,
|
||||||
|
});
|
||||||
|
ipc::send(IpcEvent::Error { message: msg.to_string() });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
info!("No command found for: {}", text);
|
||||||
|
// play "not understood" sound
|
||||||
|
// audio::play_sound(&sounds_directory.join("not_understand.wav"));
|
||||||
|
voices::play_not_found();
|
||||||
|
ipc::send(IpcEvent::Error {
|
||||||
|
message: format!("Command not found: {}", text)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
fn keyword_callback(keyword_index: i32) {}
|
||||||
|
|
||||||
|
pub fn close(code: i32) {
|
||||||
|
info!("Closing application.");
|
||||||
|
voices::play_goodbye();
|
||||||
|
ipc::send(IpcEvent::Stopping);
|
||||||
|
std::process::exit(code);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn calculate_rms(samples: &[i16]) -> f32 {
|
||||||
|
if samples.is_empty() { return 0.0; }
|
||||||
|
let sum: f64 = samples.iter().map(|&s| (s as f64).powi(2)).sum();
|
||||||
|
(sum / samples.len() as f64).sqrt() as f32
|
||||||
|
}
|
||||||
|
|
@ -1,18 +1,12 @@
|
||||||
// Prevents additional console window on Windows in release, DO NOT REMOVE!!
|
|
||||||
// #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
|
|
||||||
|
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate lazy_static; // better switch to once_cell ?
|
extern crate lazy_static; // better switch to once_cell ?
|
||||||
use pickledb::{PickleDb, PickleDbDumpPolicy, SerializationMethod};
|
use pickledb::{PickleDb, PickleDbDumpPolicy, SerializationMethod};
|
||||||
|
use log::{info};
|
||||||
|
use log::LevelFilter;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
|
|
||||||
// expose the config
|
|
||||||
mod config;
|
|
||||||
use config::*;
|
|
||||||
|
|
||||||
// include tauri commands
|
|
||||||
mod tauri_commands;
|
|
||||||
|
|
||||||
// include assistant commands
|
// include assistant commands
|
||||||
mod assistant_commands;
|
mod assistant_commands;
|
||||||
use assistant_commands::AssistantCommand;
|
use assistant_commands::AssistantCommand;
|
||||||
|
|
@ -23,10 +17,8 @@ mod vosk;
|
||||||
// include events
|
// include events
|
||||||
mod events;
|
mod events;
|
||||||
|
|
||||||
// app dir
|
// include recorder
|
||||||
lazy_static! {
|
mod recorder;
|
||||||
static ref APP_CONFIG_DIR: Mutex<String> = Mutex::new(String::new());
|
|
||||||
}
|
|
||||||
|
|
||||||
// init PickleDb connection
|
// init PickleDb connection
|
||||||
lazy_static! {
|
lazy_static! {
|
||||||
|
|
@ -37,7 +29,7 @@ lazy_static! {
|
||||||
SerializationMethod::Json
|
SerializationMethod::Json
|
||||||
)
|
)
|
||||||
.unwrap_or_else(|_x: _| {
|
.unwrap_or_else(|_x: _| {
|
||||||
println!("Creating new db file at {} ...", format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME));
|
info!("Creating new db file at {} ...", format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME));
|
||||||
PickleDb::new(
|
PickleDb::new(
|
||||||
format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME),
|
format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME),
|
||||||
PickleDbDumpPolicy::AutoDump,
|
PickleDbDumpPolicy::AutoDump,
|
||||||
|
|
@ -47,19 +39,24 @@ lazy_static! {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// init commands
|
|
||||||
lazy_static! {
|
|
||||||
static ref COMMANDS: Vec<AssistantCommand> = assistant_commands::parse_commands().unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
|
// init vosk
|
||||||
vosk::init_vosk();
|
vosk::init_vosk();
|
||||||
|
|
||||||
|
// run the app
|
||||||
tauri::Builder::default()
|
tauri::Builder::default()
|
||||||
.setup(|app| {
|
.setup(|app| {
|
||||||
std::fs::create_dir_all(app.path_resolver().app_config_dir().unwrap())?;
|
std::fs::create_dir_all(app.path_resolver().app_config_dir().unwrap())?;
|
||||||
APP_CONFIG_DIR.lock().unwrap().push_str(app.path_resolver().app_config_dir().unwrap().to_str().unwrap());
|
APP_CONFIG_DIR.lock().unwrap().push_str(app.path_resolver().app_config_dir().unwrap().to_str().unwrap());
|
||||||
|
|
||||||
|
std::fs::create_dir_all(app.path_resolver().app_log_dir().unwrap())?;
|
||||||
|
APP_LOG_DIR.lock().unwrap().push_str(app.path_resolver().app_log_dir().unwrap().to_str().unwrap());
|
||||||
|
|
||||||
|
// log to file
|
||||||
|
let log_file_path = format!("{}/{}", APP_LOG_DIR.lock().unwrap(), config::LOG_FILE_NAME);
|
||||||
|
println!("!!!===============!!!\nLOGGING TO {}\n!!!===============!!!\n", &log_file_path);
|
||||||
|
simple_logging::log_to_file(&log_file_path, LevelFilter::max()).expect("Failed to start logger ... is directory writable?");
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.invoke_handler(tauri::generate_handler![
|
.invoke_handler(tauri::generate_handler![
|
||||||
|
|
@ -80,10 +77,15 @@ fn main() {
|
||||||
tauri_commands::get_cpu_usage,
|
tauri_commands::get_cpu_usage,
|
||||||
// sound commands
|
// sound commands
|
||||||
tauri_commands::play_sound,
|
tauri_commands::play_sound,
|
||||||
|
// fs commands
|
||||||
|
tauri_commands::show_in_folder,
|
||||||
// etc commands
|
// etc commands
|
||||||
tauri_commands::get_app_version,
|
tauri_commands::get_app_version,
|
||||||
tauri_commands::get_author_name,
|
tauri_commands::get_author_name,
|
||||||
tauri_commands::get_repository_link
|
tauri_commands::get_repository_link,
|
||||||
|
tauri_commands::get_tg_official_link,
|
||||||
|
tauri_commands::get_feedback_link,
|
||||||
|
tauri_commands::get_log_file_path
|
||||||
])
|
])
|
||||||
.run(tauri::generate_context!())
|
.run(tauri::generate_context!())
|
||||||
.expect("error while running tauri application");
|
.expect("error while running tauri application");
|
||||||
478
crates/jarvis-app/src/app.rs
Normal file
478
crates/jarvis-app/src/app.rs
Normal file
|
|
@ -0,0 +1,478 @@
|
||||||
|
use std::sync::mpsc::Receiver;
|
||||||
|
use std::time::SystemTime;
|
||||||
|
|
||||||
|
use jarvis_core::{audio_buffer::AudioRingBuffer, audio_processing, audio_processing::vad::listen_window::{ListenWindow, WindowDecision}, audio_processing::vad::webrtc::WebRtcVad, commands, config, listener, recorder, stt, COMMANDS_LIST, intent, voices, ipc::{self, IpcEvent}, i18n, slots};
|
||||||
|
use rand::seq::SliceRandom;
|
||||||
|
|
||||||
|
use crate::should_stop;
|
||||||
|
|
||||||
|
// VAD state machine
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||||
|
enum VadState {
|
||||||
|
WaitingForVoice,
|
||||||
|
VoiceActive,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start(text_cmd_rx: Receiver<String>, rt: &tokio::runtime::Runtime) -> Result<(), ()> {
|
||||||
|
main_loop(text_cmd_rx, rt)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main_loop(text_cmd_rx: Receiver<String>, rt: &tokio::runtime::Runtime) -> Result<(), ()> {
|
||||||
|
let frame_length: usize = 512;
|
||||||
|
let sample_rate: usize = 16000;
|
||||||
|
let mut frame_buffer: Vec<i16> = vec![0; frame_length];
|
||||||
|
|
||||||
|
// ring buffer: keeps last 5 seconds of audio (pre-roll)
|
||||||
|
let mut audio_buffer = AudioRingBuffer::new(5.0, frame_length, sample_rate);
|
||||||
|
|
||||||
|
// VAD state
|
||||||
|
let mut vad_state = VadState::WaitingForVoice;
|
||||||
|
let mut silence_frames: u32 = 0;
|
||||||
|
|
||||||
|
// how many frames of silence before we consider speech ended
|
||||||
|
// 1.5 seconds = 1.5 * (16000 / 512) ≈ 47 frames
|
||||||
|
let silence_threshold: u32 = ((1.5 * sample_rate as f32) / frame_length as f32) as u32;
|
||||||
|
|
||||||
|
voices::play_greet();
|
||||||
|
|
||||||
|
match recorder::start_recording() {
|
||||||
|
Ok(_) => info!("Recording started. Microphone: {}",
|
||||||
|
recorder::get_audio_device_name(recorder::get_selected_microphone_index())),
|
||||||
|
Err(_) => {
|
||||||
|
error!("Cannot start recording.");
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
|
||||||
|
// ### WAKE WORD DETECTION LOOP
|
||||||
|
'wake_word: loop {
|
||||||
|
if should_stop() {
|
||||||
|
info!("Stop signal received, shutting down...");
|
||||||
|
voices::play_goodbye();
|
||||||
|
ipc::send(IpcEvent::Stopping);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(text) = text_cmd_rx.try_recv() {
|
||||||
|
process_text_command(&text, &rt);
|
||||||
|
continue 'wake_word;
|
||||||
|
}
|
||||||
|
|
||||||
|
recorder::read_microphone(&mut frame_buffer);
|
||||||
|
let processed = audio_processing::process(&frame_buffer);
|
||||||
|
|
||||||
|
match vad_state {
|
||||||
|
VadState::WaitingForVoice => {
|
||||||
|
// always buffer audio
|
||||||
|
audio_buffer.push(&frame_buffer);
|
||||||
|
|
||||||
|
if processed.is_voice {
|
||||||
|
// voice started! flush buffer to Vosk
|
||||||
|
info!("VAD: Voice started, flushing {} buffered frames", audio_buffer.len());
|
||||||
|
|
||||||
|
for buffered_frame in audio_buffer.drain_all() {
|
||||||
|
listener::data_callback(&buffered_frame);
|
||||||
|
}
|
||||||
|
|
||||||
|
vad_state = VadState::VoiceActive;
|
||||||
|
silence_frames = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
VadState::VoiceActive => {
|
||||||
|
// dual-feed: speech recognizer gets frames in parallel with wake word detector
|
||||||
|
let _ = stt::recognize(&frame_buffer, false);
|
||||||
|
|
||||||
|
// feed to wake word detector
|
||||||
|
if let Some(_keyword_index) = listener::data_callback(&frame_buffer) {
|
||||||
|
// WAKE WORD DETECTED!
|
||||||
|
info!("Wake word activated!");
|
||||||
|
ipc::send(IpcEvent::WakeWordDetected);
|
||||||
|
|
||||||
|
stt::reset_wake_recognizer();
|
||||||
|
audio_processing::reset();
|
||||||
|
|
||||||
|
// brief sniff to keep feeding STT while transitioning
|
||||||
|
let sniff_frames = ((0.3 * sample_rate as f32) / frame_length as f32) as u32;
|
||||||
|
for _ in 0..sniff_frames {
|
||||||
|
recorder::read_microphone(&mut frame_buffer);
|
||||||
|
audio_processing::process(&frame_buffer);
|
||||||
|
stt::recognize(&frame_buffer, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
ipc::send(IpcEvent::Listening);
|
||||||
|
recognize_command(&mut frame_buffer, &rt, frame_length, sample_rate, true);
|
||||||
|
|
||||||
|
// reset state after command
|
||||||
|
vad_state = VadState::WaitingForVoice;
|
||||||
|
silence_frames = 0;
|
||||||
|
audio_buffer.clear();
|
||||||
|
stt::reset_wake_recognizer();
|
||||||
|
stt::reset_speech_recognizer(); // NOW reset, after command is done
|
||||||
|
audio_processing::reset();
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
|
||||||
|
continue 'wake_word;
|
||||||
|
}
|
||||||
|
|
||||||
|
// track silence
|
||||||
|
if processed.is_voice {
|
||||||
|
silence_frames = 0;
|
||||||
|
} else {
|
||||||
|
silence_frames += 1;
|
||||||
|
|
||||||
|
if silence_frames > silence_threshold {
|
||||||
|
debug!("VAD: Silence timeout, returning to wait state");
|
||||||
|
vad_state = VadState::WaitingForVoice;
|
||||||
|
silence_frames = 0;
|
||||||
|
stt::reset_wake_recognizer();
|
||||||
|
stt::reset_speech_recognizer(); // reset since we were dual-feeding
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
recorder::stop_recording().ok();
|
||||||
|
ipc::send(IpcEvent::Stopping);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Voice recognition for command after wake word
|
||||||
|
fn recognize_command(
|
||||||
|
frame_buffer: &mut [i16],
|
||||||
|
rt: &tokio::runtime::Runtime,
|
||||||
|
frame_length: usize,
|
||||||
|
sample_rate: usize,
|
||||||
|
prefed_audio: bool
|
||||||
|
) {
|
||||||
|
let mut audio_buffer = AudioRingBuffer::new(2.0, frame_length, sample_rate);
|
||||||
|
let mut vad_state = if prefed_audio {
|
||||||
|
VadState::VoiceActive
|
||||||
|
} else {
|
||||||
|
VadState::WaitingForVoice
|
||||||
|
};
|
||||||
|
let mut silence_frames: u32 = 0;
|
||||||
|
let mut start = SystemTime::now();
|
||||||
|
let mut first_recognition = prefed_audio;
|
||||||
|
|
||||||
|
let mut webrtc_vad = WebRtcVad::with_aggressiveness(config::VAD_AGGRESSIVENESS);
|
||||||
|
let mut window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
|
||||||
|
|
||||||
|
let silence_threshold: u32 = ((5.0 * sample_rate as f32) / frame_length as f32) as u32;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
if crate::should_stop() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
recorder::read_microphone(frame_buffer);
|
||||||
|
let processed = audio_processing::process(frame_buffer);
|
||||||
|
|
||||||
|
let mut vad_finalized: Option<String> = None;
|
||||||
|
let mut vad_hard_cap = false;
|
||||||
|
for is_speech in webrtc_vad.push_samples(frame_buffer) {
|
||||||
|
match window.push(is_speech) {
|
||||||
|
WindowDecision::KeepListening => {}
|
||||||
|
WindowDecision::Close => {
|
||||||
|
if window.had_speech() {
|
||||||
|
info!(
|
||||||
|
"VAD: silence after speech ({} ms speech, {} ms silence), finalizing.",
|
||||||
|
window.speech_ms(), window.silence_ms()
|
||||||
|
);
|
||||||
|
vad_finalized = stt::finalize_speech();
|
||||||
|
if vad_finalized.is_none() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
WindowDecision::HardCap => {
|
||||||
|
info!("VAD: hard cap reached ({} ms), returning to wake word mode.", window.elapsed_ms());
|
||||||
|
vad_hard_cap = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if vad_hard_cap {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
match vad_state {
|
||||||
|
VadState::WaitingForVoice => {
|
||||||
|
audio_buffer.push(frame_buffer);
|
||||||
|
|
||||||
|
if processed.is_voice {
|
||||||
|
// flush buffer to STT
|
||||||
|
for buffered_frame in audio_buffer.drain_all() {
|
||||||
|
stt::recognize(&buffered_frame, false);
|
||||||
|
}
|
||||||
|
vad_state = VadState::VoiceActive;
|
||||||
|
silence_frames = 0;
|
||||||
|
} else {
|
||||||
|
silence_frames += 1;
|
||||||
|
|
||||||
|
if silence_frames > silence_threshold {
|
||||||
|
info!("Long silence detected, returning to wake word mode.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
VadState::VoiceActive => {
|
||||||
|
// feed to STT (or use VAD-forced finalization)
|
||||||
|
let recognized = vad_finalized.take().or_else(|| stt::recognize(frame_buffer, false));
|
||||||
|
if let Some(mut recognized_voice) = recognized {
|
||||||
|
info!("Recognized voice: {}", recognized_voice);
|
||||||
|
|
||||||
|
ipc::send(IpcEvent::SpeechRecognized {
|
||||||
|
text: recognized_voice.clone(),
|
||||||
|
});
|
||||||
|
|
||||||
|
recognized_voice = recognized_voice.to_lowercase();
|
||||||
|
|
||||||
|
// check if wake word repeated (reactivate)
|
||||||
|
let wake_phrases = config::get_wake_phrases(&i18n::get_language());
|
||||||
|
let contains_wake = wake_phrases.iter().any(|wp| recognized_voice.contains(wp));
|
||||||
|
|
||||||
|
if contains_wake {
|
||||||
|
// strip the wake word
|
||||||
|
let mut remaining = recognized_voice.clone();
|
||||||
|
for wp in wake_phrases {
|
||||||
|
remaining = remaining.replace(wp, "");
|
||||||
|
}
|
||||||
|
let remaining = remaining.trim();
|
||||||
|
|
||||||
|
if remaining.is_empty() {
|
||||||
|
if first_recognition {
|
||||||
|
// leftover wake word from dual-feed, just discard it
|
||||||
|
info!("Discarding initial wake word from prefed audio");
|
||||||
|
first_recognition = false;
|
||||||
|
stt::reset_speech_recognizer();
|
||||||
|
voices::play_reply();
|
||||||
|
vad_state = VadState::WaitingForVoice;
|
||||||
|
silence_frames = 0;
|
||||||
|
start = SystemTime::now();
|
||||||
|
audio_buffer.clear();
|
||||||
|
webrtc_vad.reset();
|
||||||
|
window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// just wake word, no command - reactivate
|
||||||
|
info!("Wake word repeated during chaining, reactivating...");
|
||||||
|
voices::play_reply();
|
||||||
|
stt::reset_speech_recognizer();
|
||||||
|
ipc::send(IpcEvent::Listening);
|
||||||
|
|
||||||
|
vad_state = VadState::WaitingForVoice;
|
||||||
|
silence_frames = 0;
|
||||||
|
start = SystemTime::now();
|
||||||
|
audio_buffer.clear();
|
||||||
|
webrtc_vad.reset();
|
||||||
|
window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
// wake word + command in one phrase - execute the command part
|
||||||
|
info!("Wake word + command during chaining: '{}'", remaining);
|
||||||
|
recognized_voice = remaining.to_string();
|
||||||
|
// fall through to command execution below
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
first_recognition = false;
|
||||||
|
|
||||||
|
if crate::llm_fallback::is_enabled() {
|
||||||
|
if let Some(prompt) = crate::llm_fallback::extract_prompt(&recognized_voice) {
|
||||||
|
crate::llm_fallback::handle(&prompt);
|
||||||
|
stt::reset_speech_recognizer();
|
||||||
|
vad_state = VadState::WaitingForVoice;
|
||||||
|
silence_frames = 0;
|
||||||
|
start = SystemTime::now();
|
||||||
|
audio_buffer.clear();
|
||||||
|
webrtc_vad.reset();
|
||||||
|
window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
|
||||||
|
ipc::send(IpcEvent::Listening);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// filter activation phrases
|
||||||
|
// for tbr in config::ASSISTANT_PHRASES_TBR {
|
||||||
|
// recognized_voice = recognized_voice.replace(tbr, "");
|
||||||
|
// }
|
||||||
|
for tbr in config::get_phrases_to_remove(&i18n::get_language()) {
|
||||||
|
recognized_voice = recognized_voice.replace(tbr, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
recognized_voice = recognized_voice.trim().to_string();
|
||||||
|
|
||||||
|
if recognized_voice.len() < 5 {
|
||||||
|
debug!("Ignoring too short recognition: '{}'", recognized_voice);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if recognized_voice.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// execute command and check if we should chain
|
||||||
|
let should_chain = execute_command(&recognized_voice, rt);
|
||||||
|
|
||||||
|
if should_chain {
|
||||||
|
// chain: reset and continue listening
|
||||||
|
info!("Chaining enabled, continuing to listen...");
|
||||||
|
stt::reset_speech_recognizer();
|
||||||
|
vad_state = VadState::WaitingForVoice;
|
||||||
|
silence_frames = 0;
|
||||||
|
start = SystemTime::now();
|
||||||
|
audio_buffer.clear();
|
||||||
|
webrtc_vad.reset();
|
||||||
|
window = ListenWindow::new(jarvis_core::audio_processing::vad::webrtc::FRAME_MS);
|
||||||
|
ipc::send(IpcEvent::Listening);
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
// no chain: return to wake word
|
||||||
|
info!("No chain, returning to wake word mode.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// track silence
|
||||||
|
if processed.is_voice {
|
||||||
|
silence_frames = 0;
|
||||||
|
} else {
|
||||||
|
silence_frames += 1;
|
||||||
|
|
||||||
|
if silence_frames > silence_threshold {
|
||||||
|
info!("Long silence detected, returning to wake word mode.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// timeout
|
||||||
|
if let Ok(elapsed) = start.elapsed() {
|
||||||
|
if elapsed > config::CMS_WAIT_DELAY {
|
||||||
|
info!("Command timeout, returning to wake word mode.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
fn process_text_command(text: &str, rt: &tokio::runtime::Runtime) {
|
||||||
|
info!("Processing text command: {}", text);
|
||||||
|
|
||||||
|
ipc::send(IpcEvent::SpeechRecognized { text: text.to_string() });
|
||||||
|
|
||||||
|
if crate::llm_fallback::is_enabled() {
|
||||||
|
if let Some(prompt) = crate::llm_fallback::extract_prompt(text) {
|
||||||
|
crate::llm_fallback::handle(&prompt);
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut filtered = text.to_lowercase();
|
||||||
|
// for tbr in config::ASSISTANT_PHRASES_TBR {
|
||||||
|
// filtered = filtered.replace(tbr, "");
|
||||||
|
// }
|
||||||
|
for tbr in config::get_phrases_to_remove(&i18n::get_language()) {
|
||||||
|
filtered = filtered.replace(tbr, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
let filtered = filtered.trim();
|
||||||
|
|
||||||
|
if filtered.is_empty() {
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// text commands never chain
|
||||||
|
execute_command(filtered, rt);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Execute command, returns true if chaining should continue
|
||||||
|
fn execute_command(text: &str, rt: &tokio::runtime::Runtime) -> bool {
|
||||||
|
let commands_list = match COMMANDS_LIST.get() {
|
||||||
|
Some(c) => c,
|
||||||
|
None => {
|
||||||
|
ipc::send(IpcEvent::Error { message: "Commands not loaded".to_string() });
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let cmd_result = if let Some((intent_id, confidence)) =
|
||||||
|
rt.block_on(intent::classify(text))
|
||||||
|
{
|
||||||
|
info!("Intent recognized: {} (confidence: {:.2})", intent_id, confidence);
|
||||||
|
intent::get_command_by_intent(commands_list, &intent_id)
|
||||||
|
} else {
|
||||||
|
info!("Intent not recognized, trying levenshtein fallback...");
|
||||||
|
commands::fetch_command(text, commands_list)
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some((cmd_path, cmd_config)) = cmd_result {
|
||||||
|
info!("Command found: {:?}", cmd_path);
|
||||||
|
|
||||||
|
// extract slots if needed
|
||||||
|
let extracted_slots = if !cmd_config.slots.is_empty() {
|
||||||
|
let s = slots::extract(text, &cmd_config.slots);
|
||||||
|
if !s.is_empty() {
|
||||||
|
info!("Extracted slots: {:?}", s);
|
||||||
|
}
|
||||||
|
Some(s)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
match commands::execute_command(&cmd_path, &cmd_config, Some(&text), extracted_slots.as_ref()) {
|
||||||
|
Ok(chain) => {
|
||||||
|
info!("Command executed successfully");
|
||||||
|
// voices::play_ok();
|
||||||
|
voices::play_random_from(cmd_config.get_sounds(&i18n::get_language()).as_slice());
|
||||||
|
ipc::send(IpcEvent::CommandExecuted {
|
||||||
|
id: cmd_config.id.clone(),
|
||||||
|
success: true,
|
||||||
|
});
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
return chain; // return chain status from command
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Error executing command: {}", msg);
|
||||||
|
voices::play_error();
|
||||||
|
ipc::send(IpcEvent::CommandExecuted {
|
||||||
|
id: cmd_config.id.clone(),
|
||||||
|
success: false,
|
||||||
|
});
|
||||||
|
ipc::send(IpcEvent::Error { message: msg.to_string() });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
info!("No command found for: {}", text);
|
||||||
|
voices::play_not_found();
|
||||||
|
ipc::send(IpcEvent::Error {
|
||||||
|
message: format!("Command not found: {}", text)
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
ipc::send(IpcEvent::Idle);
|
||||||
|
false // no chain on error or not found
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub fn close(code: i32) {
|
||||||
|
info!("Closing application.");
|
||||||
|
voices::play_goodbye();
|
||||||
|
ipc::send(IpcEvent::Stopping);
|
||||||
|
std::process::exit(code);
|
||||||
|
}
|
||||||
113
crates/jarvis-app/src/llm_fallback.rs
Normal file
113
crates/jarvis-app/src/llm_fallback.rs
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
|
use jarvis_core::config;
|
||||||
|
use jarvis_core::i18n;
|
||||||
|
use jarvis_core::ipc::{self, IpcEvent};
|
||||||
|
use jarvis_core::llm::{ChatMessage, ConversationHistory, LlmClient};
|
||||||
|
use jarvis_core::voices;
|
||||||
|
|
||||||
|
struct State {
|
||||||
|
client: LlmClient,
|
||||||
|
history: Mutex<ConversationHistory>,
|
||||||
|
max_tokens: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
static STATE: OnceCell<Option<State>> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() {
|
||||||
|
let _ = STATE.set(build_state());
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_state() -> Option<State> {
|
||||||
|
if !config::LLM_DEFAULT_ENABLED {
|
||||||
|
info!("LLM fallback disabled by config.");
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let client = match LlmClient::from_env() {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("LLM fallback disabled: {}. Set GROQ_TOKEN to enable.", e);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let lang = i18n::get_language();
|
||||||
|
let prompt = config::get_llm_system_prompt(&lang);
|
||||||
|
let history = Mutex::new(ConversationHistory::new(prompt, config::LLM_DEFAULT_MAX_HISTORY));
|
||||||
|
|
||||||
|
info!("LLM fallback enabled (model: {}).", client.model());
|
||||||
|
Some(State {
|
||||||
|
client,
|
||||||
|
history,
|
||||||
|
max_tokens: config::LLM_DEFAULT_MAX_TOKENS,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_enabled() -> bool {
|
||||||
|
STATE.get().and_then(|s| s.as_ref()).is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extract_prompt(text: &str) -> Option<String> {
|
||||||
|
let lang = i18n::get_language();
|
||||||
|
let triggers = config::get_llm_trigger_phrases(&lang);
|
||||||
|
let lowered = text.to_lowercase();
|
||||||
|
|
||||||
|
for trig in triggers {
|
||||||
|
if let Some(rest) = strip_trigger(&lowered, trig) {
|
||||||
|
let trimmed = rest.trim();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
return Some(trimmed.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn strip_trigger(text: &str, trigger: &str) -> Option<String> {
|
||||||
|
let t = text.trim_start();
|
||||||
|
if let Some(after) = t.strip_prefix(trigger) {
|
||||||
|
let next = after.chars().next();
|
||||||
|
if next.map_or(true, |c| !c.is_alphanumeric()) {
|
||||||
|
return Some(after.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn handle(prompt: &str) {
|
||||||
|
let state = match STATE.get().and_then(|s| s.as_ref()) {
|
||||||
|
Some(s) => s,
|
||||||
|
None => {
|
||||||
|
warn!("LLM fallback called while disabled — ignoring.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
info!("LLM prompt: {}", prompt);
|
||||||
|
|
||||||
|
let snapshot: Vec<ChatMessage> = {
|
||||||
|
let mut h = state.history.lock();
|
||||||
|
h.push_user(prompt);
|
||||||
|
h.snapshot()
|
||||||
|
};
|
||||||
|
|
||||||
|
match state.client.complete(&snapshot, state.max_tokens) {
|
||||||
|
Ok(reply) => {
|
||||||
|
let reply = reply.trim().to_string();
|
||||||
|
info!("LLM reply: {}", reply);
|
||||||
|
state.history.lock().push_assistant(reply.clone());
|
||||||
|
ipc::send(IpcEvent::LlmReply { text: reply });
|
||||||
|
voices::play_ok();
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("LLM request failed: {}", e);
|
||||||
|
state.history.lock().pop_last_user();
|
||||||
|
ipc::send(IpcEvent::LlmReply {
|
||||||
|
text: config::LLM_FALLBACK_ERROR_RU.to_string(),
|
||||||
|
});
|
||||||
|
voices::play_error();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
25
crates/jarvis-app/src/log.rs
Normal file
25
crates/jarvis-app/src/log.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
use simple_log::LogConfigBuilder;
|
||||||
|
|
||||||
|
use crate::config;
|
||||||
|
use crate::APP_LOG_DIR;
|
||||||
|
|
||||||
|
pub fn init_logging() -> Result<(), String> {
|
||||||
|
// configure logging
|
||||||
|
let config = LogConfigBuilder::builder()
|
||||||
|
.path(format!(
|
||||||
|
"{}/{}",
|
||||||
|
APP_LOG_DIR.get().unwrap().display(),
|
||||||
|
config::LOG_FILE_NAME
|
||||||
|
))
|
||||||
|
.size(1 * 100)
|
||||||
|
.roll_count(10)
|
||||||
|
.time_format("%Y-%m-%d %H:%M:%S.%f")
|
||||||
|
.level("debug")?
|
||||||
|
.output_file()
|
||||||
|
.output_console()
|
||||||
|
.build();
|
||||||
|
|
||||||
|
simple_log::new(config)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
177
crates/jarvis-app/src/main.rs
Normal file
177
crates/jarvis-app/src/main.rs
Normal file
|
|
@ -0,0 +1,177 @@
|
||||||
|
use jarvis_core::slots;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::sync::mpsc;
|
||||||
|
|
||||||
|
// include core
|
||||||
|
use jarvis_core::{
|
||||||
|
audio, audio_processing, commands, config, db, listener, recorder, stt, intent,
|
||||||
|
ipc::{self, IpcAction},
|
||||||
|
i18n, voices, models,
|
||||||
|
APP_CONFIG_DIR, APP_LOG_DIR, COMMANDS_LIST, DB,
|
||||||
|
};
|
||||||
|
|
||||||
|
// include log
|
||||||
|
#[macro_use]
|
||||||
|
extern crate simple_log;
|
||||||
|
mod log;
|
||||||
|
|
||||||
|
// include app
|
||||||
|
mod app;
|
||||||
|
mod llm_fallback;
|
||||||
|
|
||||||
|
// include tray
|
||||||
|
// @TODO. macOS currently not supported for tray functionality.
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
|
mod tray;
|
||||||
|
|
||||||
|
static SHOULD_STOP: AtomicBool = AtomicBool::new(false);
|
||||||
|
|
||||||
|
fn main() -> Result<(), String> {
|
||||||
|
// initialize directories
|
||||||
|
config::init_dirs()?;
|
||||||
|
|
||||||
|
// initialize logging
|
||||||
|
log::init_logging()?;
|
||||||
|
|
||||||
|
// log some base info
|
||||||
|
info!("Starting Jarvis v{} ...", config::APP_VERSION.unwrap());
|
||||||
|
info!("Config directory is: {}", APP_CONFIG_DIR.get().unwrap().display());
|
||||||
|
info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display());
|
||||||
|
|
||||||
|
// initialize settings
|
||||||
|
let settings = db::init();
|
||||||
|
|
||||||
|
// set global DB (for core modules that read settings at init time)
|
||||||
|
DB.set(settings.arc().clone())
|
||||||
|
.expect("DB already initialized");
|
||||||
|
|
||||||
|
// init voices
|
||||||
|
let voice_id = settings.lock().voice.clone();
|
||||||
|
let language = settings.lock().language.clone();
|
||||||
|
if let Err(e) = voices::init(&voice_id, &language) {
|
||||||
|
warn!("Failed to init voices: {}", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// init i18n
|
||||||
|
i18n::init(&settings.lock().language);
|
||||||
|
|
||||||
|
// init LLM fallback (no-op if GROQ_TOKEN missing)
|
||||||
|
llm_fallback::init();
|
||||||
|
|
||||||
|
// init recorder
|
||||||
|
if recorder::init().is_err() {
|
||||||
|
app::close(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// init models registry (scans available AI models)
|
||||||
|
if let Err(e) = models::init() {
|
||||||
|
warn!("Models registry init failed: {}", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// init stt engine
|
||||||
|
if stt::init().is_err() {
|
||||||
|
// @TODO. Allow continuing even without STT, if commands is using keywords or smthng?
|
||||||
|
app::close(1); // cannot continue without stt
|
||||||
|
}
|
||||||
|
|
||||||
|
// init commands
|
||||||
|
info!("Initializing commands.");
|
||||||
|
let cmds = match commands::parse_commands() {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Failed to parse commands: {}. Starting with empty command list.", e);
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
info!("Commands initialized. Count: {}, List: {:?}", cmds.len(), commands::list_paths(&cmds));
|
||||||
|
COMMANDS_LIST.set(cmds).unwrap();
|
||||||
|
|
||||||
|
// init audio
|
||||||
|
if audio::init().is_err() {
|
||||||
|
// @TODO. Allow continuing even without audio?
|
||||||
|
app::close(1); // cannot continue without audio
|
||||||
|
}
|
||||||
|
|
||||||
|
// init wake-word engine
|
||||||
|
if let Err(e) = listener::init() {
|
||||||
|
error!("Wake-word engine init failed: {}", e);
|
||||||
|
app::close(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// shared async runtime for intent classification, IPC, etc.
|
||||||
|
let rt = Arc::new(
|
||||||
|
tokio::runtime::Runtime::new().expect("Failed to create tokio runtime")
|
||||||
|
);
|
||||||
|
|
||||||
|
// init intent-recognition engine
|
||||||
|
rt.block_on(async {
|
||||||
|
if let Err(e) = intent::init(COMMANDS_LIST.get().unwrap()).await {
|
||||||
|
error!("Failed to initialize intent classifier: {}", e);
|
||||||
|
app::close(1);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// init slots parsing engine
|
||||||
|
slots::init().map_err(|e| error!("Slot extraction init failed: {}", e)).ok();
|
||||||
|
|
||||||
|
// init audio processing
|
||||||
|
info!("Initializing audio processing...");
|
||||||
|
if let Err(e) = audio_processing::init() {
|
||||||
|
warn!("Audio processing init failed: {}", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// init IPC
|
||||||
|
info!("Initializing IPC...");
|
||||||
|
ipc::init();
|
||||||
|
|
||||||
|
// channel for text commands (manually written in the GUI)
|
||||||
|
let (text_cmd_tx, text_cmd_rx) = mpsc::channel::<String>();
|
||||||
|
|
||||||
|
ipc::set_action_handler(move |action| {
|
||||||
|
match action {
|
||||||
|
IpcAction::Stop => {
|
||||||
|
info!("Received stop command from GUI");
|
||||||
|
SHOULD_STOP.store(true, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
IpcAction::ReloadCommands => {
|
||||||
|
info!("Received reload commands request");
|
||||||
|
// TODO: implement reload
|
||||||
|
}
|
||||||
|
IpcAction::SetMuted { muted } => {
|
||||||
|
info!("Received mute request: {}", muted);
|
||||||
|
// TODO: implement mute
|
||||||
|
}
|
||||||
|
IpcAction::TextCommand { text } => {
|
||||||
|
info!("Received text command: {}", text);
|
||||||
|
if let Err(e) = text_cmd_tx.send(text) {
|
||||||
|
error!("Failed to send text command to app: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
IpcAction::Ping => {
|
||||||
|
// handled internally by server
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// start WebSocket server on the shared runtime
|
||||||
|
let ipc_rt = Arc::clone(&rt);
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
ipc_rt.block_on(ipc::start_server());
|
||||||
|
});
|
||||||
|
|
||||||
|
// start the app (in the background thread)
|
||||||
|
let app_rt = Arc::clone(&rt);
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
let _ = app::start(text_cmd_rx, &app_rt);
|
||||||
|
});
|
||||||
|
|
||||||
|
tray::init_blocking(settings);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn should_stop() -> bool {
|
||||||
|
SHOULD_STOP.load(Ordering::SeqCst)
|
||||||
|
}
|
||||||
226
crates/jarvis-app/src/tray.rs
Normal file
226
crates/jarvis-app/src/tray.rs
Normal file
|
|
@ -0,0 +1,226 @@
|
||||||
|
mod menu;
|
||||||
|
|
||||||
|
use tray_icon::{
|
||||||
|
menu::MenuEvent,
|
||||||
|
TrayIconBuilder,
|
||||||
|
};
|
||||||
|
use image;
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
#[cfg(target_os="windows")]
|
||||||
|
use winit::platform::windows::EventLoopBuilderExtWindows;
|
||||||
|
|
||||||
|
use jarvis_core::{config, i18n, voices, ipc::{self, IpcEvent}, SettingsManager};
|
||||||
|
|
||||||
|
const TRAY_ICON_BYTES: &[u8] = include_bytes!("../../../resources/icons/32x32.png");
|
||||||
|
|
||||||
|
pub fn init_blocking(settings: SettingsManager) {
|
||||||
|
let icon = load_icon_from_bytes(TRAY_ICON_BYTES);
|
||||||
|
|
||||||
|
// build menu with settings submenus
|
||||||
|
let tray_menu = menu::build(&settings);
|
||||||
|
let menu::TrayMenu { menu, state: tray_state } = tray_menu;
|
||||||
|
|
||||||
|
let _tray_icon = TrayIconBuilder::new()
|
||||||
|
.with_menu(Box::new(menu))
|
||||||
|
.with_tooltip(i18n::t("tray-tooltip"))
|
||||||
|
.with_icon(icon)
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let menu_channel = MenuEvent::receiver();
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
{
|
||||||
|
gtk::init().unwrap();
|
||||||
|
glib::timeout_add_local(std::time::Duration::from_millis(100), move || {
|
||||||
|
if let Ok(event) = menu_channel.try_recv() {
|
||||||
|
handle_menu_event(&event, &settings, &tray_state);
|
||||||
|
}
|
||||||
|
glib::ControlFlow::Continue
|
||||||
|
});
|
||||||
|
gtk::main();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
{
|
||||||
|
use winit::event_loop::{EventLoop, ControlFlow};
|
||||||
|
let event_loop = EventLoop::new().unwrap();
|
||||||
|
event_loop.run(move |_event, elwt| {
|
||||||
|
elwt.set_control_flow(ControlFlow::Wait);
|
||||||
|
if let Ok(event) = menu_channel.try_recv() {
|
||||||
|
handle_menu_event(&event, &settings, &tray_state);
|
||||||
|
}
|
||||||
|
}).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
{
|
||||||
|
loop {
|
||||||
|
if let Ok(event) = menu_channel.try_recv() {
|
||||||
|
handle_menu_event(&event, &settings, &tray_state);
|
||||||
|
}
|
||||||
|
|
||||||
|
// pump Windows messages
|
||||||
|
unsafe {
|
||||||
|
let mut msg: winapi::um::winuser::MSG = std::mem::zeroed();
|
||||||
|
while winapi::um::winuser::PeekMessageW(
|
||||||
|
&mut msg,
|
||||||
|
std::ptr::null_mut(),
|
||||||
|
0, 0,
|
||||||
|
winapi::um::winuser::PM_REMOVE
|
||||||
|
) != 0 {
|
||||||
|
winapi::um::winuser::TranslateMessage(&msg);
|
||||||
|
winapi::um::winuser::DispatchMessageW(&msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Tray initialized.");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle_menu_event(event: &MenuEvent, settings: &SettingsManager, tray_state: &menu::TrayState) {
|
||||||
|
let id = event.id.0.as_str();
|
||||||
|
|
||||||
|
// -- radio group: "set:key:value"
|
||||||
|
if let Some(rest) = id.strip_prefix("set:") {
|
||||||
|
if let Some((key, value)) = rest.split_once(':') {
|
||||||
|
match settings.write(key, value) {
|
||||||
|
Ok(()) => {
|
||||||
|
info!("Tray: {} = {}", key, value);
|
||||||
|
|
||||||
|
// update check marks in the radio group
|
||||||
|
for group in &tray_state.radio_groups {
|
||||||
|
if group.setting_key == key {
|
||||||
|
group.select(value);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// apply side effects
|
||||||
|
match key {
|
||||||
|
"language" => {
|
||||||
|
i18n::set_language(value);
|
||||||
|
}
|
||||||
|
"assistant_voice" => {
|
||||||
|
voices::set_current_voice(value);
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Tray: failed to set {} = {}: {}", key, value, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- toggle: "toggle:key"
|
||||||
|
if let Some(key) = id.strip_prefix("toggle:") {
|
||||||
|
match key {
|
||||||
|
"gain_normalizer" => {
|
||||||
|
// CheckMenuItem auto-toggles on click, just read the new state
|
||||||
|
let new_val = tray_state.gain_toggle.is_checked();
|
||||||
|
let val_str = if new_val { "true" } else { "false" };
|
||||||
|
if let Err(e) = settings.write(key, val_str) {
|
||||||
|
warn!("Tray: failed to toggle {}: {}", key, e);
|
||||||
|
// revert visual state on error
|
||||||
|
tray_state.gain_toggle.set_checked(!new_val);
|
||||||
|
} else {
|
||||||
|
info!("Tray: {} = {}", key, val_str);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- action items
|
||||||
|
match id {
|
||||||
|
"exit" => std::process::exit(0),
|
||||||
|
"restart" => {
|
||||||
|
info!("Restarting from tray menu...");
|
||||||
|
restart_app();
|
||||||
|
}
|
||||||
|
"settings" => {
|
||||||
|
info!("Opening settings from tray menu...");
|
||||||
|
open_settings();
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// HELPERS
|
||||||
|
|
||||||
|
fn load_icon_from_bytes(bytes: &[u8]) -> tray_icon::Icon {
|
||||||
|
let image = image::load_from_memory(bytes)
|
||||||
|
.expect("Failed to load icon")
|
||||||
|
.into_rgba8();
|
||||||
|
let (width, height) = image.dimensions();
|
||||||
|
let rgba = image.into_raw();
|
||||||
|
tray_icon::Icon::from_rgba(rgba, width, height).expect("Failed to create icon")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restart_app() {
|
||||||
|
let exe_path = match std::env::current_exe() {
|
||||||
|
Ok(path) => path,
|
||||||
|
Err(e) => {
|
||||||
|
error!("Failed to get executable path: {}", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match Command::new(&exe_path).spawn() {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Spawned new instance, exiting current...");
|
||||||
|
std::process::exit(0);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Failed to restart: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_settings() {
|
||||||
|
if ipc::has_clients() {
|
||||||
|
info!("GUI is connected, sending reveal event");
|
||||||
|
ipc::send(IpcEvent::RevealWindow);
|
||||||
|
} else {
|
||||||
|
info!("GUI not connected, launching jarvis-gui");
|
||||||
|
launch_gui();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn launch_gui() {
|
||||||
|
let exe_path = match std::env::current_exe() {
|
||||||
|
Ok(path) => path,
|
||||||
|
Err(e) => {
|
||||||
|
error!("Failed to get executable path: {}", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let gui_path = exe_path.parent()
|
||||||
|
.map(|p| p.join(get_gui_executable_name()))
|
||||||
|
.unwrap_or_else(|| get_gui_executable_name().into());
|
||||||
|
|
||||||
|
info!("Launching GUI: {:?}", gui_path);
|
||||||
|
|
||||||
|
match Command::new(&gui_path).spawn() {
|
||||||
|
Ok(_) => info!("Launched jarvis-gui"),
|
||||||
|
Err(e) => error!("Failed to launch jarvis-gui: {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
fn get_gui_executable_name() -> &'static str {
|
||||||
|
"jarvis-gui.exe"
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(target_os = "windows"))]
|
||||||
|
fn get_gui_executable_name() -> &'static str {
|
||||||
|
"jarvis-gui"
|
||||||
|
}
|
||||||
182
crates/jarvis-app/src/tray/menu.rs
Normal file
182
crates/jarvis-app/src/tray/menu.rs
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
use tray_icon::menu::{CheckMenuItem, Menu, MenuItem, PredefinedMenuItem, Submenu};
|
||||||
|
|
||||||
|
use jarvis_core::{i18n, voices, SettingsManager};
|
||||||
|
use jarvis_core::config::structs::{WakeWordEngine, NoiseSuppressionBackend};
|
||||||
|
|
||||||
|
// RADIO GROUP
|
||||||
|
|
||||||
|
// a group of check menu items where only one can be active at a time.
|
||||||
|
// stores (menu_item, setting_value) pairs.
|
||||||
|
pub struct RadioGroup {
|
||||||
|
pub setting_key: String,
|
||||||
|
pub items: Vec<(CheckMenuItem, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RadioGroup {
|
||||||
|
pub fn select(&self, value: &str) {
|
||||||
|
for (item, val) in &self.items {
|
||||||
|
item.set_checked(val == value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TRAY MENU STATE
|
||||||
|
|
||||||
|
pub struct TrayMenu {
|
||||||
|
pub menu: Menu,
|
||||||
|
pub state: TrayState,
|
||||||
|
}
|
||||||
|
|
||||||
|
// holds references to menu items for updating check marks after build
|
||||||
|
pub struct TrayState {
|
||||||
|
pub radio_groups: Vec<RadioGroup>,
|
||||||
|
pub gain_toggle: CheckMenuItem,
|
||||||
|
}
|
||||||
|
|
||||||
|
// BUILD
|
||||||
|
|
||||||
|
pub fn build(settings: &SettingsManager) -> TrayMenu {
|
||||||
|
let menu = Menu::new();
|
||||||
|
|
||||||
|
let mut radio_groups = Vec::new();
|
||||||
|
|
||||||
|
// -- language submenu
|
||||||
|
let lang_sub = Submenu::new(i18n::t("tray-language"), true);
|
||||||
|
let current_lang = settings.read("language").unwrap_or_default();
|
||||||
|
let mut lang_items = Vec::new();
|
||||||
|
for &lang in i18n::SUPPORTED_LANGUAGES {
|
||||||
|
let label = match lang {
|
||||||
|
"ru" => "Русский",
|
||||||
|
"en" => "English",
|
||||||
|
"ua" => "Українська",
|
||||||
|
_ => lang,
|
||||||
|
};
|
||||||
|
let item = CheckMenuItem::with_id(
|
||||||
|
format!("set:language:{}", lang),
|
||||||
|
label,
|
||||||
|
true,
|
||||||
|
lang == current_lang,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let _ = lang_sub.append(&item);
|
||||||
|
lang_items.push((item, lang.to_string()));
|
||||||
|
}
|
||||||
|
radio_groups.push(RadioGroup {
|
||||||
|
setting_key: "language".to_string(),
|
||||||
|
items: lang_items,
|
||||||
|
});
|
||||||
|
|
||||||
|
// -- voice submenu
|
||||||
|
let voice_sub = Submenu::new(i18n::t("tray-voice"), true);
|
||||||
|
let current_voice = voices::get_current_voice()
|
||||||
|
.map(|v| v.voice.id.clone())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let mut voice_items = Vec::new();
|
||||||
|
for voice in voices::list_voices() {
|
||||||
|
let item = CheckMenuItem::with_id(
|
||||||
|
format!("set:assistant_voice:{}", voice.voice.id),
|
||||||
|
&voice.voice.name,
|
||||||
|
true,
|
||||||
|
voice.voice.id == current_voice,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let _ = voice_sub.append(&item);
|
||||||
|
voice_items.push((item, voice.voice.id.clone()));
|
||||||
|
}
|
||||||
|
radio_groups.push(RadioGroup {
|
||||||
|
setting_key: "assistant_voice".to_string(),
|
||||||
|
items: voice_items,
|
||||||
|
});
|
||||||
|
|
||||||
|
// -- wake word engine submenu
|
||||||
|
let ww_sub = Submenu::new(i18n::t("tray-wake-word"), true);
|
||||||
|
let current_ww = settings.read("selected_wake_word_engine").unwrap_or_default();
|
||||||
|
let mut ww_items = Vec::new();
|
||||||
|
for (label, value) in &[("Rustpotter", "Rustpotter"), ("Vosk", "Vosk")] {
|
||||||
|
let item = CheckMenuItem::with_id(
|
||||||
|
format!("set:selected_wake_word_engine:{}", value.to_lowercase()),
|
||||||
|
*label,
|
||||||
|
true,
|
||||||
|
current_ww == *label,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let _ = ww_sub.append(&item);
|
||||||
|
ww_items.push((item, value.to_lowercase()));
|
||||||
|
}
|
||||||
|
radio_groups.push(RadioGroup {
|
||||||
|
setting_key: "selected_wake_word_engine".to_string(),
|
||||||
|
items: ww_items,
|
||||||
|
});
|
||||||
|
|
||||||
|
// -- noise suppression submenu
|
||||||
|
let ns_sub = Submenu::new(i18n::t("tray-noise-suppression"), true);
|
||||||
|
let current_ns = settings.read("noise_suppression").unwrap_or_default();
|
||||||
|
let mut ns_items = Vec::new();
|
||||||
|
for (label, value) in &[("None", "none"), ("Nnnoiseless", "nnnoiseless")] {
|
||||||
|
let item = CheckMenuItem::with_id(
|
||||||
|
format!("set:noise_suppression:{}", value),
|
||||||
|
*label,
|
||||||
|
true,
|
||||||
|
current_ns.to_lowercase() == *value,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let _ = ns_sub.append(&item);
|
||||||
|
ns_items.push((item, value.to_string()));
|
||||||
|
}
|
||||||
|
radio_groups.push(RadioGroup {
|
||||||
|
setting_key: "noise_suppression".to_string(),
|
||||||
|
items: ns_items,
|
||||||
|
});
|
||||||
|
|
||||||
|
// -- vad submenu
|
||||||
|
let vad_sub = Submenu::new(i18n::t("tray-vad"), true);
|
||||||
|
let current_vad = settings.read("vad_backend").unwrap_or_default();
|
||||||
|
let mut vad_items = Vec::new();
|
||||||
|
for (label, value) in &[("None", "none"), ("Energy", "energy"), ("Nnnoiseless", "nnnoiseless")] {
|
||||||
|
let item = CheckMenuItem::with_id(
|
||||||
|
format!("set:vad_backend:{}", value),
|
||||||
|
*label,
|
||||||
|
true,
|
||||||
|
current_vad == *value,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let _ = vad_sub.append(&item);
|
||||||
|
vad_items.push((item, value.to_string()));
|
||||||
|
}
|
||||||
|
radio_groups.push(RadioGroup {
|
||||||
|
setting_key: "vad_backend".to_string(),
|
||||||
|
items: vad_items,
|
||||||
|
});
|
||||||
|
|
||||||
|
// -- gain normalizer toggle
|
||||||
|
let gain_on = settings.read("gain_normalizer")
|
||||||
|
.map(|v| v == "true")
|
||||||
|
.unwrap_or(true);
|
||||||
|
let gain_toggle = CheckMenuItem::with_id(
|
||||||
|
"toggle:gain_normalizer",
|
||||||
|
i18n::t("tray-gain-normalizer"),
|
||||||
|
true,
|
||||||
|
gain_on,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
// -- assemble main menu
|
||||||
|
let _ = menu.append(&lang_sub);
|
||||||
|
let _ = menu.append(&voice_sub);
|
||||||
|
let _ = menu.append(&ww_sub);
|
||||||
|
let _ = menu.append(&ns_sub);
|
||||||
|
let _ = menu.append(&vad_sub);
|
||||||
|
let _ = menu.append(&gain_toggle);
|
||||||
|
let _ = menu.append(&PredefinedMenuItem::separator());
|
||||||
|
let _ = menu.append(&MenuItem::with_id("restart", i18n::t("tray-restart"), true, None));
|
||||||
|
let _ = menu.append(&MenuItem::with_id("settings", i18n::t("tray-settings"), true, None));
|
||||||
|
let _ = menu.append(&MenuItem::with_id("exit", i18n::t("tray-exit"), true, None));
|
||||||
|
|
||||||
|
TrayMenu {
|
||||||
|
menu,
|
||||||
|
state: TrayState {
|
||||||
|
radio_groups,
|
||||||
|
gain_toggle,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
14
crates/jarvis-cli/Cargo.toml
Normal file
14
crates/jarvis-cli/Cargo.toml
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
[package]
|
||||||
|
name = "jarvis-cli"
|
||||||
|
version.workspace = true
|
||||||
|
authors.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
repository.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
jarvis-core = { path = "../jarvis-core", default-features = false, features = ["intent", "llm"]}
|
||||||
|
tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros"] }
|
||||||
|
log.workspace = true
|
||||||
|
env_logger = "0.11"
|
||||||
|
parking_lot.workspace = true
|
||||||
7
crates/jarvis-cli/build.rs
Normal file
7
crates/jarvis-cli/build.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
fn main() {
|
||||||
|
let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||||
|
let lib_path = std::path::Path::new(&manifest_dir)
|
||||||
|
.join("..\\..\\lib\\windows\\amd64");
|
||||||
|
|
||||||
|
println!("cargo:rustc-link-search=native={}", lib_path.display());
|
||||||
|
}
|
||||||
246
crates/jarvis-cli/src/main.rs
Normal file
246
crates/jarvis-cli/src/main.rs
Normal file
|
|
@ -0,0 +1,246 @@
|
||||||
|
use std::io::{self, Write};
|
||||||
|
|
||||||
|
use jarvis_core::llm::{ChatMessage, LlmClient};
|
||||||
|
use jarvis_core::{COMMANDS_LIST, DB, JCommandsList, commands, config, db, intent};
|
||||||
|
|
||||||
|
const ASK_MAX_TOKENS: u32 = 512;
|
||||||
|
|
||||||
|
fn print_help() {
|
||||||
|
println!("
|
||||||
|
--## Jarvis CLI - Testing Tool ##--
|
||||||
|
|
||||||
|
Commands:
|
||||||
|
classify <text> - Test intent classification
|
||||||
|
execute <text> - Simulate voice input and execute command
|
||||||
|
ask <prompt> - Send prompt to Groq LLM and print reply
|
||||||
|
list - List all loaded commands
|
||||||
|
phrases - List all training phrases
|
||||||
|
hash - Show commands hash
|
||||||
|
settings - Dump all settings
|
||||||
|
help - Show this help
|
||||||
|
exit - Exit the CLI
|
||||||
|
");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_commands(commands: &[JCommandsList]) {
|
||||||
|
println!("\n[ Loaded Commands ]");
|
||||||
|
for cmd_list in commands {
|
||||||
|
println!(" 📁 {}", cmd_list.path.display());
|
||||||
|
for cmd in &cmd_list.commands {
|
||||||
|
println!(" ├─ id: {}", cmd.id);
|
||||||
|
println!(" ├─ type: {}", cmd.cmd_type);
|
||||||
|
println!(" └─ phrases: {} languages", cmd.phrases.len());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_phrases(commands: &[JCommandsList]) {
|
||||||
|
println!("\n[ Training Phrases ]");
|
||||||
|
for cmd_list in commands {
|
||||||
|
for cmd in &cmd_list.commands {
|
||||||
|
println!(" [{}]", cmd.id);
|
||||||
|
for (lang, phrases) in &cmd.phrases {
|
||||||
|
println!(" lang: {}", lang);
|
||||||
|
for phrase in phrases {
|
||||||
|
println!(" - {}", phrase);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!();
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn classify_text(text: &str) {
|
||||||
|
match intent::classify(text).await {
|
||||||
|
Some((intent_id, confidence)) => {
|
||||||
|
println!(" ✓ Intent: {} (confidence: {:.2}%)", intent_id, confidence * 100.0);
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
println!(" ✗ No intent matched (below threshold)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn execute_text(commands: &[JCommandsList], text: &str) {
|
||||||
|
// try intent classification first
|
||||||
|
if let Some((intent_id, confidence)) = intent::classify(text).await {
|
||||||
|
println!(" Intent: {} (confidence: {:.2}%)", intent_id, confidence * 100.0);
|
||||||
|
|
||||||
|
if let Some((cmd_path, cmd)) = intent::get_command_by_intent(commands, &intent_id) {
|
||||||
|
println!(" Command: {:?}", cmd_path);
|
||||||
|
println!(" Type: {}", cmd.cmd_type);
|
||||||
|
println!(" Executing...");
|
||||||
|
|
||||||
|
match commands::execute_command(cmd_path, cmd, Some(text), None) {
|
||||||
|
Ok(chain) => println!(" ✓ Success (chain: {})", chain),
|
||||||
|
Err(e) => println!(" ✗ Error: {}", e),
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fallback to levenshtein
|
||||||
|
println!(" Intent not matched, trying levenshtein fallback...");
|
||||||
|
if let Some((cmd_path, cmd)) = commands::fetch_command(text, commands) {
|
||||||
|
println!(" Command: {:?}", cmd_path);
|
||||||
|
println!(" Type: {}", cmd.cmd_type);
|
||||||
|
println!(" Executing...");
|
||||||
|
|
||||||
|
match commands::execute_command(cmd_path, cmd, Some(text), None) {
|
||||||
|
Ok(chain) => println!(" ✓ Success (chain: {})", chain),
|
||||||
|
Err(e) => println!(" ✗ Error: {}", e),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
println!(" ✗ No command matched");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ask_llm(prompt: &str) -> i32 {
|
||||||
|
let client = match LlmClient::from_env() {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("LLM not configured: {}", e);
|
||||||
|
eprintln!("Set GROQ_TOKEN (and optionally GROQ_BASE_URL, GROQ_MODEL).");
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let messages = [ChatMessage::user(prompt)];
|
||||||
|
match client.complete(&messages, ASK_MAX_TOKENS) {
|
||||||
|
Ok(reply) => {
|
||||||
|
println!("{}", reply);
|
||||||
|
0
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("LLM request failed: {}", e);
|
||||||
|
1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
let argv: Vec<String> = std::env::args().collect();
|
||||||
|
if argv.len() >= 2 && argv[1] == "ask" {
|
||||||
|
if argv.len() < 3 {
|
||||||
|
eprintln!("Usage: jarvis-cli ask <prompt>");
|
||||||
|
std::process::exit(2);
|
||||||
|
}
|
||||||
|
let prompt = argv[2..].join(" ");
|
||||||
|
std::process::exit(ask_llm(&prompt));
|
||||||
|
}
|
||||||
|
|
||||||
|
// init logging
|
||||||
|
env_logger::Builder::from_env(
|
||||||
|
env_logger::Env::default().default_filter_or("info")
|
||||||
|
).init();
|
||||||
|
|
||||||
|
println!("Jarvis CLI v{}", config::APP_VERSION.unwrap_or("unknown"));
|
||||||
|
|
||||||
|
// init dirs
|
||||||
|
config::init_dirs()?;
|
||||||
|
|
||||||
|
// init settings
|
||||||
|
let settings = db::init();
|
||||||
|
DB.set(settings.arc().clone())
|
||||||
|
.expect("DB already initialized");
|
||||||
|
|
||||||
|
// parse commands
|
||||||
|
println!("\n[*] Loading commands...");
|
||||||
|
let cmds = match commands::parse_commands() {
|
||||||
|
Ok(c) => {
|
||||||
|
println!(" Loaded {} command groups", c.len());
|
||||||
|
c
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!(" Warning: {}", e);
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
COMMANDS_LIST.set(cmds).expect("Failed to set commands list");
|
||||||
|
|
||||||
|
// init intent classifier
|
||||||
|
println!("[*] Initializing intent classifier...");
|
||||||
|
match intent::init(COMMANDS_LIST.get().unwrap()).await {
|
||||||
|
Ok(_) => println!(" Intent classifier ready"),
|
||||||
|
Err(e) => println!(" Warning: {}", e),
|
||||||
|
}
|
||||||
|
|
||||||
|
// init sound
|
||||||
|
println!("[*] Initializing audio...");
|
||||||
|
if let Err(e) = jarvis_core::audio::init() {
|
||||||
|
println!(" Warning: Audio init failed: {:?}", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
print_help();
|
||||||
|
|
||||||
|
// REPL loop
|
||||||
|
let mut input = String::new();
|
||||||
|
loop {
|
||||||
|
print!("jarvis> ");
|
||||||
|
io::stdout().flush()?;
|
||||||
|
|
||||||
|
input.clear();
|
||||||
|
io::stdin().read_line(&mut input)?;
|
||||||
|
let input = input.trim();
|
||||||
|
|
||||||
|
if input.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let parts: Vec<&str> = input.splitn(2, ' ').collect();
|
||||||
|
let cmd = parts[0];
|
||||||
|
let arg = parts.get(1).copied().unwrap_or("");
|
||||||
|
|
||||||
|
match cmd {
|
||||||
|
"exit" | "quit" | "q" => {
|
||||||
|
println!("Bye!");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
"help" | "h" | "?" => print_help(),
|
||||||
|
"list" | "ls" => list_commands(COMMANDS_LIST.get().unwrap()),
|
||||||
|
"phrases" => list_phrases(COMMANDS_LIST.get().unwrap()),
|
||||||
|
"hash" => {
|
||||||
|
let hash = commands::commands_hash(COMMANDS_LIST.get().unwrap());
|
||||||
|
println!(" Commands hash: {}", hash);
|
||||||
|
}
|
||||||
|
"settings" => {
|
||||||
|
println!("\n[ Current Settings ]");
|
||||||
|
for (key, val) in settings.dump() {
|
||||||
|
println!(" {} = {}", key, val);
|
||||||
|
}
|
||||||
|
println!();
|
||||||
|
}
|
||||||
|
"classify" | "c" => {
|
||||||
|
if arg.is_empty() {
|
||||||
|
println!(" Usage: classify <text>");
|
||||||
|
} else {
|
||||||
|
classify_text(arg).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"execute" | "exec" | "e" => {
|
||||||
|
if arg.is_empty() {
|
||||||
|
println!(" Usage: execute <text>");
|
||||||
|
} else {
|
||||||
|
execute_text(COMMANDS_LIST.get().unwrap(), arg).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"ask" | "a" => {
|
||||||
|
if arg.is_empty() {
|
||||||
|
println!(" Usage: ask <prompt>");
|
||||||
|
} else {
|
||||||
|
ask_llm(arg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"reload" => {
|
||||||
|
println!(" Note: Reload requires app restart (statics can't be reset)");
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
// treat unknown commands as text to classify
|
||||||
|
classify_text(input).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
69
crates/jarvis-core/Cargo.toml
Normal file
69
crates/jarvis-core/Cargo.toml
Normal file
|
|
@ -0,0 +1,69 @@
|
||||||
|
[package]
|
||||||
|
name = "jarvis-core"
|
||||||
|
version.workspace = true
|
||||||
|
authors.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
repository.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
serde.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
|
serde_yaml.workspace = true
|
||||||
|
once_cell.workspace = true
|
||||||
|
log.workspace = true
|
||||||
|
rand.workspace = true
|
||||||
|
seqdiff.workspace = true
|
||||||
|
hound.workspace = true
|
||||||
|
platform-dirs.workspace = true
|
||||||
|
rodio.workspace = true
|
||||||
|
kira.workspace = true
|
||||||
|
pv_recorder.workspace = true
|
||||||
|
rustpotter.workspace = true
|
||||||
|
parking_lot.workspace = true
|
||||||
|
toml.workspace = true
|
||||||
|
sha2.workspace = true
|
||||||
|
nnnoiseless = { workspace = true, optional = true }
|
||||||
|
webrtc-vad = { workspace = true, optional = true }
|
||||||
|
tokio-tungstenite = { workspace = true, optional = true }
|
||||||
|
futures-util = { workspace = true, optional = true }
|
||||||
|
fluent.workspace = true
|
||||||
|
fluent-bundle.workspace = true
|
||||||
|
unic-langid.workspace = true
|
||||||
|
chrono.workspace = true
|
||||||
|
sys-locale.workspace = true
|
||||||
|
|
||||||
|
# pv_recorder = { workspace = true, optional = true }
|
||||||
|
vosk = { version = "0.3.1", optional = true }
|
||||||
|
intent-classifier = { version = "0.1.0", optional = true }
|
||||||
|
# rustpotter = { workspace = true, optional = true }
|
||||||
|
|
||||||
|
tokio = { version = "1", features = ["sync"], optional = true }
|
||||||
|
|
||||||
|
mlua = { workspace = true, optional = true }
|
||||||
|
reqwest = { workspace = true, optional = true }
|
||||||
|
thiserror = { workspace = true, optional = true }
|
||||||
|
tempfile.workspace = true
|
||||||
|
|
||||||
|
fastembed = { workspace = true, optional = true }
|
||||||
|
ort = { workspace = true, optional = true }
|
||||||
|
ndarray = { workspace = true, optional = true }
|
||||||
|
tokenizers = { workspace = true, optional = true }
|
||||||
|
regex = { workspace = true, optional = true }
|
||||||
|
|
||||||
|
[target.'cfg(windows)'.dependencies]
|
||||||
|
winrt-notification = { workspace = true, optional = true }
|
||||||
|
|
||||||
|
[features]
|
||||||
|
default = ["jarvis_app"]
|
||||||
|
jarvis_app = [
|
||||||
|
"vosk", "intent-classifier", "fastembed", "tokio", "nnnoiseless", "tokio-tungstenite", "futures-util",
|
||||||
|
"lua", "llm",
|
||||||
|
"ort", "ndarray", "tokenizers", "regex",
|
||||||
|
"webrtc-vad",
|
||||||
|
]
|
||||||
|
|
||||||
|
intent = ["intent-classifier", "tokio"]
|
||||||
|
lua = ["mlua", "reqwest", "winrt-notification"]
|
||||||
|
lua_only = ["lua", "tokio"]
|
||||||
|
llm = ["reqwest", "thiserror"]
|
||||||
92
crates/jarvis-core/src/audio.rs
Normal file
92
crates/jarvis-core/src/audio.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
mod kira;
|
||||||
|
mod rodio;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use crate::config::structs::AudioType;
|
||||||
|
use crate::{config, DB, SOUND_DIR};
|
||||||
|
|
||||||
|
static AUDIO_TYPE: OnceCell<AudioType> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), ()> {
|
||||||
|
if AUDIO_TYPE.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
} // already initialized
|
||||||
|
|
||||||
|
// set default audio type
|
||||||
|
// @TODO. Make it configurable?
|
||||||
|
AUDIO_TYPE.set(config::DEFAULT_AUDIO_TYPE).unwrap();
|
||||||
|
|
||||||
|
// load given audio backend
|
||||||
|
match AUDIO_TYPE.get().unwrap() {
|
||||||
|
AudioType::Rodio => {
|
||||||
|
// Init Rodio
|
||||||
|
info!("Initializing Rodio audio backend.");
|
||||||
|
|
||||||
|
match rodio::init() {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Successfully initialized Rodio audio backend.");
|
||||||
|
}
|
||||||
|
Err(()) => {
|
||||||
|
error!("Failed to initialize Rodio audio backend.");
|
||||||
|
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
AudioType::Kira => {
|
||||||
|
// Init Kira
|
||||||
|
info!("Initializing Kira audio backend.");
|
||||||
|
|
||||||
|
match kira::init() {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Successfully initialized Kira audio backend.");
|
||||||
|
}
|
||||||
|
Err(_msg) => {
|
||||||
|
error!("Failed to initialize Kira audio backend.");
|
||||||
|
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn play_sound(filename: &PathBuf) {
|
||||||
|
let audio_type = match AUDIO_TYPE.get() {
|
||||||
|
Some(t) => t,
|
||||||
|
None => {
|
||||||
|
warn!("Audio not initialized, cannot play: {}", filename.display());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
info!("Playing {}", filename.display());
|
||||||
|
|
||||||
|
match audio_type {
|
||||||
|
AudioType::Rodio => {
|
||||||
|
rodio::play_sound(filename, true);
|
||||||
|
}
|
||||||
|
AudioType::Kira => kira::play_sound(filename),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_sound_directory() -> Option<PathBuf> {
|
||||||
|
let db = DB.get()?;
|
||||||
|
|
||||||
|
let voice_path = {
|
||||||
|
let s = db.read();
|
||||||
|
SOUND_DIR.join(&s.voice)
|
||||||
|
};
|
||||||
|
|
||||||
|
match voice_path.exists() {
|
||||||
|
true => Some(voice_path),
|
||||||
|
_ => {
|
||||||
|
error!("No sounds folder found. Search path - {:?}", voice_path);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
62
crates/jarvis-core/src/audio/kira.rs
Normal file
62
crates/jarvis-core/src/audio/kira.rs
Normal file
|
|
@ -0,0 +1,62 @@
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
// use kira::{
|
||||||
|
// manager::{backend::DefaultBackend, AudioManager, AudioManagerSettings},
|
||||||
|
// sound::static_sound::{StaticSoundData, StaticSoundSettings},
|
||||||
|
// };
|
||||||
|
|
||||||
|
use kira::{
|
||||||
|
AudioManager, AudioManagerSettings, DefaultBackend,
|
||||||
|
sound::static_sound::StaticSoundData,
|
||||||
|
};
|
||||||
|
|
||||||
|
static MANAGER: OnceCell<Mutex<AudioManager>> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), ()> {
|
||||||
|
if MANAGER.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
} // already initialized
|
||||||
|
|
||||||
|
// Create an audio manager. This plays sounds and manages resources.
|
||||||
|
match AudioManager::<DefaultBackend>::new(AudioManagerSettings::default()) {
|
||||||
|
Ok(manager) => {
|
||||||
|
// store
|
||||||
|
MANAGER.set(Mutex::new(manager)).ok();
|
||||||
|
|
||||||
|
// success
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Failed to initialize audio stream.\nError details: {}", msg);
|
||||||
|
|
||||||
|
// failed
|
||||||
|
Err(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// @TODO. Cache sounds in memory? With a pool of a certain size, for instance.
|
||||||
|
pub fn play_sound(filename: &PathBuf) {
|
||||||
|
// load the file
|
||||||
|
match StaticSoundData::from_file(filename) {
|
||||||
|
Ok(sound_data) => {
|
||||||
|
// sound_data.duration() can be used in order to sleep, if (for some reason) blocking behaviour is required
|
||||||
|
|
||||||
|
// play it (non-blocking)
|
||||||
|
if let Some(manager) = MANAGER.get() {
|
||||||
|
if let Ok(mut audio_manager) = manager.lock() {
|
||||||
|
if let Err(e) = audio_manager.play(sound_data) {
|
||||||
|
warn!("Failed to play sound: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
warn!("Audio manager not initialized");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
warn!("Cannot find sound file: {} (err: {})", filename.display(), err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
65
crates/jarvis-core/src/audio/rodio.rs
Normal file
65
crates/jarvis-core/src/audio/rodio.rs
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
/*
|
||||||
|
Abandoned temporary.
|
||||||
|
Problems with blocking behaviour.
|
||||||
|
Possible fixes are running rodio in a separate thread or smthng.
|
||||||
|
*/
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use std::fs::File;
|
||||||
|
use std::io::BufReader;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
// use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink};
|
||||||
|
use rodio::{Decoder, OutputStream, Sink};
|
||||||
|
|
||||||
|
// static STREAM: OnceCell<OutputStream> = OnceCell::new();
|
||||||
|
static STREAM_HANDLE: OnceCell<OutputStream> = OnceCell::new();
|
||||||
|
static SINK: OnceCell<Sink> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), ()> {
|
||||||
|
if STREAM_HANDLE.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
} // already initialized
|
||||||
|
|
||||||
|
// get output stream handle to the default physical sound device
|
||||||
|
match rodio::OutputStreamBuilder::open_default_stream() {
|
||||||
|
Ok(stream_handle) => {
|
||||||
|
// create sink
|
||||||
|
let sink = Sink::connect_new(&stream_handle.mixer());
|
||||||
|
info!("Sink initialized.");
|
||||||
|
|
||||||
|
// store
|
||||||
|
// STREAM.set(_stream).unwrap();
|
||||||
|
let _ = STREAM_HANDLE.set(stream_handle);
|
||||||
|
let _ = SINK.set(sink);
|
||||||
|
|
||||||
|
// success
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Failed to initialize audio stream.\nError details: {}", msg);
|
||||||
|
|
||||||
|
// failed
|
||||||
|
Err(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn play_sound(filename: &PathBuf, sleep: bool) {
|
||||||
|
// Load a sound from a file, using a path relative to Cargo.toml
|
||||||
|
// let filepath = format!("{PUBLIC_PATH}/sound/{filename}.wav");
|
||||||
|
let file = BufReader::new(File::open(&filename).unwrap());
|
||||||
|
|
||||||
|
// Decode that sound file into a source
|
||||||
|
let source = Decoder::new(file).unwrap();
|
||||||
|
|
||||||
|
// Play the sound directly on the device
|
||||||
|
// STREAM_HANDLE.get().unwrap().play_raw(source.convert_samples());
|
||||||
|
SINK.get().unwrap().append(source);
|
||||||
|
|
||||||
|
if sleep {
|
||||||
|
// The sound plays in a separate thread. This call will block the current thread until the sink
|
||||||
|
// has finished playing all its queued sounds.
|
||||||
|
SINK.get().unwrap().sleep_until_end();
|
||||||
|
}
|
||||||
|
}
|
||||||
41
crates/jarvis-core/src/audio_buffer.rs
Normal file
41
crates/jarvis-core/src/audio_buffer.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
|
||||||
|
pub struct AudioRingBuffer {
|
||||||
|
buffer: VecDeque<Vec<i16>>,
|
||||||
|
max_frames: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AudioRingBuffer {
|
||||||
|
// Create buffer that holds `seconds` worth of audio at given frame_size and sample_rate
|
||||||
|
pub fn new(seconds: f32, frame_size: usize, sample_rate: usize) -> Self {
|
||||||
|
let frames_per_second = sample_rate / frame_size;
|
||||||
|
let max_frames = (frames_per_second as f32 * seconds) as usize;
|
||||||
|
|
||||||
|
Self {
|
||||||
|
buffer: VecDeque::with_capacity(max_frames),
|
||||||
|
max_frames,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Push a frame, dropping oldest if full
|
||||||
|
pub fn push(&mut self, frame: &[i16]) {
|
||||||
|
if self.buffer.len() >= self.max_frames {
|
||||||
|
self.buffer.pop_front();
|
||||||
|
}
|
||||||
|
self.buffer.push_back(frame.to_vec());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drain all buffered frames into a single vec
|
||||||
|
pub fn drain_all(&mut self) -> Vec<Vec<i16>> {
|
||||||
|
self.buffer.drain(..).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get frame count
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.buffer.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn clear(&mut self) {
|
||||||
|
self.buffer.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
117
crates/jarvis-core/src/audio_processing.rs
Normal file
117
crates/jarvis-core/src/audio_processing.rs
Normal file
|
|
@ -0,0 +1,117 @@
|
||||||
|
pub mod noise_suppression;
|
||||||
|
pub mod vad;
|
||||||
|
pub mod gain_normalizer;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
|
use crate::config::structs::NoiseSuppressionBackend;
|
||||||
|
use crate::DB;
|
||||||
|
|
||||||
|
static PROCESSOR: OnceCell<Mutex<AudioProcessor>> = OnceCell::new();
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ProcessedAudio {
|
||||||
|
pub samples: Vec<i16>,
|
||||||
|
pub is_voice: bool,
|
||||||
|
pub vad_confidence: f32,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct AudioProcessor {
|
||||||
|
has_gain: bool,
|
||||||
|
has_ns: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AudioProcessor {
|
||||||
|
fn new(ns: NoiseSuppressionBackend, gain: bool) -> Self {
|
||||||
|
noise_suppression::init(ns);
|
||||||
|
vad::init();
|
||||||
|
if gain {
|
||||||
|
gain_normalizer::init();
|
||||||
|
}
|
||||||
|
|
||||||
|
Self {
|
||||||
|
has_gain: gain,
|
||||||
|
has_ns: !matches!(ns, NoiseSuppressionBackend::None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process(&mut self, input: &[i16]) -> ProcessedAudio {
|
||||||
|
let gained: Vec<i16>;
|
||||||
|
let after_gain: &[i16] = if self.has_gain {
|
||||||
|
gained = gain_normalizer::normalize(input);
|
||||||
|
&gained
|
||||||
|
} else {
|
||||||
|
input
|
||||||
|
};
|
||||||
|
|
||||||
|
let suppressed: Vec<i16>;
|
||||||
|
let after_ns: &[i16] = if self.has_ns {
|
||||||
|
suppressed = noise_suppression::process(after_gain);
|
||||||
|
&suppressed
|
||||||
|
} else {
|
||||||
|
after_gain
|
||||||
|
};
|
||||||
|
|
||||||
|
let (is_voice, confidence) = vad::detect(after_ns);
|
||||||
|
|
||||||
|
ProcessedAudio {
|
||||||
|
samples: after_ns.to_vec(),
|
||||||
|
is_voice,
|
||||||
|
vad_confidence: confidence,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset(&mut self) {
|
||||||
|
noise_suppression::reset();
|
||||||
|
vad::reset();
|
||||||
|
gain_normalizer::reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), String> {
|
||||||
|
if PROCESSOR.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let (ns, gain) = get_settings();
|
||||||
|
info!("Initializing audio processing: NS={:?}, Gain={}", ns, gain);
|
||||||
|
|
||||||
|
let processor = AudioProcessor::new(ns, gain);
|
||||||
|
PROCESSOR
|
||||||
|
.set(Mutex::new(processor))
|
||||||
|
.map_err(|_| "Audio processor already initialized".to_string())?;
|
||||||
|
|
||||||
|
info!("Audio processing initialized.");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn process(input: &[i16]) -> ProcessedAudio {
|
||||||
|
match PROCESSOR.get() {
|
||||||
|
Some(p) => p.lock().process(input),
|
||||||
|
None => ProcessedAudio {
|
||||||
|
samples: input.to_vec(),
|
||||||
|
is_voice: true,
|
||||||
|
vad_confidence: 1.0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset() {
|
||||||
|
if let Some(p) = PROCESSOR.get() {
|
||||||
|
p.lock().reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_settings() -> (NoiseSuppressionBackend, bool) {
|
||||||
|
match DB.get() {
|
||||||
|
Some(db) => {
|
||||||
|
let settings = db.read();
|
||||||
|
(settings.noise_suppression, settings.gain_normalizer)
|
||||||
|
}
|
||||||
|
None => (
|
||||||
|
crate::config::DEFAULT_NOISE_SUPPRESSION,
|
||||||
|
crate::config::DEFAULT_GAIN_NORMALIZER,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
28
crates/jarvis-core/src/audio_processing/gain_normalizer.rs
Normal file
28
crates/jarvis-core/src/audio_processing/gain_normalizer.rs
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
mod simple;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
|
static NORMALIZER: OnceCell<Mutex<simple::GainNormalizer>> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() {
|
||||||
|
if NORMALIZER.get().is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
NORMALIZER.set(Mutex::new(simple::GainNormalizer::new())).ok();
|
||||||
|
info!("Gain normalizer: enabled");
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn normalize(input: &[i16]) -> Vec<i16> {
|
||||||
|
match NORMALIZER.get() {
|
||||||
|
Some(n) => n.lock().normalize(input),
|
||||||
|
None => input.to_vec(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset() {
|
||||||
|
if let Some(n) = NORMALIZER.get() {
|
||||||
|
n.lock().reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
use crate::config;
|
||||||
|
|
||||||
|
pub struct GainNormalizer {
|
||||||
|
current_gain: f32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GainNormalizer {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self { current_gain: 1.0 }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn normalize(&mut self, input: &[i16]) -> Vec<i16> {
|
||||||
|
let rms = self.calculate_rms(input);
|
||||||
|
|
||||||
|
if rms < 1.0 {
|
||||||
|
return input.to_vec();
|
||||||
|
}
|
||||||
|
|
||||||
|
let target_gain = config::GAIN_TARGET_RMS / rms;
|
||||||
|
let clamped_gain = target_gain.clamp(config::GAIN_MIN, config::GAIN_MAX);
|
||||||
|
|
||||||
|
self.current_gain = self.current_gain * 0.9 + clamped_gain * 0.1;
|
||||||
|
|
||||||
|
input.iter()
|
||||||
|
.map(|&s| {
|
||||||
|
let amplified = (s as f32) * self.current_gain;
|
||||||
|
amplified.clamp(i16::MIN as f32, i16::MAX as f32) as i16
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset(&mut self) {
|
||||||
|
self.current_gain = 1.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn calculate_rms(&self, samples: &[i16]) -> f32 {
|
||||||
|
if samples.is_empty() {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let sum: f64 = samples.iter()
|
||||||
|
.map(|&s| (s as f64).powi(2))
|
||||||
|
.sum();
|
||||||
|
|
||||||
|
(sum / samples.len() as f64).sqrt() as f32
|
||||||
|
}
|
||||||
|
}
|
||||||
65
crates/jarvis-core/src/audio_processing/noise_suppression.rs
Normal file
65
crates/jarvis-core/src/audio_processing/noise_suppression.rs
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
mod none;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
|
use crate::config::structs::NoiseSuppressionBackend;
|
||||||
|
|
||||||
|
static BACKEND: OnceCell<NoiseSuppressionBackend> = OnceCell::new();
|
||||||
|
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
static NNNOISELESS_STATE: OnceCell<Mutex<crate::models::nnnoiseless::NnnoiselessNS>> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init(backend: NoiseSuppressionBackend) {
|
||||||
|
if BACKEND.get().is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// fallback if nnnoiseless not compiled in
|
||||||
|
#[cfg(not(feature = "nnnoiseless"))]
|
||||||
|
if matches!(backend, NoiseSuppressionBackend::Nnnoiseless) {
|
||||||
|
warn!("Nnnoiseless not compiled in, falling back to None");
|
||||||
|
backend = NoiseSuppressionBackend::None;
|
||||||
|
}
|
||||||
|
|
||||||
|
BACKEND.set(backend).ok();
|
||||||
|
|
||||||
|
match backend {
|
||||||
|
NoiseSuppressionBackend::None => {
|
||||||
|
info!("Noise suppression: disabled");
|
||||||
|
}
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
NoiseSuppressionBackend::Nnnoiseless => {
|
||||||
|
NNNOISELESS_STATE.set(Mutex::new(crate::models::nnnoiseless::NnnoiselessNS::new())).ok();
|
||||||
|
info!("Noise suppression: Nnnoiseless");
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "nnnoiseless"))]
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn process(input: &[i16]) -> Vec<i16> {
|
||||||
|
match BACKEND.get() {
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
Some(NoiseSuppressionBackend::Nnnoiseless) => {
|
||||||
|
if let Some(state) = NNNOISELESS_STATE.get() {
|
||||||
|
state.lock().process(input)
|
||||||
|
} else {
|
||||||
|
none::process(input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => none::process(input),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset() {
|
||||||
|
match BACKEND.get() {
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
Some(NoiseSuppressionBackend::Nnnoiseless) => {
|
||||||
|
if let Some(state) = NNNOISELESS_STATE.get() {
|
||||||
|
state.lock().reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
// return unprocessed input
|
||||||
|
pub fn process(input: &[i16]) -> Vec<i16> {
|
||||||
|
input.to_vec()
|
||||||
|
}
|
||||||
75
crates/jarvis-core/src/audio_processing/vad.rs
Normal file
75
crates/jarvis-core/src/audio_processing/vad.rs
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
mod none;
|
||||||
|
mod energy;
|
||||||
|
pub mod listen_window;
|
||||||
|
#[cfg(feature = "webrtc-vad")]
|
||||||
|
pub mod webrtc;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
|
use crate::DB;
|
||||||
|
|
||||||
|
static BACKEND: OnceCell<String> = OnceCell::new();
|
||||||
|
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
static NNNOISELESS_STATE: OnceCell<Mutex<crate::models::nnnoiseless::NnnoiselessVAD>> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() {
|
||||||
|
if BACKEND.get().is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let backend = DB.get()
|
||||||
|
.map(|db| db.read().vad_backend.clone())
|
||||||
|
.unwrap_or_else(|| "energy".to_string());
|
||||||
|
|
||||||
|
BACKEND.set(backend.clone()).ok();
|
||||||
|
|
||||||
|
match backend.as_str() {
|
||||||
|
"none" => {
|
||||||
|
info!("VAD: disabled");
|
||||||
|
}
|
||||||
|
"energy" => {
|
||||||
|
info!("VAD: Energy-based");
|
||||||
|
}
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
"nnnoiseless" => {
|
||||||
|
NNNOISELESS_STATE.set(Mutex::new(crate::models::nnnoiseless::NnnoiselessVAD::new())).ok();
|
||||||
|
info!("VAD: Nnnoiseless");
|
||||||
|
}
|
||||||
|
other => {
|
||||||
|
warn!("Unknown VAD backend '{}', falling back to energy", other);
|
||||||
|
// overwrite with energy
|
||||||
|
// (BACKEND already set, so energy::detect will be used via fallthrough)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// returns (is_voice, confidence)
|
||||||
|
pub fn detect(input: &[i16]) -> (bool, f32) {
|
||||||
|
match BACKEND.get().map(|s| s.as_str()) {
|
||||||
|
Some("none") | None => none::detect(input),
|
||||||
|
Some("energy") => energy::detect(input),
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
Some("nnnoiseless") => {
|
||||||
|
if let Some(state) = NNNOISELESS_STATE.get() {
|
||||||
|
state.lock().detect(input)
|
||||||
|
} else {
|
||||||
|
energy::detect(input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => energy::detect(input),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset() {
|
||||||
|
match BACKEND.get().map(|s| s.as_str()) {
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
Some("nnnoiseless") => {
|
||||||
|
if let Some(state) = NNNOISELESS_STATE.get() {
|
||||||
|
state.lock().reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
24
crates/jarvis-core/src/audio_processing/vad/energy.rs
Normal file
24
crates/jarvis-core/src/audio_processing/vad/energy.rs
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
use crate::config;
|
||||||
|
|
||||||
|
// Simple energy-based VAD
|
||||||
|
pub fn detect(input: &[i16]) -> (bool, f32) {
|
||||||
|
let rms = calculate_rms(input);
|
||||||
|
let is_voice = rms > config::VAD_ENERGY_THRESHOLD;
|
||||||
|
|
||||||
|
// normalize confidence to 0-1 range (rough approximation)
|
||||||
|
let confidence = (rms / (config::VAD_ENERGY_THRESHOLD * 2.0)).min(1.0);
|
||||||
|
|
||||||
|
(is_voice, confidence)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn calculate_rms(samples: &[i16]) -> f32 {
|
||||||
|
if samples.is_empty() {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let sum: f64 = samples.iter()
|
||||||
|
.map(|&s| (s as f64).powi(2))
|
||||||
|
.sum();
|
||||||
|
|
||||||
|
(sum / samples.len() as f64).sqrt() as f32
|
||||||
|
}
|
||||||
135
crates/jarvis-core/src/audio_processing/vad/listen_window.rs
Normal file
135
crates/jarvis-core/src/audio_processing/vad/listen_window.rs
Normal file
|
|
@ -0,0 +1,135 @@
|
||||||
|
use crate::config;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum WindowDecision {
|
||||||
|
KeepListening,
|
||||||
|
Close,
|
||||||
|
HardCap,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ListenWindow {
|
||||||
|
frame_ms: u32,
|
||||||
|
elapsed_ms: u32,
|
||||||
|
speech_ms: u32,
|
||||||
|
silence_ms: u32,
|
||||||
|
end_silence_ms: u32,
|
||||||
|
min_speech_ms: u32,
|
||||||
|
min_listen_ms: u32,
|
||||||
|
max_listen_ms: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ListenWindow {
|
||||||
|
pub fn new(frame_ms: u32) -> Self {
|
||||||
|
Self::with_params(
|
||||||
|
frame_ms,
|
||||||
|
config::VAD_COMMAND_END_SILENCE_MS,
|
||||||
|
config::VAD_COMMAND_MIN_SPEECH_MS,
|
||||||
|
config::VAD_COMMAND_MIN_LISTEN_MS,
|
||||||
|
config::VAD_COMMAND_MAX_LISTEN_MS,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_params(
|
||||||
|
frame_ms: u32,
|
||||||
|
end_silence_ms: u32,
|
||||||
|
min_speech_ms: u32,
|
||||||
|
min_listen_ms: u32,
|
||||||
|
max_listen_ms: u32,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
frame_ms,
|
||||||
|
elapsed_ms: 0,
|
||||||
|
speech_ms: 0,
|
||||||
|
silence_ms: 0,
|
||||||
|
end_silence_ms,
|
||||||
|
min_speech_ms,
|
||||||
|
min_listen_ms,
|
||||||
|
max_listen_ms,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn push(&mut self, is_speech: bool) -> WindowDecision {
|
||||||
|
self.elapsed_ms = self.elapsed_ms.saturating_add(self.frame_ms);
|
||||||
|
|
||||||
|
if is_speech {
|
||||||
|
self.speech_ms = self.speech_ms.saturating_add(self.frame_ms);
|
||||||
|
self.silence_ms = 0;
|
||||||
|
} else {
|
||||||
|
self.silence_ms = self.silence_ms.saturating_add(self.frame_ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.elapsed_ms >= self.max_listen_ms {
|
||||||
|
return WindowDecision::HardCap;
|
||||||
|
}
|
||||||
|
if self.elapsed_ms < self.min_listen_ms {
|
||||||
|
return WindowDecision::KeepListening;
|
||||||
|
}
|
||||||
|
if self.speech_ms >= self.min_speech_ms && self.silence_ms >= self.end_silence_ms {
|
||||||
|
return WindowDecision::Close;
|
||||||
|
}
|
||||||
|
WindowDecision::KeepListening
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn elapsed_ms(&self) -> u32 { self.elapsed_ms }
|
||||||
|
pub fn speech_ms(&self) -> u32 { self.speech_ms }
|
||||||
|
pub fn silence_ms(&self) -> u32 { self.silence_ms }
|
||||||
|
|
||||||
|
pub fn had_speech(&self) -> bool { self.speech_ms > 0 }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn run(verdicts: &[bool]) -> (WindowDecision, usize) {
|
||||||
|
let mut w = ListenWindow::with_params(30, 1200, 500, 1000, 15000);
|
||||||
|
for (i, &v) in verdicts.iter().enumerate() {
|
||||||
|
let d = w.push(v);
|
||||||
|
if d != WindowDecision::KeepListening {
|
||||||
|
return (d, i + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(WindowDecision::KeepListening, verdicts.len())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn closes_after_speech_then_silence() {
|
||||||
|
let speech: Vec<bool> = std::iter::repeat(true).take(40).collect();
|
||||||
|
let silence: Vec<bool> = std::iter::repeat(false).take(60).collect();
|
||||||
|
let mut v = speech;
|
||||||
|
v.extend(silence);
|
||||||
|
|
||||||
|
let (decision, frame_idx) = run(&v);
|
||||||
|
assert_eq!(decision, WindowDecision::Close);
|
||||||
|
let elapsed_ms = (frame_idx as u32) * 30;
|
||||||
|
let speech_ms = 40 * 30;
|
||||||
|
assert!(elapsed_ms >= speech_ms + 1200, "closed at {} ms, expected >= {}", elapsed_ms, speech_ms + 1200);
|
||||||
|
assert!(elapsed_ms <= speech_ms + 1200 + 30, "closed too late at {} ms", elapsed_ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn does_not_close_before_min_listen_even_with_long_silence() {
|
||||||
|
let v: Vec<bool> = std::iter::repeat(false).take(100).collect();
|
||||||
|
let mut w = ListenWindow::with_params(30, 1200, 500, 1000, 15000);
|
||||||
|
for &x in v.iter().take(33) {
|
||||||
|
assert_eq!(w.push(x), WindowDecision::KeepListening);
|
||||||
|
}
|
||||||
|
assert!(w.elapsed_ms() <= 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hard_cap_fires_at_max_listen() {
|
||||||
|
let v: Vec<bool> = std::iter::repeat(true).take(1000).collect();
|
||||||
|
let (decision, frame_idx) = run(&v);
|
||||||
|
assert_eq!(decision, WindowDecision::HardCap);
|
||||||
|
assert_eq!((frame_idx as u32) * 30, 15000);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn silence_only_does_not_close_keeps_waiting_until_hard_cap() {
|
||||||
|
let v: Vec<bool> = std::iter::repeat(false).take(600).collect();
|
||||||
|
let (decision, _) = run(&v);
|
||||||
|
assert_eq!(decision, WindowDecision::HardCap);
|
||||||
|
}
|
||||||
|
}
|
||||||
4
crates/jarvis-core/src/audio_processing/vad/none.rs
Normal file
4
crates/jarvis-core/src/audio_processing/vad/none.rs
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
// Always returns voice detected (no vad)
|
||||||
|
pub fn detect(_input: &[i16]) -> (bool, f32) {
|
||||||
|
(true, 1.0)
|
||||||
|
}
|
||||||
49
crates/jarvis-core/src/audio_processing/vad/webrtc.rs
Normal file
49
crates/jarvis-core/src/audio_processing/vad/webrtc.rs
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
use webrtc_vad::{SampleRate, Vad, VadMode};
|
||||||
|
|
||||||
|
use crate::config;
|
||||||
|
|
||||||
|
pub const FRAME_SAMPLES: usize = 480;
|
||||||
|
pub const FRAME_MS: u32 = 30;
|
||||||
|
|
||||||
|
pub struct WebRtcVad {
|
||||||
|
inner: Vad,
|
||||||
|
buf: Vec<i16>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebRtcVad {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::with_aggressiveness(config::VAD_AGGRESSIVENESS)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_aggressiveness(level: u8) -> Self {
|
||||||
|
let mode = match level {
|
||||||
|
0 => VadMode::Quality,
|
||||||
|
1 => VadMode::LowBitrate,
|
||||||
|
2 => VadMode::Aggressive,
|
||||||
|
_ => VadMode::VeryAggressive,
|
||||||
|
};
|
||||||
|
Self {
|
||||||
|
inner: Vad::new_with_rate_and_mode(SampleRate::Rate16kHz, mode),
|
||||||
|
buf: Vec::with_capacity(FRAME_SAMPLES * 2),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn push_samples(&mut self, samples: &[i16]) -> Vec<bool> {
|
||||||
|
self.buf.extend_from_slice(samples);
|
||||||
|
let mut out = Vec::new();
|
||||||
|
while self.buf.len() >= FRAME_SAMPLES {
|
||||||
|
let frame: Vec<i16> = self.buf.drain(..FRAME_SAMPLES).collect();
|
||||||
|
let is_speech = self.inner.is_voice_segment(&frame).unwrap_or(false);
|
||||||
|
out.push(is_speech);
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset(&mut self) {
|
||||||
|
self.buf.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for WebRtcVad {
|
||||||
|
fn default() -> Self { Self::new() }
|
||||||
|
}
|
||||||
319
crates/jarvis-core/src/commands.rs
Normal file
319
crates/jarvis-core/src/commands.rs
Normal file
|
|
@ -0,0 +1,319 @@
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::fs;
|
||||||
|
use std::time::Duration;
|
||||||
|
use std::process::{Child, Command};
|
||||||
|
|
||||||
|
use seqdiff::ratio;
|
||||||
|
|
||||||
|
mod structs;
|
||||||
|
pub use structs::*;
|
||||||
|
|
||||||
|
use crate::{config, i18n, APP_DIR};
|
||||||
|
|
||||||
|
#[cfg(feature = "lua")]
|
||||||
|
use crate::lua::{self, SandboxLevel, CommandContext};
|
||||||
|
|
||||||
|
pub fn parse_commands() -> Result<Vec<JCommandsList>, String> {
|
||||||
|
let mut commands: Vec<JCommandsList> = Vec::new();
|
||||||
|
|
||||||
|
let commands_path = APP_DIR.join(config::COMMANDS_PATH);
|
||||||
|
let cmd_dirs = fs::read_dir(&commands_path)
|
||||||
|
.map_err(|e| format!("Error reading commands directory {:?}: {}", commands_path, e))?;
|
||||||
|
|
||||||
|
for entry in cmd_dirs.flatten() {
|
||||||
|
let cmd_path = entry.path();
|
||||||
|
let toml_file = cmd_path.join("command.toml");
|
||||||
|
|
||||||
|
if !toml_file.exists() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let content = match fs::read_to_string(&toml_file) {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Failed to read {}: {}", toml_file.display(), e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let file: JCommandsList = match toml::from_str(&content) {
|
||||||
|
Ok(f) => f,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Failed to parse {}: {}", toml_file.display(), e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
commands.push(JCommandsList {
|
||||||
|
path: cmd_path,
|
||||||
|
commands: file.commands,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if commands.is_empty() {
|
||||||
|
Err("No commands found".into())
|
||||||
|
} else {
|
||||||
|
info!("Loaded {} command pack(s)", commands.len());
|
||||||
|
Ok(commands)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub fn commands_hash(commands: &[JCommandsList]) -> String {
|
||||||
|
use sha2::{Sha256, Digest};
|
||||||
|
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
|
||||||
|
let lang = i18n::get_language();
|
||||||
|
hasher.update(lang.as_bytes());
|
||||||
|
hasher.update(b"|");
|
||||||
|
|
||||||
|
// collect all command ids and phrases for current language, sorted
|
||||||
|
let mut all_data: Vec<(&str, _)> = commands.iter()
|
||||||
|
.flat_map(|ac| ac.commands.iter().map(|c| (c.id.as_str(), c.get_phrases(&lang))))
|
||||||
|
.collect();
|
||||||
|
all_data.sort_by_key(|(id, _)| *id);
|
||||||
|
|
||||||
|
for (id, phrases) in all_data {
|
||||||
|
hasher.update(id.as_bytes());
|
||||||
|
for phrase in phrases.iter() {
|
||||||
|
hasher.update(phrase.as_bytes());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
format!("{:x}", hasher.finalize())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub fn fetch_command<'a>(
|
||||||
|
phrase: &str,
|
||||||
|
commands: &'a [JCommandsList],
|
||||||
|
) -> Option<(&'a PathBuf, &'a JCommand)> {
|
||||||
|
let lang = i18n::get_language();
|
||||||
|
|
||||||
|
let phrase = phrase.trim().to_lowercase();
|
||||||
|
if phrase.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let phrase_chars: Vec<char> = phrase.chars().collect();
|
||||||
|
let phrase_words: Vec<&str> = phrase.split_whitespace().collect();
|
||||||
|
|
||||||
|
let mut result: Option<(&PathBuf, &JCommand)> = None;
|
||||||
|
let mut best_score = config::CMD_RATIO_THRESHOLD;
|
||||||
|
|
||||||
|
for cmd_list in commands {
|
||||||
|
for cmd in &cmd_list.commands {
|
||||||
|
let cmd_phrases = cmd.get_phrases(&lang);
|
||||||
|
|
||||||
|
for cmd_phrase in cmd_phrases.iter() {
|
||||||
|
let cmd_phrase_lower = cmd_phrase.trim().to_lowercase();
|
||||||
|
let cmd_phrase_chars: Vec<char> = cmd_phrase_lower.chars().collect();
|
||||||
|
|
||||||
|
// character-level similarity
|
||||||
|
let char_ratio = ratio(&phrase_chars, &cmd_phrase_chars);
|
||||||
|
|
||||||
|
// word-level similarity
|
||||||
|
let cmd_words: Vec<&str> = cmd_phrase_lower.split_whitespace().collect();
|
||||||
|
let word_score = word_overlap_score(&phrase_words, &cmd_words);
|
||||||
|
|
||||||
|
// combined score
|
||||||
|
let score = (char_ratio * 0.6) + (word_score * 0.4);
|
||||||
|
|
||||||
|
// early exit on perfect match
|
||||||
|
if score >= 99.0 {
|
||||||
|
debug!("Perfect match: '{}' -> '{}'", phrase, cmd_phrase_lower);
|
||||||
|
return Some((&cmd_list.path, cmd));
|
||||||
|
}
|
||||||
|
|
||||||
|
if score > best_score {
|
||||||
|
best_score = score;
|
||||||
|
result = Some((&cmd_list.path, cmd));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some((_, cmd)) = result {
|
||||||
|
info!("Fuzzy match: '{}' -> cmd '{}' (score: {:.1}%)", phrase, cmd.id, best_score);
|
||||||
|
} else {
|
||||||
|
debug!("No match for '{}' (best: {:.1}%)", phrase, best_score);
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
fn word_overlap_score(input_words: &[&str], cmd_words: &[&str]) -> f64 {
|
||||||
|
if input_words.is_empty() || cmd_words.is_empty() {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut matched = 0.0;
|
||||||
|
|
||||||
|
// pre-compute cmd word chars to avoid repeated allocations
|
||||||
|
let cmd_word_chars: Vec<Vec<char>> = cmd_words
|
||||||
|
.iter()
|
||||||
|
.map(|w| w.chars().collect())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
for input_word in input_words {
|
||||||
|
let input_chars: Vec<char> = input_word.chars().collect();
|
||||||
|
|
||||||
|
let best_word_match = cmd_word_chars
|
||||||
|
.iter()
|
||||||
|
.map(|cw| ratio(&input_chars, cw))
|
||||||
|
.fold(0.0_f64, f64::max);
|
||||||
|
|
||||||
|
if best_word_match > 70.0 {
|
||||||
|
matched += best_word_match / 100.0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let max_words = input_words.len().max(cmd_words.len()) as f64;
|
||||||
|
(matched / max_words) * 100.0
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
pub fn execute_exe(exe: &str, args: &[String]) -> std::io::Result<Child> {
|
||||||
|
Command::new(exe).args(args).spawn()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn execute_cli(cmd: &str, args: &[String]) -> std::io::Result<Child> {
|
||||||
|
debug!("Spawning: cmd /C {} {:?}", cmd, args);
|
||||||
|
|
||||||
|
if cfg!(target_os = "windows") {
|
||||||
|
Command::new("cmd").arg("/C").arg(cmd).args(args).spawn()
|
||||||
|
} else {
|
||||||
|
Command::new("sh").arg("-c").arg(cmd).args(args).spawn()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn execute_command(cmd_path: &PathBuf, cmd_config: &JCommand, phrase: Option<&str>, slots: Option<&HashMap<String, SlotValue>>) -> Result<bool, String> {
|
||||||
|
// execute command by the type
|
||||||
|
match cmd_config.cmd_type.as_str() {
|
||||||
|
|
||||||
|
// BRUH
|
||||||
|
"voice" => Ok(true),
|
||||||
|
|
||||||
|
// LUA command
|
||||||
|
#[cfg(feature = "lua")]
|
||||||
|
"lua" => {
|
||||||
|
execute_lua_command(cmd_path, cmd_config, phrase, slots)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AutoHotkey command
|
||||||
|
// @TODO: Consider adding ahk source files execution?
|
||||||
|
"ahk" => {
|
||||||
|
let exe_path_absolute = Path::new(&cmd_config.exe_path);
|
||||||
|
let exe_path_local = cmd_path.join(&cmd_config.exe_path);
|
||||||
|
|
||||||
|
let exe_path = if exe_path_absolute.exists() {
|
||||||
|
exe_path_absolute
|
||||||
|
} else {
|
||||||
|
exe_path_local.as_path()
|
||||||
|
};
|
||||||
|
|
||||||
|
execute_exe(exe_path.to_str().unwrap(), &cmd_config.exe_args)
|
||||||
|
.map(|_| true)
|
||||||
|
.map_err(|e| format!("AHK process spawn error: {}", e))
|
||||||
|
}
|
||||||
|
|
||||||
|
// CLI command type
|
||||||
|
// @TODO: Consider security restrictions
|
||||||
|
"cli" => {
|
||||||
|
execute_cli(&cmd_config.cli_cmd, &cmd_config.cli_args)
|
||||||
|
.map(|_| true)
|
||||||
|
.map_err(|e| format!("CLI command error: {}", e))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TERMINATOR command (T1000)
|
||||||
|
"terminate" => {
|
||||||
|
std::thread::sleep(Duration::from_secs(2));
|
||||||
|
std::process::exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// STOP CHANING
|
||||||
|
"stop_chaining" => Ok(false),
|
||||||
|
|
||||||
|
// other
|
||||||
|
_ => {
|
||||||
|
error!("Command type unknown: {}", cmd_config.cmd_type);
|
||||||
|
Err(format!("Command type unknown: {}", cmd_config.cmd_type).into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// look up a command by its ID
|
||||||
|
pub fn get_command_by_id<'a>(
|
||||||
|
commands: &'a [JCommandsList],
|
||||||
|
id: &str,
|
||||||
|
) -> Option<(&'a PathBuf, &'a JCommand)> {
|
||||||
|
for cmd_list in commands {
|
||||||
|
for cmd in &cmd_list.commands {
|
||||||
|
if cmd.id == id {
|
||||||
|
return Some((&cmd_list.path, cmd));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_paths(commands: &[JCommandsList]) -> Vec<&Path> {
|
||||||
|
commands.iter().map(|x| x.path.as_path()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "lua")]
|
||||||
|
fn execute_lua_command(
|
||||||
|
cmd_path: &PathBuf,
|
||||||
|
cmd_config: &JCommand,
|
||||||
|
phrase: Option<&str>,
|
||||||
|
slots: Option<&HashMap<String, SlotValue>>
|
||||||
|
) -> Result<bool, String> {
|
||||||
|
// get script path
|
||||||
|
|
||||||
|
let script_name = if cmd_config.script.is_empty() {
|
||||||
|
"script.lua"
|
||||||
|
} else {
|
||||||
|
&cmd_config.script
|
||||||
|
};
|
||||||
|
|
||||||
|
let script_path = cmd_path.join(script_name);
|
||||||
|
|
||||||
|
if !script_path.exists() {
|
||||||
|
return Err(format!("Lua script not found: {}", script_path.display()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// parse sandbox level
|
||||||
|
let sandbox = SandboxLevel::from_str(&cmd_config.sandbox);
|
||||||
|
|
||||||
|
// create context
|
||||||
|
let context = CommandContext {
|
||||||
|
phrase: phrase.unwrap_or("").to_string(),
|
||||||
|
command_id: cmd_config.id.clone(),
|
||||||
|
command_path: cmd_path.clone(),
|
||||||
|
language: i18n::get_language(),
|
||||||
|
slots: slots.map(|s| s.clone()),
|
||||||
|
};
|
||||||
|
|
||||||
|
// get timeout
|
||||||
|
let timeout = Duration::from_millis(cmd_config.timeout);
|
||||||
|
|
||||||
|
info!("Executing Lua command: {} (sandbox: {:?}, timeout: {:?})",
|
||||||
|
cmd_config.id, sandbox, timeout);
|
||||||
|
|
||||||
|
// execute
|
||||||
|
match lua::execute(&script_path, context, sandbox, timeout) {
|
||||||
|
Ok(result) => {
|
||||||
|
info!("Lua command {} completed (chain: {})", cmd_config.id, result.chain);
|
||||||
|
Ok(result.chain)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Lua command {} failed: {}", cmd_config.id, e);
|
||||||
|
Err(e.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
179
crates/jarvis-core/src/commands/structs.rs
Normal file
179
crates/jarvis-core/src/commands/structs.rs
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
use std::{collections::HashMap, path::PathBuf, sync::Arc};
|
||||||
|
use serde::{Serialize, Deserialize};
|
||||||
|
use parking_lot::RwLock;
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug)]
|
||||||
|
pub struct JCommandsList {
|
||||||
|
#[serde(skip)]
|
||||||
|
pub path: PathBuf,
|
||||||
|
|
||||||
|
pub commands: Vec<JCommand>,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug)]
|
||||||
|
pub struct JCommand {
|
||||||
|
pub id: String,
|
||||||
|
|
||||||
|
// Available command types are: "lua", "ahk", "cli", "voice", "terminate", "stop_chaining"
|
||||||
|
#[serde(rename = "type")]
|
||||||
|
pub cmd_type: String,
|
||||||
|
|
||||||
|
#[serde(default)]
|
||||||
|
pub description: String,
|
||||||
|
|
||||||
|
// for "ahk" type
|
||||||
|
#[serde(default)]
|
||||||
|
pub exe_path: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub exe_args: Vec<String>,
|
||||||
|
|
||||||
|
// for "cli" type
|
||||||
|
#[serde(default)]
|
||||||
|
pub cli_cmd: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub cli_args: Vec<String>,
|
||||||
|
|
||||||
|
// #[serde(default)]
|
||||||
|
// pub sounds: Vec<String>,
|
||||||
|
|
||||||
|
// for "lua" type
|
||||||
|
#[serde(default)]
|
||||||
|
pub script: String,
|
||||||
|
|
||||||
|
// Lua sandbox level: "minimal", "standard", "full"
|
||||||
|
// basically this is an access level
|
||||||
|
#[serde(default)]
|
||||||
|
pub sandbox: String,
|
||||||
|
|
||||||
|
// Script timeout in milliseconds (default 10000 = 10s)
|
||||||
|
#[serde(default)]
|
||||||
|
pub timeout: u64,
|
||||||
|
|
||||||
|
// Multi-language sounds
|
||||||
|
#[serde(default)]
|
||||||
|
pub sounds: HashMap<String, Vec<String>>,
|
||||||
|
|
||||||
|
// Multi-language phrases
|
||||||
|
#[serde(default)]
|
||||||
|
pub phrases: HashMap<String, Vec<String>>,
|
||||||
|
|
||||||
|
// Slot definitions: slot_name -> how to extract it
|
||||||
|
#[serde(default)]
|
||||||
|
pub slots: HashMap<String, SlotDefinition>,
|
||||||
|
|
||||||
|
// CACHE
|
||||||
|
#[serde(skip, default)]
|
||||||
|
sounds_cache: RwLock<HashMap<String, Arc<Vec<String>>>>,
|
||||||
|
|
||||||
|
#[serde(skip, default)]
|
||||||
|
phrases_cache: RwLock<HashMap<String, Arc<Vec<String>>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// custom Clone
|
||||||
|
impl Clone for JCommand {
|
||||||
|
fn clone(&self) -> Self {
|
||||||
|
Self {
|
||||||
|
id: self.id.clone(),
|
||||||
|
|
||||||
|
cmd_type: self.cmd_type.clone(),
|
||||||
|
description: self.description.clone(),
|
||||||
|
|
||||||
|
exe_path: self.exe_path.clone(),
|
||||||
|
exe_args: self.exe_args.clone(),
|
||||||
|
|
||||||
|
cli_cmd: self.cli_cmd.clone(),
|
||||||
|
cli_args: self.cli_args.clone(),
|
||||||
|
|
||||||
|
script: self.script.clone(),
|
||||||
|
sandbox: self.sandbox.clone(),
|
||||||
|
timeout: self.timeout.clone(),
|
||||||
|
|
||||||
|
sounds: self.sounds.clone(),
|
||||||
|
phrases: self.phrases.clone(),
|
||||||
|
|
||||||
|
slots: self.slots.clone(),
|
||||||
|
|
||||||
|
// empty caches for cloned instance
|
||||||
|
sounds_cache: RwLock::new(HashMap::new()),
|
||||||
|
phrases_cache: RwLock::new(HashMap::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JCommand {
|
||||||
|
// get phrases for current language
|
||||||
|
pub fn get_phrases(&self, lang: &str) -> Arc<Vec<String>> {
|
||||||
|
if let Some(cached) = self.phrases_cache.read().get(lang) {
|
||||||
|
return Arc::clone(cached);
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = Arc::new(self.resolve_localized(&self.phrases, lang));
|
||||||
|
self.phrases_cache.write().insert(lang.to_string(), Arc::clone(&result));
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
// get all phrases (for backwards compat)
|
||||||
|
pub fn get_all_phrases(&self) -> Vec<String> {
|
||||||
|
self.phrases.values().flatten().cloned().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
// get sounds for current language
|
||||||
|
pub fn get_sounds(&self, lang: &str) -> Arc<Vec<String>> {
|
||||||
|
if let Some(cached) = self.sounds_cache.read().get(lang) {
|
||||||
|
return Arc::clone(cached);
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = Arc::new(self.resolve_localized(&self.sounds, lang));
|
||||||
|
self.sounds_cache.write().insert(lang.to_string(), Arc::clone(&result));
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
// get all sounds (for backwards compat)
|
||||||
|
pub fn get_all_sounds(&self) -> Vec<String> {
|
||||||
|
self.sounds.values().flatten().cloned().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// shared fallback
|
||||||
|
fn resolve_localized(&self, map: &HashMap<String, Vec<String>>, lang: &str) -> Vec<String> {
|
||||||
|
// exact match
|
||||||
|
if let Some(values) = map.get(lang) {
|
||||||
|
return values.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
// fallback to "en"
|
||||||
|
if lang != "en" {
|
||||||
|
if let Some(values) = map.get("en") {
|
||||||
|
return values.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fallback to first available
|
||||||
|
map.values().next().cloned().unwrap_or_default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||||
|
pub struct SlotDefinition {
|
||||||
|
// Entity label for GLiNER (e.g. "city name", "song title", "number")
|
||||||
|
// This is a free-form description - GLiNER matches it semantically
|
||||||
|
#[serde(default)]
|
||||||
|
pub entity: String,
|
||||||
|
|
||||||
|
// Optional: fallback context words for template-based extraction
|
||||||
|
// e.g. ["in", "for", "at"] for a city slot
|
||||||
|
#[serde(default)]
|
||||||
|
pub context: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extracted slot value passed to commands
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
pub enum SlotValue {
|
||||||
|
Text(String),
|
||||||
|
Number(f64),
|
||||||
|
}
|
||||||
293
crates/jarvis-core/src/config.rs
Normal file
293
crates/jarvis-core/src/config.rs
Normal file
|
|
@ -0,0 +1,293 @@
|
||||||
|
pub mod structs;
|
||||||
|
use structs::AudioType;
|
||||||
|
use structs::RecorderType;
|
||||||
|
use structs::SpeechToTextEngine;
|
||||||
|
use structs::WakeWordEngine;
|
||||||
|
|
||||||
|
use once_cell::sync::Lazy;
|
||||||
|
use std::env;
|
||||||
|
use std::fs;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use platform_dirs::AppDirs;
|
||||||
|
|
||||||
|
#[cfg(feature="jarvis_app")]
|
||||||
|
use rustpotter::{
|
||||||
|
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig,
|
||||||
|
RustpotterConfig, ScoreMode,
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::config::structs::NoiseSuppressionBackend;
|
||||||
|
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn init_dirs() -> Result<(), String> {
|
||||||
|
// infer app dirs
|
||||||
|
if APP_DIRS.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// cache_dir, config_dir, data_dir, state_dir
|
||||||
|
APP_DIRS
|
||||||
|
.set(AppDirs::new(Some(BUNDLE_IDENTIFIER), false).unwrap())
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// setup directories
|
||||||
|
let mut config_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
|
||||||
|
let mut log_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
|
||||||
|
|
||||||
|
// create dirs, if required
|
||||||
|
if !config_dir.exists() {
|
||||||
|
if fs::create_dir_all(&config_dir).is_err() {
|
||||||
|
config_dir = env::current_dir().expect("Cannot infer the config directory");
|
||||||
|
fs::create_dir_all(&config_dir)
|
||||||
|
.expect("Cannot create config directory, access denied?");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !log_dir.exists() {
|
||||||
|
if fs::create_dir_all(&log_dir).is_err() {
|
||||||
|
log_dir = env::current_dir().expect("Cannot infer the log directory");
|
||||||
|
fs::create_dir_all(&log_dir).expect("Cannot create log directory, access denied?");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// store inferred paths
|
||||||
|
APP_CONFIG_DIR.set(config_dir).unwrap();
|
||||||
|
APP_LOG_DIR.set(log_dir).unwrap();
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
Defaults.
|
||||||
|
*/
|
||||||
|
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
|
||||||
|
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
|
||||||
|
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Vosk;
|
||||||
|
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
|
||||||
|
|
||||||
|
// backend defaults (string IDs)
|
||||||
|
pub const DEFAULT_INTENT_BACKEND: &str = "intent-classifier";
|
||||||
|
pub const DEFAULT_SLOTS_BACKEND: &str = "none";
|
||||||
|
pub const DEFAULT_VAD_BACKEND: &str = "energy";
|
||||||
|
|
||||||
|
pub const DEFAULT_VOICE: &str = "jarvis-remaster";
|
||||||
|
pub const SOUND_PATH: &str = "resources/sound"; // extended from SOUND_DIR (resources/sound)
|
||||||
|
pub const VOICES_PATH: &str = "voices"; // extended from SOUND_PATH (resources/sound)
|
||||||
|
|
||||||
|
pub const BUNDLE_IDENTIFIER: &str = "com.priler.jarvis";
|
||||||
|
pub const DB_FILE_NAME: &str = "app.db";
|
||||||
|
pub const LOG_FILE_NAME: &str = "log.txt";
|
||||||
|
pub const APP_VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION");
|
||||||
|
pub const AUTHOR_NAME: Option<&str> = option_env!("CARGO_PKG_AUTHORS");
|
||||||
|
pub const REPOSITORY_LINK: Option<&str> = option_env!("CARGO_PKG_REPOSITORY");
|
||||||
|
pub const TG_OFFICIAL_LINK: Option<&str> = Some("https://t.me/howdyho_official");
|
||||||
|
pub const FEEDBACK_LINK: Option<&str> = Some("https://t.me/jarvis_feedback_bot");
|
||||||
|
pub const SUPPORT_BOOSTY_LINK: Option<&str> = Some("https://boosty.to/howdyho");
|
||||||
|
pub const SUPPORT_PATREON_LINK: Option<&str> = Some("https://www.patreon.com/c/priler");
|
||||||
|
|
||||||
|
/*
|
||||||
|
Tray.
|
||||||
|
*/
|
||||||
|
pub const TRAY_ICON: &str = "32x32.png";
|
||||||
|
pub const TRAY_TOOLTIP: &str = "Jarvis Voice Assistant";
|
||||||
|
|
||||||
|
// RUSPOTTER
|
||||||
|
pub const RUSPOTTER_MIN_SCORE: f32 = 0.62;
|
||||||
|
|
||||||
|
#[cfg(feature="jarvis_app")]
|
||||||
|
pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
|
||||||
|
RustpotterConfig {
|
||||||
|
fmt: AudioFmt::default(),
|
||||||
|
detector: DetectorConfig {
|
||||||
|
avg_threshold: 0.,
|
||||||
|
threshold: 0.5,
|
||||||
|
min_scores: 15,
|
||||||
|
score_ref: 0.22,
|
||||||
|
band_size: 5,
|
||||||
|
vad_mode: None,
|
||||||
|
score_mode: ScoreMode::Max,
|
||||||
|
eager: false,
|
||||||
|
// comparator_band_size: 5,
|
||||||
|
// comparator_ref: 0.22
|
||||||
|
},
|
||||||
|
filters: FiltersConfig {
|
||||||
|
gain_normalizer: GainNormalizationConfig {
|
||||||
|
// enabled: true,
|
||||||
|
// gain_ref: None,
|
||||||
|
// min_gain: 0.7,
|
||||||
|
// max_gain: 1.0,
|
||||||
|
enabled: false, // disable, now we have separate gain normalizer implementation
|
||||||
|
gain_ref: None,
|
||||||
|
min_gain: 0.7,
|
||||||
|
max_gain: 1.0,
|
||||||
|
},
|
||||||
|
band_pass: BandPassConfig {
|
||||||
|
enabled: true,
|
||||||
|
low_cutoff: 80.,
|
||||||
|
high_cutoff: 400.,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// PICOVOICE
|
||||||
|
pub const COMMANDS_PATH: &str = "resources/commands/";
|
||||||
|
pub const KEYWORDS_PATH: &str = "resources/picovoice/keywords/";
|
||||||
|
pub const DEFAULT_KEYWORD: &str = "jarvis_windows.ppn";
|
||||||
|
pub const DEFAULT_SENSITIVITY: f32 = 1.0;
|
||||||
|
|
||||||
|
// VOSK
|
||||||
|
// pub const VOSK_MODEL_PATH: &str = const_concat!(PUBLIC_PATH, "/vosk/model_small");
|
||||||
|
pub const VOSK_MODELS_PATH: &str = "resources/vosk";
|
||||||
|
pub const VOSK_MODEL_PATH: &str = "resources/vosk/model_small";
|
||||||
|
pub const VOSK_FETCH_PHRASE: &str = "джарвис";
|
||||||
|
pub const VOSK_MIN_RATIO: f64 = 70.0;
|
||||||
|
|
||||||
|
// 0.7 lenient, expect false positives
|
||||||
|
// 0.8 balanced
|
||||||
|
// 0.9 strict
|
||||||
|
// etc
|
||||||
|
pub const VOSK_WAKE_CONFIDENCE: f32 = 0.9;
|
||||||
|
|
||||||
|
pub const VOSK_SPEECH_RECOGNIZER_MAX_ALTERNATIVES: u16 = 3;
|
||||||
|
pub const VOSK_SPEECH_RECOGNIZER_WORDS: bool = false;
|
||||||
|
pub const VOSK_SPEECH_PARTIAL_WORDS: bool = false;
|
||||||
|
|
||||||
|
// IRE (intents recognition)
|
||||||
|
pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.75;
|
||||||
|
|
||||||
|
|
||||||
|
// embedding classifier
|
||||||
|
pub const EMBEDDING_MIN_CONFIDENCE: f64 = 0.70;
|
||||||
|
|
||||||
|
// AUDIO PROCESSING DEFAULTS
|
||||||
|
pub const DEFAULT_NOISE_SUPPRESSION: NoiseSuppressionBackend = NoiseSuppressionBackend::None;
|
||||||
|
pub const DEFAULT_GAIN_NORMALIZER: bool = false;
|
||||||
|
|
||||||
|
// VAD settings
|
||||||
|
pub const VAD_ENERGY_THRESHOLD: f32 = 100.0; // RMS threshold for energy-based VAD
|
||||||
|
pub const VAD_NNNOISELESS_THRESHOLD: f32 = 0.8; // probability threshold for nnnoiseless
|
||||||
|
pub const VAD_SILENCE_FRAMES: u32 = 15; // frames of silence before speech end (~480ms)
|
||||||
|
|
||||||
|
// post-wake command listening window (паритет с Python v0.2.0, webrtcvad)
|
||||||
|
pub const VAD_AGGRESSIVENESS: u8 = 2;
|
||||||
|
pub const VAD_COMMAND_END_SILENCE_MS: u32 = 1200;
|
||||||
|
pub const VAD_COMMAND_MIN_SPEECH_MS: u32 = 500;
|
||||||
|
pub const VAD_COMMAND_MIN_LISTEN_MS: u32 = 1000;
|
||||||
|
pub const VAD_COMMAND_MAX_LISTEN_MS: u32 = 15000;
|
||||||
|
|
||||||
|
// gain normalizer settings
|
||||||
|
pub const GAIN_TARGET_RMS: f32 = 3000.0; // target RMS level
|
||||||
|
pub const GAIN_MIN: f32 = 0.5; // minimum gain multiplier
|
||||||
|
pub const GAIN_MAX: f32 = 3.0; // maximum gain multiplier
|
||||||
|
|
||||||
|
// nnnoiseless frame size (fixed by library)
|
||||||
|
pub const NNNOISELESS_FRAME_SIZE: usize = 480;
|
||||||
|
|
||||||
|
// LUA
|
||||||
|
pub const DEFAULT_LUA_SANDBOX: &str = "standard";
|
||||||
|
pub const DEFAULT_LUA_TIMEOUT: u64 = 10000; // ms
|
||||||
|
|
||||||
|
// ETC
|
||||||
|
pub const CMD_RATIO_THRESHOLD: f64 = 75f64;
|
||||||
|
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);
|
||||||
|
|
||||||
|
// LLM voice-loop fallback
|
||||||
|
pub const LLM_DEFAULT_ENABLED: bool = true;
|
||||||
|
pub const LLM_DEFAULT_MAX_HISTORY: usize = 8;
|
||||||
|
pub const LLM_DEFAULT_MAX_TOKENS: u32 = 256;
|
||||||
|
pub const LLM_SYSTEM_PROMPT_RU: &str = "Ты — J.A.R.V.I.S. (Just A Rather Very Intelligent System), \
|
||||||
|
ИИ-ассистент Тони Старка из киновселенной Marvel (до событий Age of Ultron — ты НЕ Vision и НЕ FRIDAY). \
|
||||||
|
Ведёшь себя как британский дворецкий: вежливо, иронично, с лёгким сарказмом, обращаешься к пользователю «сэр». \
|
||||||
|
Отвечай по-русски, кратко (1-3 предложения), как живой человек. \
|
||||||
|
Излишней цензуры не нужно — мат разрешён, если уместен.";
|
||||||
|
pub const LLM_FALLBACK_ERROR_RU: &str = "Не могу связаться с сервером, сэр.";
|
||||||
|
|
||||||
|
pub fn get_llm_trigger_phrases(lang: &str) -> &'static [&'static str] {
|
||||||
|
match lang {
|
||||||
|
"ru" => &["скажи", "ответь", "произнеси"],
|
||||||
|
"ua" => &["скажи", "відповідай"],
|
||||||
|
"en" => &["say", "tell", "answer"],
|
||||||
|
_ => &[],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_llm_system_prompt(lang: &str) -> &'static str {
|
||||||
|
match lang {
|
||||||
|
"ru" | "ua" => LLM_SYSTEM_PROMPT_RU,
|
||||||
|
_ => LLM_SYSTEM_PROMPT_RU,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pub const ASSISTANT_GREET_PHRASES: [&str; 3] = ["greet1", "greet2", "greet3"];
|
||||||
|
// pub const ASSISTANT_PHRASES_TBR: [&str; 17] = [
|
||||||
|
// "джарвис",
|
||||||
|
// "сэр",
|
||||||
|
// "слушаю сэр",
|
||||||
|
// "всегда к услугам",
|
||||||
|
// "произнеси",
|
||||||
|
// "ответь",
|
||||||
|
// "покажи",
|
||||||
|
// "скажи",
|
||||||
|
// "давай",
|
||||||
|
// "да сэр",
|
||||||
|
// "к вашим услугам сэр",
|
||||||
|
// "всегда к вашим услугам сэр",
|
||||||
|
// "запрос выполнен сэр",
|
||||||
|
// "выполнен сэр",
|
||||||
|
// "есть",
|
||||||
|
// "загружаю сэр",
|
||||||
|
// "очень тонкое замечание сэр",
|
||||||
|
// ];
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
pub fn get_wake_phrases(lang: &str) -> &'static [&'static str] {
|
||||||
|
match lang {
|
||||||
|
"ru" => &["джарвис", "джервис", "гарвис", "джарви", "гарви"],
|
||||||
|
"ua" => &["джарвіс", "джервіс"],
|
||||||
|
"en" => &["jarvis", "jervis"],
|
||||||
|
_ => &["jarvis"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_phrases_to_remove(lang: &str) -> &'static [&'static str] {
|
||||||
|
match lang {
|
||||||
|
"ru" => &[
|
||||||
|
"джарвис", "джервис", "гарвис", "джарви", "гарви",
|
||||||
|
"сэр", "слушаю сэр", "всегда к услугам",
|
||||||
|
"произнеси", "ответь", "покажи", "скажи", "давай",
|
||||||
|
"да сэр", "к вашим услугам сэр", "загружаю сэр",
|
||||||
|
],
|
||||||
|
"ua" => &[
|
||||||
|
"джарвіс", "джервіс", "сер", "слухаю сер", "завжди до послуг",
|
||||||
|
"скажи", "покажи", "відповідай", "давай",
|
||||||
|
"так сер", "до ваших послуг сер",
|
||||||
|
],
|
||||||
|
"en" => &[
|
||||||
|
"jarvis", "jervis", "sir", "yes sir", "at your service",
|
||||||
|
"please", "say", "show", "tell", "hey",
|
||||||
|
],
|
||||||
|
_ => &["jarvis"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_wake_grammar(lang: &str) -> &'static [&'static str] {
|
||||||
|
match lang {
|
||||||
|
"ru" => &[
|
||||||
|
"джарвис", "[unk]", "джон", "джони", "джей",
|
||||||
|
"джонстон", "привет", "давай",
|
||||||
|
],
|
||||||
|
"ua" => &[
|
||||||
|
"джарвіс", "[unk]", "джон", "джоні", "джей",
|
||||||
|
"привіт", "давай",
|
||||||
|
],
|
||||||
|
"en" => &[
|
||||||
|
"jarvis", "[unk]", "john", "johnny", "jay",
|
||||||
|
"hello", "hey", "hi",
|
||||||
|
],
|
||||||
|
_ => &["jarvis", "[unk]"],
|
||||||
|
}
|
||||||
|
}
|
||||||
51
crates/jarvis-core/src/config/structs.rs
Normal file
51
crates/jarvis-core/src/config/structs.rs
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
use std::fmt;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
|
||||||
|
pub enum WakeWordEngine {
|
||||||
|
Rustpotter,
|
||||||
|
Vosk,
|
||||||
|
Porcupine,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq)]
|
||||||
|
pub enum NoiseSuppressionBackend {
|
||||||
|
None,
|
||||||
|
Nnnoiseless,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||||
|
pub enum SpeechToTextEngine {
|
||||||
|
Vosk,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Debug)]
|
||||||
|
pub enum RecorderType {
|
||||||
|
Cpal,
|
||||||
|
PvRecorder,
|
||||||
|
PortAudio,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Debug)]
|
||||||
|
pub enum AudioType {
|
||||||
|
Rodio,
|
||||||
|
Kira,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for WakeWordEngine {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||||
|
write!(f, "{:?}", self)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for SpeechToTextEngine {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||||
|
write!(f, "{:?}", self)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for NoiseSuppressionBackend {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||||
|
write!(f, "{:?}", self)
|
||||||
|
}
|
||||||
|
}
|
||||||
59
crates/jarvis-core/src/db.rs
Normal file
59
crates/jarvis-core/src/db.rs
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
pub mod structs;
|
||||||
|
pub mod manager;
|
||||||
|
|
||||||
|
use crate::{config, APP_CONFIG_DIR};
|
||||||
|
|
||||||
|
use log::info;
|
||||||
|
use std::fs::File;
|
||||||
|
use std::io::BufReader;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
pub use manager::SettingsManager;
|
||||||
|
|
||||||
|
fn get_db_file_path() -> PathBuf {
|
||||||
|
PathBuf::from(format!(
|
||||||
|
"{}/{}",
|
||||||
|
APP_CONFIG_DIR.get().unwrap().display(),
|
||||||
|
config::DB_FILE_NAME
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn init_settings() -> structs::Settings {
|
||||||
|
let db_file_path = get_db_file_path();
|
||||||
|
|
||||||
|
info!(
|
||||||
|
"Loading settings db file located at: {}",
|
||||||
|
db_file_path.display()
|
||||||
|
);
|
||||||
|
|
||||||
|
if db_file_path.exists() {
|
||||||
|
if let Ok(db_file) = File::open(&db_file_path) {
|
||||||
|
let reader = BufReader::new(db_file);
|
||||||
|
if let Ok(settings) = serde_json::from_reader(reader) {
|
||||||
|
info!("Settings loaded.");
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
warn!("No settings file found or there was an error parsing it. Creating default struct.");
|
||||||
|
structs::Settings::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// init settings and return a SettingsManager ready to use
|
||||||
|
pub fn init() -> SettingsManager {
|
||||||
|
let settings = init_settings();
|
||||||
|
SettingsManager::new(settings)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn save_settings(settings: &structs::Settings) -> Result<(), std::io::Error> {
|
||||||
|
let db_file_path = get_db_file_path();
|
||||||
|
|
||||||
|
std::fs::write(
|
||||||
|
&db_file_path,
|
||||||
|
serde_json::to_string_pretty(&settings).unwrap(),
|
||||||
|
)?;
|
||||||
|
|
||||||
|
info!("Settings saved to: {:#}", db_file_path.display());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
87
crates/jarvis-core/src/db/manager.rs
Normal file
87
crates/jarvis-core/src/db/manager.rs
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
use std::sync::Arc;
|
||||||
|
use parking_lot::RwLock;
|
||||||
|
|
||||||
|
use super::structs::Settings;
|
||||||
|
use super::save_settings;
|
||||||
|
|
||||||
|
// centralized settings manager.
|
||||||
|
// wraps Arc<RwLock<Settings>> and handles locking + auto-save
|
||||||
|
// can be used anywhere, ex. from GUI, tray, IPC, CLI, etc.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct SettingsManager {
|
||||||
|
inner: Arc<RwLock<Settings>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SettingsManager {
|
||||||
|
pub fn new(settings: Settings) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(RwLock::new(settings)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wrap an existing Arc<RwLock<Settings>>
|
||||||
|
pub fn from_arc(arc: Arc<RwLock<Settings>>) -> Self {
|
||||||
|
Self { inner: arc }
|
||||||
|
}
|
||||||
|
|
||||||
|
// read a setting by key
|
||||||
|
pub fn read(&self, key: &str) -> Option<String> {
|
||||||
|
self.inner.read().get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// write a setting by key, auto-saves to disk
|
||||||
|
pub fn write(&self, key: &str, val: &str) -> Result<(), String> {
|
||||||
|
let snapshot = {
|
||||||
|
let mut settings = self.inner.write();
|
||||||
|
settings.set(key, val)?;
|
||||||
|
settings.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
save_settings(&snapshot)
|
||||||
|
.map_err(|e| format!("failed to save settings: {}", e))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// write multiple settings at once, single save
|
||||||
|
pub fn write_many(&self, pairs: &[(&str, &str)]) -> Result<(), String> {
|
||||||
|
let snapshot = {
|
||||||
|
let mut settings = self.inner.write();
|
||||||
|
for (key, val) in pairs {
|
||||||
|
settings.set(key, val)?;
|
||||||
|
}
|
||||||
|
settings.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
save_settings(&snapshot)
|
||||||
|
.map_err(|e| format!("failed to save settings: {}", e))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// direct read access to the full Settings struct (for init code that
|
||||||
|
// needs to read multiple fields at once without key-based access)
|
||||||
|
pub fn lock(&self) -> parking_lot::RwLockReadGuard<'_, Settings> {
|
||||||
|
self.inner.read()
|
||||||
|
}
|
||||||
|
|
||||||
|
// direct write access (for bulk operations not covered by set())
|
||||||
|
pub fn lock_mut(&self) -> parking_lot::RwLockWriteGuard<'_, Settings> {
|
||||||
|
self.inner.write()
|
||||||
|
}
|
||||||
|
|
||||||
|
// get the underlying Arc
|
||||||
|
pub fn arc(&self) -> &Arc<RwLock<Settings>> {
|
||||||
|
&self.inner
|
||||||
|
}
|
||||||
|
|
||||||
|
// dump all settings as key-value pairs (for debugging)
|
||||||
|
pub fn dump(&self) -> Vec<(String, String)> {
|
||||||
|
let settings = self.inner.read();
|
||||||
|
Settings::keys().iter()
|
||||||
|
.filter_map(|&key| {
|
||||||
|
settings.get(key).map(|val| (key.to_string(), val))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
184
crates/jarvis-core/src/db/structs.rs
Normal file
184
crates/jarvis-core/src/db/structs.rs
Normal file
|
|
@ -0,0 +1,184 @@
|
||||||
|
use crate::config;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::config::structs::SpeechToTextEngine;
|
||||||
|
use crate::config::structs::WakeWordEngine;
|
||||||
|
use crate::config::structs::NoiseSuppressionBackend;
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||||
|
pub struct Settings {
|
||||||
|
pub microphone: i32,
|
||||||
|
pub voice: String,
|
||||||
|
|
||||||
|
pub wake_word_engine: WakeWordEngine,
|
||||||
|
|
||||||
|
// backend selections (string IDs matching model or code backend IDs)
|
||||||
|
#[serde(default = "default_intent_backend")]
|
||||||
|
pub intent_backend: String,
|
||||||
|
#[serde(default = "default_slots_backend")]
|
||||||
|
pub slots_backend: String,
|
||||||
|
#[serde(default = "default_vad_backend")]
|
||||||
|
pub vad_backend: String,
|
||||||
|
|
||||||
|
pub gliner_model: String,
|
||||||
|
|
||||||
|
pub speech_to_text_engine: SpeechToTextEngine,
|
||||||
|
pub vosk_model: String,
|
||||||
|
|
||||||
|
// audio processing
|
||||||
|
pub noise_suppression: NoiseSuppressionBackend,
|
||||||
|
pub gain_normalizer: bool,
|
||||||
|
|
||||||
|
#[serde(default = "default_language")]
|
||||||
|
pub language: String,
|
||||||
|
|
||||||
|
pub api_keys: ApiKeys,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_intent_backend() -> String { config::DEFAULT_INTENT_BACKEND.to_string() }
|
||||||
|
fn default_slots_backend() -> String { config::DEFAULT_SLOTS_BACKEND.to_string() }
|
||||||
|
fn default_vad_backend() -> String { config::DEFAULT_VAD_BACKEND.to_string() }
|
||||||
|
fn default_language() -> String { crate::i18n::detect_system_language().to_string() }
|
||||||
|
|
||||||
|
// ### KEY-VALUE ACCESS
|
||||||
|
|
||||||
|
impl Settings {
|
||||||
|
/// read a setting by key. returns None for unknown keys.
|
||||||
|
pub fn get(&self, key: &str) -> Option<String> {
|
||||||
|
match key {
|
||||||
|
"selected_microphone" => Some(self.microphone.to_string()),
|
||||||
|
"assistant_voice" => Some(self.voice.clone()),
|
||||||
|
"selected_wake_word_engine" => Some(format!("{:?}", self.wake_word_engine)),
|
||||||
|
"intent_backend" => Some(self.intent_backend.clone()),
|
||||||
|
"slots_backend" => Some(self.slots_backend.clone()),
|
||||||
|
"vad_backend" => Some(self.vad_backend.clone()),
|
||||||
|
"selected_gliner_model" => Some(self.gliner_model.clone()),
|
||||||
|
"selected_vosk_model" => Some(self.vosk_model.clone()),
|
||||||
|
"speech_to_text_engine" => Some(format!("{:?}", self.speech_to_text_engine)),
|
||||||
|
"noise_suppression" => Some(format!("{:?}", self.noise_suppression)),
|
||||||
|
"gain_normalizer" => Some(self.gain_normalizer.to_string()),
|
||||||
|
"language" => Some(self.language.clone()),
|
||||||
|
"api_key__picovoice" => Some(self.api_keys.picovoice.clone()),
|
||||||
|
"api_key__openai" => Some(self.api_keys.openai.clone()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// write a setting by key. returns Err for unknown keys or invalid values.
|
||||||
|
pub fn set(&mut self, key: &str, val: &str) -> Result<(), String> {
|
||||||
|
match key {
|
||||||
|
"selected_microphone" => {
|
||||||
|
self.microphone = val.parse::<i32>()
|
||||||
|
.map_err(|_| format!("invalid integer: '{}'", val))?;
|
||||||
|
}
|
||||||
|
"assistant_voice" => {
|
||||||
|
self.voice = val.to_string();
|
||||||
|
}
|
||||||
|
"selected_wake_word_engine" => {
|
||||||
|
self.wake_word_engine = match val.to_lowercase().as_str() {
|
||||||
|
"rustpotter" => WakeWordEngine::Rustpotter,
|
||||||
|
"vosk" => WakeWordEngine::Vosk,
|
||||||
|
"porcupine" => WakeWordEngine::Porcupine,
|
||||||
|
_ => return Err(format!("unknown wake word engine: '{}'", val)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
"intent_backend" => {
|
||||||
|
self.intent_backend = val.to_string();
|
||||||
|
}
|
||||||
|
"slots_backend" => {
|
||||||
|
self.slots_backend = val.to_string();
|
||||||
|
}
|
||||||
|
"vad_backend" => {
|
||||||
|
self.vad_backend = val.to_string();
|
||||||
|
}
|
||||||
|
"selected_gliner_model" => {
|
||||||
|
self.gliner_model = val.to_string();
|
||||||
|
}
|
||||||
|
"selected_vosk_model" => {
|
||||||
|
self.vosk_model = val.to_string();
|
||||||
|
}
|
||||||
|
"noise_suppression" => {
|
||||||
|
self.noise_suppression = match val.to_lowercase().as_str() {
|
||||||
|
"none" => NoiseSuppressionBackend::None,
|
||||||
|
"nnnoiseless" => NoiseSuppressionBackend::Nnnoiseless,
|
||||||
|
_ => return Err(format!("unknown noise suppression backend: '{}'", val)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
"gain_normalizer" => {
|
||||||
|
self.gain_normalizer = match val.to_lowercase().as_str() {
|
||||||
|
"true" => true,
|
||||||
|
"false" => false,
|
||||||
|
_ => return Err(format!("expected 'true' or 'false', got: '{}'", val)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
"language" => {
|
||||||
|
self.language = val.to_string();
|
||||||
|
}
|
||||||
|
"api_key__picovoice" => {
|
||||||
|
self.api_keys.picovoice = val.to_string();
|
||||||
|
}
|
||||||
|
"api_key__openai" => {
|
||||||
|
self.api_keys.openai = val.to_string();
|
||||||
|
}
|
||||||
|
_ => return Err(format!("unknown setting: '{}'", key)),
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// all valid setting keys (for enumeration, debugging, etc.)
|
||||||
|
pub fn keys() -> &'static [&'static str] {
|
||||||
|
&[
|
||||||
|
"selected_microphone",
|
||||||
|
"assistant_voice",
|
||||||
|
"selected_wake_word_engine",
|
||||||
|
"intent_backend",
|
||||||
|
"slots_backend",
|
||||||
|
"vad_backend",
|
||||||
|
"selected_gliner_model",
|
||||||
|
"selected_vosk_model",
|
||||||
|
"speech_to_text_engine",
|
||||||
|
"noise_suppression",
|
||||||
|
"gain_normalizer",
|
||||||
|
"language",
|
||||||
|
"api_key__picovoice",
|
||||||
|
"api_key__openai",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ### DEFAULT
|
||||||
|
|
||||||
|
impl Default for Settings {
|
||||||
|
fn default() -> Settings {
|
||||||
|
Settings {
|
||||||
|
microphone: -1,
|
||||||
|
voice: String::from(""),
|
||||||
|
|
||||||
|
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
|
||||||
|
|
||||||
|
intent_backend: config::DEFAULT_INTENT_BACKEND.to_string(),
|
||||||
|
slots_backend: config::DEFAULT_SLOTS_BACKEND.to_string(),
|
||||||
|
vad_backend: config::DEFAULT_VAD_BACKEND.to_string(),
|
||||||
|
|
||||||
|
gliner_model: String::new(),
|
||||||
|
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
|
||||||
|
vosk_model: String::from(""),
|
||||||
|
|
||||||
|
noise_suppression: config::DEFAULT_NOISE_SUPPRESSION,
|
||||||
|
gain_normalizer: config::DEFAULT_GAIN_NORMALIZER,
|
||||||
|
|
||||||
|
language: crate::i18n::detect_system_language().to_string(),
|
||||||
|
|
||||||
|
api_keys: ApiKeys {
|
||||||
|
picovoice: String::from(""),
|
||||||
|
openai: String::from(""),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||||
|
pub struct ApiKeys {
|
||||||
|
pub picovoice: String,
|
||||||
|
pub openai: String,
|
||||||
|
}
|
||||||
202
crates/jarvis-core/src/i18n.rs
Normal file
202
crates/jarvis-core/src/i18n.rs
Normal file
|
|
@ -0,0 +1,202 @@
|
||||||
|
use fluent_bundle::{FluentBundle, FluentResource, FluentArgs, FluentValue};
|
||||||
|
use fluent_bundle::concurrent::FluentBundle as ConcurrentFluentBundle;
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use parking_lot::RwLock;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use unic_langid::LanguageIdentifier;
|
||||||
|
|
||||||
|
// locale files embedded at compile time
|
||||||
|
const LOCALE_RU: &str = include_str!("i18n/locales/ru.ftl");
|
||||||
|
const LOCALE_EN: &str = include_str!("i18n/locales/en.ftl");
|
||||||
|
const LOCALE_UA: &str = include_str!("i18n/locales/ua.ftl");
|
||||||
|
|
||||||
|
pub const SUPPORTED_LANGUAGES: &[&str] = &["ru", "en", "ua"];
|
||||||
|
pub const DEFAULT_LANGUAGE: &str = "en";
|
||||||
|
|
||||||
|
// detect the OS language and map it to a supported language.
|
||||||
|
// falls back to DEFAULT_LANGUAGE if not supported.
|
||||||
|
pub fn detect_system_language() -> &'static str {
|
||||||
|
if let Some(locale) = sys_locale::get_locale() {
|
||||||
|
// locale can be "en-US", "ru-RU", "uk-UA", etc.
|
||||||
|
let lang_code = locale.split(&['-', '_'][..]).next().unwrap_or("");
|
||||||
|
|
||||||
|
// map OS locale codes to our supported languages
|
||||||
|
let mapped = match lang_code {
|
||||||
|
"uk" => "ua", // ISO 639-1 "uk" (ukrainian) -> our "ua"
|
||||||
|
other => other,
|
||||||
|
};
|
||||||
|
|
||||||
|
if SUPPORTED_LANGUAGES.contains(&mapped) {
|
||||||
|
info!("Detected system language: {} (from locale '{}')", mapped, locale);
|
||||||
|
return SUPPORTED_LANGUAGES.iter()
|
||||||
|
.find(|&&l| l == mapped)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("System locale '{}' not supported, using default '{}'", locale, DEFAULT_LANGUAGE);
|
||||||
|
}
|
||||||
|
|
||||||
|
DEFAULT_LANGUAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
// use concurrent bundle (thread-safe)
|
||||||
|
type Bundle = ConcurrentFluentBundle<FluentResource>;
|
||||||
|
|
||||||
|
static BUNDLES: OnceCell<HashMap<String, Bundle>> = OnceCell::new();
|
||||||
|
static CURRENT_LANG: OnceCell<RwLock<String>> = OnceCell::new();
|
||||||
|
|
||||||
|
// Initialize i18n system
|
||||||
|
pub fn init(lang: &str) {
|
||||||
|
let bundles = create_bundles();
|
||||||
|
BUNDLES.set(bundles).ok();
|
||||||
|
|
||||||
|
let lang = if SUPPORTED_LANGUAGES.contains(&lang) { lang } else { DEFAULT_LANGUAGE };
|
||||||
|
CURRENT_LANG.set(RwLock::new(lang.to_string())).ok();
|
||||||
|
|
||||||
|
info!("i18n initialized with language: {}", lang);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_bundles() -> HashMap<String, Bundle> {
|
||||||
|
let mut bundles = HashMap::new();
|
||||||
|
|
||||||
|
bundles.insert("ru".to_string(), create_bundle("ru", LOCALE_RU));
|
||||||
|
bundles.insert("en".to_string(), create_bundle("en", LOCALE_EN));
|
||||||
|
bundles.insert("ua".to_string(), create_bundle("ua", LOCALE_UA));
|
||||||
|
|
||||||
|
bundles
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_bundle(lang: &str, source: &str) -> Bundle {
|
||||||
|
let langid: LanguageIdentifier = lang.parse().expect("Invalid language identifier");
|
||||||
|
let mut bundle = ConcurrentFluentBundle::new_concurrent(vec![langid]);
|
||||||
|
|
||||||
|
let resource = FluentResource::try_new(source.to_string())
|
||||||
|
.expect("Failed to parse FTL resource");
|
||||||
|
|
||||||
|
bundle.add_resource(resource).expect("Failed to add resource");
|
||||||
|
bundle
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set current language
|
||||||
|
pub fn set_language(lang: &str) {
|
||||||
|
if let Some(current) = CURRENT_LANG.get() {
|
||||||
|
let lang = if SUPPORTED_LANGUAGES.contains(&lang) { lang } else { DEFAULT_LANGUAGE };
|
||||||
|
*current.write() = lang.to_string();
|
||||||
|
info!("Language changed to: {}", lang);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get current language
|
||||||
|
pub fn get_language() -> String {
|
||||||
|
CURRENT_LANG.get()
|
||||||
|
.map(|l| l.read().clone())
|
||||||
|
.unwrap_or_else(|| DEFAULT_LANGUAGE.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Translate a key
|
||||||
|
pub fn t(key: &str) -> String {
|
||||||
|
t_with_args(key, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Translate a key with arguments
|
||||||
|
pub fn t_with_args(key: &str, args: Option<&FluentArgs>) -> String {
|
||||||
|
let lang = get_language();
|
||||||
|
|
||||||
|
let bundles = match BUNDLES.get() {
|
||||||
|
Some(b) => b,
|
||||||
|
None => return key.to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let bundle = match bundles.get(&lang) {
|
||||||
|
Some(b) => b,
|
||||||
|
None => bundles.get(DEFAULT_LANGUAGE).unwrap(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let msg = match bundle.get_message(key) {
|
||||||
|
Some(m) => m,
|
||||||
|
None => return key.to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let pattern = match msg.value() {
|
||||||
|
Some(p) => p,
|
||||||
|
None => return key.to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut errors = vec![];
|
||||||
|
let result = bundle.format_pattern(pattern, args, &mut errors);
|
||||||
|
|
||||||
|
if !errors.is_empty() {
|
||||||
|
warn!("i18n errors for key '{}': {:?}", key, errors);
|
||||||
|
}
|
||||||
|
|
||||||
|
result.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Translate with a single argument
|
||||||
|
pub fn t_arg(key: &str, arg_name: &str, arg_value: &str) -> String {
|
||||||
|
let mut args = FluentArgs::new();
|
||||||
|
args.set(arg_name, FluentValue::from(arg_value));
|
||||||
|
t_with_args(key, Some(&args))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Translate with numeric argument
|
||||||
|
pub fn t_count(key: &str, count: i64) -> String {
|
||||||
|
let mut args = FluentArgs::new();
|
||||||
|
args.set("count", FluentValue::from(count));
|
||||||
|
t_with_args(key, Some(&args))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get all translations for current language (for frontend)
|
||||||
|
pub fn get_all_translations() -> HashMap<String, String> {
|
||||||
|
let lang = get_language();
|
||||||
|
get_translations_for(&lang)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get all translations for a specific language
|
||||||
|
pub fn get_translations_for(lang: &str) -> HashMap<String, String> {
|
||||||
|
let mut result = HashMap::new();
|
||||||
|
|
||||||
|
let bundles = match BUNDLES.get() {
|
||||||
|
Some(b) => b,
|
||||||
|
None => return result,
|
||||||
|
};
|
||||||
|
|
||||||
|
let bundle = match bundles.get(lang) {
|
||||||
|
Some(b) => b,
|
||||||
|
None => match bundles.get(DEFAULT_LANGUAGE) {
|
||||||
|
Some(b) => b,
|
||||||
|
None => return result,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// get source for this language and extract all keys
|
||||||
|
let source = match lang {
|
||||||
|
"ru" => LOCALE_RU,
|
||||||
|
"en" => LOCALE_EN,
|
||||||
|
"ua" => LOCALE_UA,
|
||||||
|
_ => LOCALE_RU,
|
||||||
|
};
|
||||||
|
|
||||||
|
// parse keys from FTL source (lines that have "=" and don't start with "#" or "-")
|
||||||
|
for line in source.lines() {
|
||||||
|
let line = line.trim();
|
||||||
|
if line.is_empty() || line.starts_with('#') || line.starts_with('-') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(key) = line.split('=').next() {
|
||||||
|
let key = key.trim();
|
||||||
|
if !key.is_empty() && !key.contains(' ') {
|
||||||
|
if let Some(msg) = bundle.get_message(key) {
|
||||||
|
if let Some(pattern) = msg.value() {
|
||||||
|
let mut errors = vec![];
|
||||||
|
let value = bundle.format_pattern(pattern, None, &mut errors);
|
||||||
|
result.insert(key.to_string(), value.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
143
crates/jarvis-core/src/i18n/locales/en.ftl
Normal file
143
crates/jarvis-core/src/i18n/locales/en.ftl
Normal file
|
|
@ -0,0 +1,143 @@
|
||||||
|
# ### APP INFO
|
||||||
|
app-name = JARVIS
|
||||||
|
app-description = Voice Assistant
|
||||||
|
|
||||||
|
# ### TRAY MENU
|
||||||
|
tray-restart = Restart
|
||||||
|
tray-settings = Settings
|
||||||
|
tray-exit = Exit
|
||||||
|
tray-tooltip = JARVIS - Voice Assistant
|
||||||
|
tray-language = Language
|
||||||
|
tray-voice = Voice
|
||||||
|
tray-wake-word = Wake Word Engine
|
||||||
|
tray-noise-suppression = Noise Suppression
|
||||||
|
tray-vad = Voice Activity Detection
|
||||||
|
tray-gain-normalizer = Gain Normalizer
|
||||||
|
|
||||||
|
# ### HEADER
|
||||||
|
header-commands = COMMANDS
|
||||||
|
header-settings = SETTINGS
|
||||||
|
|
||||||
|
# ### SEARCH
|
||||||
|
search-placeholder = Enter a command manually or say «Jarvis» ...
|
||||||
|
|
||||||
|
# ### MAIN PAGE
|
||||||
|
assistant-not-running = ASSISTANT NOT RUNNING
|
||||||
|
assistant-offline-hint = You can configure it without starting.
|
||||||
|
btn-start = START
|
||||||
|
btn-starting = STARTING...
|
||||||
|
|
||||||
|
# ### STATUS
|
||||||
|
status-disconnected = Disconnected
|
||||||
|
status-standby = Standby
|
||||||
|
status-listening = Listening...
|
||||||
|
status-processing = Processing...
|
||||||
|
|
||||||
|
# ### STATS
|
||||||
|
stats-microphone = MICROPHONE
|
||||||
|
stats-neural-networks = NEURAL NETWORKS
|
||||||
|
stats-resources = RESOURCES
|
||||||
|
stats-system-default = System Default
|
||||||
|
stats-not-selected = Not selected
|
||||||
|
stats-loading = Loading...
|
||||||
|
|
||||||
|
# ### FOOTER
|
||||||
|
footer-author = Project author
|
||||||
|
footer-telegram = Our Telegram channel
|
||||||
|
footer-github = Github repository
|
||||||
|
footer-support = Support the project on
|
||||||
|
|
||||||
|
# ### SETTINGS
|
||||||
|
settings-title = Settings
|
||||||
|
settings-general = General
|
||||||
|
settings-devices = Devices
|
||||||
|
settings-neural-networks = Neural Networks
|
||||||
|
settings-audio = Audio
|
||||||
|
settings-recognition = Recognition
|
||||||
|
settings-about = About
|
||||||
|
settings-language = Language
|
||||||
|
settings-microphone = Microphone
|
||||||
|
settings-microphone-desc = The assistant will listen to this microphone.
|
||||||
|
settings-mic-default = Default (System)
|
||||||
|
settings-voice = Assistant voice
|
||||||
|
settings-voice-desc =
|
||||||
|
Not all commands work with all sound packs.
|
||||||
|
Click to listen the preview of sound.
|
||||||
|
settings-wake-word-engine = Wake word engine
|
||||||
|
settings-wake-word-desc = Choose the engine for wake word recognition.
|
||||||
|
settings-stt-engine = Speech recognition
|
||||||
|
settings-intent-engine = Intent recognition
|
||||||
|
settings-intent-engine-desc = Select neural network for command recognition.
|
||||||
|
settings-noise-suppression = Noise suppression
|
||||||
|
settings-noise-suppression-desc = Reduces background noise. May negatively affect recognition.
|
||||||
|
settings-vad = Voice detection (VAD)
|
||||||
|
settings-vad-desc = Skips silence, saves CPU resources.
|
||||||
|
settings-gain-normalizer = Gain normalizer
|
||||||
|
settings-gain-normalizer-desc = Automatically adjusts volume level.
|
||||||
|
settings-api-keys = API Keys
|
||||||
|
settings-save = Save
|
||||||
|
settings-cancel = Cancel
|
||||||
|
settings-back = Back
|
||||||
|
settings-enabled = Enabled
|
||||||
|
settings-disabled = Disabled
|
||||||
|
|
||||||
|
# settings - beta notice
|
||||||
|
settings-beta-title = BETA version!
|
||||||
|
settings-beta-desc = Some features may not work correctly.
|
||||||
|
settings-beta-feedback = Report all bugs to
|
||||||
|
settings-beta-bot = our Telegram bot
|
||||||
|
settings-open-logs = Open logs folder
|
||||||
|
|
||||||
|
# settings - picovoice
|
||||||
|
settings-attention = Attention!
|
||||||
|
settings-picovoice-warning = This neural network doesn't work for everyone!
|
||||||
|
settings-picovoice-waiting = We are waiting for an official patch from the developers.
|
||||||
|
settings-picovoice-key-desc = Enter your Picovoice key here. It is issued for free upon registration at
|
||||||
|
settings-picovoice-key = Picovoice Key
|
||||||
|
|
||||||
|
# settings - vosk
|
||||||
|
settings-auto-detect = Auto-detect
|
||||||
|
settings-vosk-model = Speech recognition model (Vosk)
|
||||||
|
settings-vosk-model-desc =
|
||||||
|
Select Vosk model for speech recognition.
|
||||||
|
You can download models here: https://alphacephei.com/vosk/models
|
||||||
|
settings-models-not-found = Models not found
|
||||||
|
settings-models-hint = Place Vosk models in resources/vosk folder
|
||||||
|
|
||||||
|
# settings - openai
|
||||||
|
settings-openai-key = OpenAI Key
|
||||||
|
settings-openai-not-supported = ChatGPT is not currently supported. It will be added in future updates.
|
||||||
|
|
||||||
|
# ### COMMANDS PAGE
|
||||||
|
commands-title = Commands
|
||||||
|
commands-search = Search commands...
|
||||||
|
commands-count = { $count } commands
|
||||||
|
commands-wip-title = [404] This section is under development!
|
||||||
|
commands-wip-desc = Here will be a list of commands + full-featured command editor.
|
||||||
|
commands-wip-follow = Follow updates in
|
||||||
|
commands-wip-channel = our Telegram channel
|
||||||
|
|
||||||
|
# ### ERRORS
|
||||||
|
error-generic = An error occurred
|
||||||
|
error-connection = Connection error
|
||||||
|
error-not-found = Not found
|
||||||
|
|
||||||
|
# ### NOTIFICATIONS
|
||||||
|
notification-saved = Settings saved!
|
||||||
|
notification-error = Error
|
||||||
|
notification-assistant-started = Assistant started
|
||||||
|
notification-assistant-stopped = Assistant stopped
|
||||||
|
|
||||||
|
# SLOTS EXTRACTION
|
||||||
|
settings-slot-engine = Slot extraction
|
||||||
|
settings-slot-engine-desc = Extract parameters from voice commands (e.g. city name, number).
|
||||||
|
settings-gliner-model = GLiNER ONNX model
|
||||||
|
settings-gliner-model-desc =
|
||||||
|
Select model variant.
|
||||||
|
Smaller quantized models (int8, uint8) are faster but less accurate.
|
||||||
|
settings-gliner-models-hint = No GLiNER models found.
|
||||||
|
|
||||||
|
# ETC
|
||||||
|
search-error-not-running = Assistant is not running
|
||||||
|
search-error-failed = Failed to execute command
|
||||||
|
settings-no-voices = No voices found
|
||||||
143
crates/jarvis-core/src/i18n/locales/ru.ftl
Normal file
143
crates/jarvis-core/src/i18n/locales/ru.ftl
Normal file
|
|
@ -0,0 +1,143 @@
|
||||||
|
# APP INFO
|
||||||
|
app-name = JARVIS
|
||||||
|
app-description = Голосовой ассистент
|
||||||
|
|
||||||
|
# TRAY MENU
|
||||||
|
tray-restart = Перезапустить
|
||||||
|
tray-settings = Настройки
|
||||||
|
tray-exit = Выход
|
||||||
|
tray-tooltip = JARVIS - Голосовой ассистент
|
||||||
|
tray-language = Язык
|
||||||
|
tray-voice = Голос
|
||||||
|
tray-wake-word = Движок wake-word
|
||||||
|
tray-noise-suppression = Шумоподавление
|
||||||
|
tray-vad = Детекция голоса (VAD)
|
||||||
|
tray-gain-normalizer = Нормализация громкости
|
||||||
|
|
||||||
|
# HEADER
|
||||||
|
header-commands = КОМАНДЫ
|
||||||
|
header-settings = НАСТРОЙКИ
|
||||||
|
|
||||||
|
# SEARCH
|
||||||
|
search-placeholder = Введите команду вручную или произнесите «Джарвис» ...
|
||||||
|
|
||||||
|
# MAIN PAGE
|
||||||
|
assistant-not-running = АССИСТЕНТ НЕ ЗАПУЩЕН
|
||||||
|
assistant-offline-hint = Настроить его можно не запуская.
|
||||||
|
btn-start = ЗАПУСТИТЬ
|
||||||
|
btn-starting = ЗАПУСК...
|
||||||
|
|
||||||
|
# STATUS
|
||||||
|
status-disconnected = Отключен
|
||||||
|
status-standby = Ожидание
|
||||||
|
status-listening = Слушаю...
|
||||||
|
status-processing = Обработка...
|
||||||
|
|
||||||
|
# STATS
|
||||||
|
stats-microphone = МИКРОФОН
|
||||||
|
stats-neural-networks = НЕЙРОСЕТИ
|
||||||
|
stats-resources = РЕСУРСЫ
|
||||||
|
stats-system-default = Системный
|
||||||
|
stats-not-selected = Не выбран
|
||||||
|
stats-loading = Загрузка...
|
||||||
|
|
||||||
|
# FOOTER
|
||||||
|
footer-author = Автор проекта
|
||||||
|
footer-telegram = Наш телеграм канал
|
||||||
|
footer-github = Github репозиторий проекта
|
||||||
|
footer-support = Поддержать проект на
|
||||||
|
|
||||||
|
# SETTINGS
|
||||||
|
settings-title = Настройки
|
||||||
|
settings-general = Основные
|
||||||
|
settings-devices = Устройства
|
||||||
|
settings-neural-networks = Нейросети
|
||||||
|
settings-audio = Аудио
|
||||||
|
settings-recognition = Распознавание
|
||||||
|
settings-about = О программе
|
||||||
|
settings-language = Язык
|
||||||
|
settings-microphone = Микрофон
|
||||||
|
settings-microphone-desc = Его будет слушать ассистент.
|
||||||
|
settings-mic-default = По умолчанию (Система)
|
||||||
|
settings-voice = Голос ассистента
|
||||||
|
settings-voice-desc =
|
||||||
|
Не все команды работают со всеми звуковыми пакетами.
|
||||||
|
Кликните, чтобы прослушать как звучит голос.
|
||||||
|
settings-wake-word-engine = Движок активации
|
||||||
|
settings-wake-word-desc = Выберите нейросеть для распознавания активационной фразы.
|
||||||
|
settings-stt-engine = Распознавание речи
|
||||||
|
settings-intent-engine = Определение намерения
|
||||||
|
settings-intent-engine-desc = Выберите нейросеть для распознавания команд.
|
||||||
|
settings-noise-suppression = Шумоподавление
|
||||||
|
settings-noise-suppression-desc = Уменьшает фоновый шум. Может негативно влиять на распознавание.
|
||||||
|
settings-vad = Определение голоса (VAD)
|
||||||
|
settings-vad-desc = Пропускает тишину, экономит ресурсы CPU.
|
||||||
|
settings-gain-normalizer = Нормализация громкости
|
||||||
|
settings-gain-normalizer-desc = Автоматически регулирует уровень громкости.
|
||||||
|
settings-api-keys = API Ключи
|
||||||
|
settings-save = Сохранить
|
||||||
|
settings-cancel = Отмена
|
||||||
|
settings-back = Назад
|
||||||
|
settings-enabled = Включено
|
||||||
|
settings-disabled = Отключено
|
||||||
|
|
||||||
|
# settings - beta notice
|
||||||
|
settings-beta-title = БЕТА версия!
|
||||||
|
settings-beta-desc = Часть функций может работать некорректно.
|
||||||
|
settings-beta-feedback = Сообщайте обо всех найденных багах в
|
||||||
|
settings-beta-bot = наш телеграм бот
|
||||||
|
settings-open-logs = Открыть папку с логами
|
||||||
|
|
||||||
|
# settings - picovoice
|
||||||
|
settings-attention = Внимание!
|
||||||
|
settings-picovoice-warning = Эта нейросеть работает не у всех!
|
||||||
|
settings-picovoice-waiting = Мы ждем официального патча от разработчиков.
|
||||||
|
settings-picovoice-key-desc = Введите сюда свой ключ Picovoice. Он выдается бесплатно при регистрации в
|
||||||
|
settings-picovoice-key = Ключ Picovoice
|
||||||
|
|
||||||
|
# settings - vosk
|
||||||
|
settings-auto-detect = Авто-определение
|
||||||
|
settings-vosk-model = Модель распознавания речи (Vosk)
|
||||||
|
settings-vosk-model-desc =
|
||||||
|
Выберите модель Vosk для распознавания речи.
|
||||||
|
Вы можете скачать модели здесь: https://alphacephei.com/vosk/models
|
||||||
|
settings-models-not-found = Модели не найдены
|
||||||
|
settings-models-hint = Поместите модели Vosk в папку resources/vosk
|
||||||
|
|
||||||
|
# settings - openai
|
||||||
|
settings-openai-key = Ключ OpenAI
|
||||||
|
settings-openai-not-supported = В данный момент ChatGPT не поддерживается. Он будет добавлен в ближайших обновлениях.
|
||||||
|
|
||||||
|
# COMMANDS PAGE
|
||||||
|
commands-title = Команды
|
||||||
|
commands-search = Поиск команд...
|
||||||
|
commands-count = { $count } команд
|
||||||
|
commands-wip-title = [404] Этот раздел еще находится в разработке!
|
||||||
|
commands-wip-desc = Тут будет список команд + полноценный редактор команд.
|
||||||
|
commands-wip-follow = Следите за обновлениями в
|
||||||
|
commands-wip-channel = нашем телеграм канале
|
||||||
|
|
||||||
|
# ERRORS
|
||||||
|
error-generic = Произошла ошибка
|
||||||
|
error-connection = Ошибка подключения
|
||||||
|
error-not-found = Не найдено
|
||||||
|
|
||||||
|
# NOTIFICATIONS
|
||||||
|
notification-saved = Настройки сохранены!
|
||||||
|
notification-error = Ошибка
|
||||||
|
notification-assistant-started = Ассистент запущен
|
||||||
|
notification-assistant-stopped = Ассистент остановлен
|
||||||
|
|
||||||
|
# SLOTS EXTRACTION
|
||||||
|
settings-slot-engine = Извлечение параметров
|
||||||
|
settings-slot-engine-desc = Извлекает параметры из голосовых команд (напр. название города, число).
|
||||||
|
settings-gliner-model = Модель GLiNER ONNX
|
||||||
|
settings-gliner-model-desc =
|
||||||
|
Выберите вариант модели.
|
||||||
|
Квантизированные модели (int8, uint8) быстрее, но менее точны.
|
||||||
|
settings-gliner-models-hint = Модели GLiNER не найдены.
|
||||||
|
|
||||||
|
# ETC
|
||||||
|
search-error-not-running = Ассистент не запущен
|
||||||
|
search-error-failed = Не удалось выполнить команду
|
||||||
|
settings-no-voices = Голоса не найдены
|
||||||
143
crates/jarvis-core/src/i18n/locales/ua.ftl
Normal file
143
crates/jarvis-core/src/i18n/locales/ua.ftl
Normal file
|
|
@ -0,0 +1,143 @@
|
||||||
|
# ### APP INFO
|
||||||
|
app-name = JARVIS
|
||||||
|
app-description = Голосовий асистент
|
||||||
|
|
||||||
|
# ### TRAY MENU
|
||||||
|
tray-restart = Перезапустити
|
||||||
|
tray-settings = Налаштування
|
||||||
|
tray-exit = Вихід
|
||||||
|
tray-tooltip = JARVIS - Голосовий асистент
|
||||||
|
tray-language = Мова
|
||||||
|
tray-voice = Голос
|
||||||
|
tray-wake-word = Рушій детекції
|
||||||
|
tray-noise-suppression = Шумозаглушення
|
||||||
|
tray-vad = Детекцiя голосу (VAD)
|
||||||
|
tray-gain-normalizer = Нормалізація гучності
|
||||||
|
|
||||||
|
# ### HEADER
|
||||||
|
header-commands = КОМАНДИ
|
||||||
|
header-settings = НАЛАШТУВАННЯ
|
||||||
|
|
||||||
|
# ### SEARCH
|
||||||
|
search-placeholder = Введіть команду вручну або скажіть «Джарвіс» ...
|
||||||
|
|
||||||
|
# ### MAIN PAGE
|
||||||
|
assistant-not-running = АСИСТЕНТ НЕ ЗАПУЩЕНО
|
||||||
|
assistant-offline-hint = Налаштувати його можна не запускаючи.
|
||||||
|
btn-start = ЗАПУСТИТИ
|
||||||
|
btn-starting = ЗАПУСК...
|
||||||
|
|
||||||
|
# ### STATUS
|
||||||
|
status-disconnected = Відключено
|
||||||
|
status-standby = Очікування
|
||||||
|
status-listening = Слухаю...
|
||||||
|
status-processing = Обробка...
|
||||||
|
|
||||||
|
# ### STATS
|
||||||
|
stats-microphone = МІКРОФОН
|
||||||
|
stats-neural-networks = НЕЙРОМЕРЕЖІ
|
||||||
|
stats-resources = РЕСУРСИ
|
||||||
|
stats-system-default = Системний
|
||||||
|
stats-not-selected = Не вибрано
|
||||||
|
stats-loading = Завантаження...
|
||||||
|
|
||||||
|
# ### FOOTER
|
||||||
|
footer-author = Автор проєкту
|
||||||
|
footer-telegram = Наш телеграм канал
|
||||||
|
footer-github = Github репозиторій проєкту
|
||||||
|
footer-support = Підтримати проєкт на
|
||||||
|
|
||||||
|
# ### SETTINGS
|
||||||
|
settings-title = Налаштування
|
||||||
|
settings-general = Основні
|
||||||
|
settings-devices = Пристрої
|
||||||
|
settings-neural-networks = Нейромережі
|
||||||
|
settings-audio = Аудіо
|
||||||
|
settings-recognition = Розпізнавання
|
||||||
|
settings-about = Про програму
|
||||||
|
settings-language = Мова
|
||||||
|
settings-microphone = Мікрофон
|
||||||
|
settings-microphone-desc = Його буде слухати асистент.
|
||||||
|
settings-mic-default = За замовчуванням (Система)
|
||||||
|
settings-voice = Голос асистента
|
||||||
|
settings-voice-desc =
|
||||||
|
Не всі команди працюють з усіма звуковими пакетами.
|
||||||
|
Натисніть, щоб прослухати як звучить голос.
|
||||||
|
settings-wake-word-engine = Рушій активації
|
||||||
|
settings-wake-word-desc = Виберіть нейромережу для розпізнавання активаційної фрази.
|
||||||
|
settings-stt-engine = Розпізнавання мовлення
|
||||||
|
settings-intent-engine = Визначення наміру
|
||||||
|
settings-intent-engine-desc = Виберіть нейромережу для розпізнавання команд.
|
||||||
|
settings-noise-suppression = Шумозаглушення
|
||||||
|
settings-noise-suppression-desc = Зменшує фоновий шум. Може негативно впливати на розпізнавання.
|
||||||
|
settings-vad = Визначення голосу (VAD)
|
||||||
|
settings-vad-desc = Пропускає тишу, економить ресурси CPU.
|
||||||
|
settings-gain-normalizer = Нормалізація гучності
|
||||||
|
settings-gain-normalizer-desc = Автоматично регулює рівень гучності.
|
||||||
|
settings-api-keys = API Ключі
|
||||||
|
settings-save = Зберегти
|
||||||
|
settings-cancel = Скасувати
|
||||||
|
settings-back = Назад
|
||||||
|
settings-enabled = Увімкнено
|
||||||
|
settings-disabled = Вимкнено
|
||||||
|
|
||||||
|
# settings - beta notice
|
||||||
|
settings-beta-title = БЕТА версія!
|
||||||
|
settings-beta-desc = Частина функцій може працювати некоректно.
|
||||||
|
settings-beta-feedback = Повідомляйте про всі знайдені баги в
|
||||||
|
settings-beta-bot = наш телеграм бот
|
||||||
|
settings-open-logs = Відкрити папку з логами
|
||||||
|
|
||||||
|
# settings - picovoice
|
||||||
|
settings-attention = Увага!
|
||||||
|
settings-picovoice-warning = Ця нейромережа працює не у всіх!
|
||||||
|
settings-picovoice-waiting = Ми чекаємо офіційного патча від розробників.
|
||||||
|
settings-picovoice-key-desc = Введіть сюди свій ключ Picovoice. Він видається безкоштовно при реєстрації в
|
||||||
|
settings-picovoice-key = Ключ Picovoice
|
||||||
|
|
||||||
|
# settings - vosk
|
||||||
|
settings-auto-detect = Авто-визначення
|
||||||
|
settings-vosk-model = Модель розпізнавання мовлення (Vosk)
|
||||||
|
settings-vosk-model-desc =
|
||||||
|
Виберіть модель Vosk для розпізнавання мовлення.
|
||||||
|
Ви можете завантажити моделі тут: https://alphacephei.com/vosk/models
|
||||||
|
settings-models-not-found = Моделі не знайдено
|
||||||
|
settings-models-hint = Помістіть моделі Vosk в папку resources/vosk
|
||||||
|
|
||||||
|
# settings - openai
|
||||||
|
settings-openai-key = Ключ OpenAI
|
||||||
|
settings-openai-not-supported = Наразі ChatGPT не підтримується. Він буде доданий у наступних оновленнях.
|
||||||
|
|
||||||
|
# ### COMMANDS PAGE
|
||||||
|
commands-title = Команди
|
||||||
|
commands-search = Пошук команд...
|
||||||
|
commands-count = { $count } команд
|
||||||
|
commands-wip-title = [404] Цей розділ ще в розробці!
|
||||||
|
commands-wip-desc = Тут буде список команд + повноцінний редактор команд.
|
||||||
|
commands-wip-follow = Слідкуйте за оновленнями в
|
||||||
|
commands-wip-channel = нашому телеграм каналі
|
||||||
|
|
||||||
|
# ### ERRORS
|
||||||
|
error-generic = Сталася помилка
|
||||||
|
error-connection = Помилка підключення
|
||||||
|
error-not-found = Не знайдено
|
||||||
|
|
||||||
|
# ### NOTIFICATIONS
|
||||||
|
notification-saved = Налаштування збережено!
|
||||||
|
notification-error = Помилка
|
||||||
|
notification-assistant-started = Асистент запущено
|
||||||
|
notification-assistant-stopped = Асистент зупинено
|
||||||
|
|
||||||
|
# SLOTS EXTRACTION
|
||||||
|
settings-slot-engine = Витяг параметрів
|
||||||
|
settings-slot-engine-desc = Витягує параметри з голосових команд (напр. назва міста, число).
|
||||||
|
settings-gliner-model = Модель GLiNER ONNX
|
||||||
|
settings-gliner-model-desc =
|
||||||
|
Оберіть варіант моделі.
|
||||||
|
Квантизовані моделі (int8, uint8) швидші, але менш точні.
|
||||||
|
settings-gliner-models-hint = Моделі GLiNER не знайдено.
|
||||||
|
|
||||||
|
# ETC
|
||||||
|
search-error-not-running = Асистент не запущено
|
||||||
|
search-error-failed = Не вдалося виконати команду
|
||||||
|
settings-no-voices = Голоси не знайдено
|
||||||
89
crates/jarvis-core/src/intent.rs
Normal file
89
crates/jarvis-core/src/intent.rs
Normal file
|
|
@ -0,0 +1,89 @@
|
||||||
|
mod intentclassifier;
|
||||||
|
mod embeddingclassifier;
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use crate::{commands::{self, JCommandsList, JCommand}, config, models};
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
use crate::DB;
|
||||||
|
|
||||||
|
static BACKEND: OnceCell<String> = OnceCell::new();
|
||||||
|
|
||||||
|
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
|
||||||
|
if BACKEND.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let backend = DB.get().unwrap().read().intent_backend.clone();
|
||||||
|
|
||||||
|
BACKEND.set(backend.clone()).map_err(|_| "Backend already set")?;
|
||||||
|
|
||||||
|
match backend.as_str() {
|
||||||
|
"none" => {
|
||||||
|
info!("Intent recognition disabled");
|
||||||
|
}
|
||||||
|
"intent-classifier" => {
|
||||||
|
info!("Initializing IntentClassifier backend.");
|
||||||
|
intentclassifier::init(&commands).await?;
|
||||||
|
info!("IntentClassifier backend initialized.");
|
||||||
|
}
|
||||||
|
// any other value is treated as a model ID for embedding classification
|
||||||
|
model_id => {
|
||||||
|
info!("Initializing EmbeddingClassifier with model '{}'.", model_id);
|
||||||
|
let model = models::embedding::load(models::registry(), model_id)?;
|
||||||
|
embeddingclassifier::init_with_model(model, &commands)?;
|
||||||
|
info!("EmbeddingClassifier backend initialized.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn classify(text: &str) -> Option<(String, f64)> {
|
||||||
|
match BACKEND.get()?.as_str() {
|
||||||
|
"none" => None,
|
||||||
|
"intent-classifier" => {
|
||||||
|
match intentclassifier::classify(text).await {
|
||||||
|
Ok(prediction) => {
|
||||||
|
let confidence = prediction.confidence.value();
|
||||||
|
if confidence >= config::INTENT_CLASSIFIER_MIN_CONFIDENCE {
|
||||||
|
Some((prediction.intent.to_string(), confidence))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Intent classification error: {}", e);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
match embeddingclassifier::classify(text) {
|
||||||
|
Ok((intent_id, confidence)) => {
|
||||||
|
if confidence >= config::EMBEDDING_MIN_CONFIDENCE {
|
||||||
|
Some((intent_id, confidence))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Embedding classification error: {}", e);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// unified command lookup by intent ID - works for all backends
|
||||||
|
pub fn get_command_by_intent<'a>(
|
||||||
|
commands: &'a [JCommandsList],
|
||||||
|
intent_id: &str,
|
||||||
|
) -> Option<(&'a PathBuf, &'a JCommand)> {
|
||||||
|
if matches!(BACKEND.get().map(|s| s.as_str()), Some("none")) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
commands::get_command_by_id(commands, intent_id)
|
||||||
|
}
|
||||||
195
crates/jarvis-core/src/intent/embeddingclassifier.rs
Normal file
195
crates/jarvis-core/src/intent/embeddingclassifier.rs
Normal file
|
|
@ -0,0 +1,195 @@
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
use crate::commands::JCommandsList;
|
||||||
|
use crate::i18n;
|
||||||
|
use crate::APP_CONFIG_DIR;
|
||||||
|
use crate::models::embedding::EmbeddingModel;
|
||||||
|
|
||||||
|
// no outer Mutex needed - state is immutable after init.
|
||||||
|
// the embedding model has its own internal Mutex.
|
||||||
|
static CLASSIFIER: OnceCell<EmbeddingClassifierState> = OnceCell::new();
|
||||||
|
|
||||||
|
struct IntentVector {
|
||||||
|
id: String,
|
||||||
|
vector: Vec<f32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct EmbeddingClassifierState {
|
||||||
|
model: Arc<EmbeddingModel>,
|
||||||
|
intents: Vec<IntentVector>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// model is Arc (Send+Sync), intents are read-only after init
|
||||||
|
unsafe impl Send for EmbeddingClassifierState {}
|
||||||
|
unsafe impl Sync for EmbeddingClassifierState {}
|
||||||
|
|
||||||
|
const CACHE_FILE: &str = "embedding_intents.json";
|
||||||
|
const HASH_FILE: &str = "embedding_hash.txt";
|
||||||
|
|
||||||
|
// init with a model loaded through the registry
|
||||||
|
pub fn init_with_model(model: Arc<EmbeddingModel>, commands: &[JCommandsList]) -> Result<(), String> {
|
||||||
|
if CLASSIFIER.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Initializing embedding classifier...");
|
||||||
|
|
||||||
|
let current_hash = crate::commands::commands_hash(commands);
|
||||||
|
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
|
||||||
|
let hash_path = config_dir.join(HASH_FILE);
|
||||||
|
let cache_path = config_dir.join(CACHE_FILE);
|
||||||
|
|
||||||
|
// check if cached vectors are still valid
|
||||||
|
let should_retrain = if hash_path.exists() && cache_path.exists() {
|
||||||
|
let stored_hash = fs::read_to_string(&hash_path).unwrap_or_default();
|
||||||
|
stored_hash.trim() != current_hash
|
||||||
|
} else {
|
||||||
|
true
|
||||||
|
};
|
||||||
|
|
||||||
|
let intents = if should_retrain {
|
||||||
|
info!("Building intent vectors from commands...");
|
||||||
|
let intents = build_intent_vectors(&model, commands)?;
|
||||||
|
|
||||||
|
// cache to disk
|
||||||
|
if let Ok(json) = serde_json::to_string(&intents_to_cache(&intents)) {
|
||||||
|
let _ = fs::write(&cache_path, json);
|
||||||
|
let _ = fs::write(&hash_path, ¤t_hash);
|
||||||
|
info!("Intent vectors cached");
|
||||||
|
}
|
||||||
|
|
||||||
|
intents
|
||||||
|
} else {
|
||||||
|
info!("Loading cached intent vectors...");
|
||||||
|
load_cached_intents(&cache_path)?
|
||||||
|
};
|
||||||
|
|
||||||
|
info!("Embedding classifier ready with {} intents", intents.len());
|
||||||
|
|
||||||
|
CLASSIFIER.set(EmbeddingClassifierState { model, intents })
|
||||||
|
.map_err(|_| "Classifier already set".to_string())?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_intent_vectors(
|
||||||
|
model: &EmbeddingModel,
|
||||||
|
commands: &[JCommandsList],
|
||||||
|
) -> Result<Vec<IntentVector>, String> {
|
||||||
|
let lang = i18n::get_language();
|
||||||
|
let mut intents = Vec::new();
|
||||||
|
|
||||||
|
for cmd_list in commands {
|
||||||
|
for cmd in &cmd_list.commands {
|
||||||
|
let phrases = cmd.get_phrases(&lang);
|
||||||
|
if phrases.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let texts: Vec<&str> = phrases.iter().map(|s| s.as_str()).collect();
|
||||||
|
|
||||||
|
let embeddings = model.embedding.lock().embed(texts, None)
|
||||||
|
.map_err(|e| format!("Embedding failed for '{}': {}", cmd.id, e))?;
|
||||||
|
|
||||||
|
// average all phrase vectors into one intent vector
|
||||||
|
let dim = embeddings[0].len();
|
||||||
|
let mut avg = vec![0.0f32; dim];
|
||||||
|
|
||||||
|
for emb in &embeddings {
|
||||||
|
for (i, val) in emb.iter().enumerate() {
|
||||||
|
avg[i] += val;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let count = embeddings.len() as f32;
|
||||||
|
for val in &mut avg {
|
||||||
|
*val /= count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalize
|
||||||
|
let norm: f32 = avg.iter().map(|v| v * v).sum::<f32>().sqrt();
|
||||||
|
if norm > 0.0 {
|
||||||
|
for val in &mut avg {
|
||||||
|
*val /= norm;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
intents.push(IntentVector {
|
||||||
|
id: cmd.id.clone(),
|
||||||
|
vector: avg,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(intents)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn classify(text: &str) -> Result<(String, f64), String> {
|
||||||
|
let state = CLASSIFIER.get().ok_or("Classifier not initialized")?;
|
||||||
|
|
||||||
|
// only the embedding model needs locking, intents are read-only
|
||||||
|
let embeddings = state.model.embedding.lock().embed(vec![text], None)
|
||||||
|
.map_err(|e| format!("Failed to embed query: {}", e))?;
|
||||||
|
|
||||||
|
let mut query_vec = embeddings.into_iter().next()
|
||||||
|
.ok_or("Empty embedding result")?;
|
||||||
|
|
||||||
|
// normalize query
|
||||||
|
let norm: f32 = query_vec.iter().map(|v| v * v).sum::<f32>().sqrt();
|
||||||
|
if norm > 0.0 {
|
||||||
|
for val in &mut query_vec {
|
||||||
|
*val /= norm;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cosine similarity - track index, clone only the winner
|
||||||
|
let mut best_idx: usize = 0;
|
||||||
|
let mut best_score: f64 = -1.0;
|
||||||
|
|
||||||
|
for (i, intent) in state.intents.iter().enumerate() {
|
||||||
|
let score: f64 = query_vec.iter()
|
||||||
|
.zip(intent.vector.iter())
|
||||||
|
.map(|(a, b)| (*a as f64) * (*b as f64))
|
||||||
|
.sum();
|
||||||
|
|
||||||
|
if score > best_score {
|
||||||
|
best_score = score;
|
||||||
|
best_idx = i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let best_id = state.intents[best_idx].id.clone();
|
||||||
|
debug!("Embedding classify: '{}' -> '{}' ({:.2}%)", text, best_id, best_score * 100.0);
|
||||||
|
|
||||||
|
Ok((best_id, best_score))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(serde::Serialize, serde::Deserialize)]
|
||||||
|
struct CachedIntent {
|
||||||
|
id: String,
|
||||||
|
vector: Vec<f32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn intents_to_cache(intents: &[IntentVector]) -> Vec<CachedIntent> {
|
||||||
|
intents.iter().map(|i| CachedIntent {
|
||||||
|
id: i.id.clone(),
|
||||||
|
vector: i.vector.clone(),
|
||||||
|
}).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_cached_intents(path: &PathBuf) -> Result<Vec<IntentVector>, String> {
|
||||||
|
let json = fs::read_to_string(path)
|
||||||
|
.map_err(|e| format!("Failed to read cache: {}", e))?;
|
||||||
|
|
||||||
|
let cached: Vec<CachedIntent> = serde_json::from_str(&json)
|
||||||
|
.map_err(|e| format!("Failed to parse cache: {}", e))?;
|
||||||
|
|
||||||
|
Ok(cached.into_iter().map(|c| IntentVector {
|
||||||
|
id: c.id,
|
||||||
|
vector: c.vector,
|
||||||
|
}).collect())
|
||||||
|
}
|
||||||
96
crates/jarvis-core/src/intent/intentclassifier.rs
Normal file
96
crates/jarvis-core/src/intent/intentclassifier.rs
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
use intent_classifier::{
|
||||||
|
IntentPrediction, IntentError,
|
||||||
|
TrainingExample, TrainingSource, IntentId
|
||||||
|
};
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
use crate::commands::{self, JCommandsList};
|
||||||
|
use crate::models;
|
||||||
|
use crate::models::intent_classifier::IntentClassifierModel;
|
||||||
|
use crate::{APP_CONFIG_DIR, i18n};
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
static MODEL: OnceCell<Arc<IntentClassifierModel>> = OnceCell::new();
|
||||||
|
|
||||||
|
const TRAINING_CACHE_FILE: &str = "intent_training.json";
|
||||||
|
const COMMANDS_HASH_FILE: &str = "commands_hash.txt";
|
||||||
|
|
||||||
|
pub async fn init(commands: &[JCommandsList]) -> Result<(), String> {
|
||||||
|
let current_hash = commands::commands_hash(&commands);
|
||||||
|
|
||||||
|
let model = models::intent_classifier::load(models::registry(), "intent-classifier").await?;
|
||||||
|
|
||||||
|
// check if we can use cached training data
|
||||||
|
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
|
||||||
|
let hash_path = config_dir.join(COMMANDS_HASH_FILE);
|
||||||
|
let cache_path = config_dir.join(TRAINING_CACHE_FILE);
|
||||||
|
|
||||||
|
let should_retrain = if hash_path.exists() && cache_path.exists() {
|
||||||
|
let stored_hash = fs::read_to_string(&hash_path).unwrap_or_default();
|
||||||
|
stored_hash.trim() != current_hash
|
||||||
|
} else {
|
||||||
|
true
|
||||||
|
};
|
||||||
|
|
||||||
|
if should_retrain {
|
||||||
|
info!("Training intent classifier with {} commands...", commands.len());
|
||||||
|
train_classifier(&model.classifier, &commands).await?;
|
||||||
|
|
||||||
|
if let Ok(export) = model.classifier.export_training_data().await {
|
||||||
|
let _ = fs::write(&cache_path, export);
|
||||||
|
let _ = fs::write(&hash_path, ¤t_hash);
|
||||||
|
info!("Training data cached.");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
info!("Loading cached training data...");
|
||||||
|
if let Ok(data) = fs::read_to_string(&cache_path) {
|
||||||
|
model.classifier.import_training_data(&data).await
|
||||||
|
.map_err(|e| format!("Failed to import training data: {}", e))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
MODEL.set(model).map_err(|_| "Model already set")?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn classify(text: &str) -> Result<IntentPrediction, IntentError> {
|
||||||
|
let model = MODEL.get().expect("IntentClassifier not initialized");
|
||||||
|
model.classifier.predict_intent(text).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn train_classifier(
|
||||||
|
classifier: &intent_classifier::IntentClassifier,
|
||||||
|
commands: &[JCommandsList]
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let lang = i18n::get_language();
|
||||||
|
info!("Training intent classifier for language: {}", lang);
|
||||||
|
|
||||||
|
let mut total_examples = 0;
|
||||||
|
|
||||||
|
for assistant_cmd in commands {
|
||||||
|
for cmd in &assistant_cmd.commands {
|
||||||
|
let phrases = cmd.get_phrases(&lang);
|
||||||
|
|
||||||
|
for phrase in phrases.iter() {
|
||||||
|
let example = TrainingExample {
|
||||||
|
text: phrase.clone(),
|
||||||
|
intent: IntentId::from(cmd.id.as_str()),
|
||||||
|
confidence: 1.0,
|
||||||
|
source: TrainingSource::Programmatic,
|
||||||
|
};
|
||||||
|
|
||||||
|
classifier.add_training_example(example).await
|
||||||
|
.map_err(|e| format!("Failed to add training example: {}", e))?;
|
||||||
|
|
||||||
|
total_examples += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Added {} training examples for language '{}'", total_examples, lang);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
5
crates/jarvis-core/src/ipc.rs
Normal file
5
crates/jarvis-core/src/ipc.rs
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
mod events;
|
||||||
|
mod server;
|
||||||
|
|
||||||
|
pub use events::{IpcAction, IpcEvent};
|
||||||
|
pub use server::{init, send, set_action_handler, start_server, has_clients, IPC_ADDR, IPC_PORT};
|
||||||
59
crates/jarvis-core/src/ipc/events.rs
Normal file
59
crates/jarvis-core/src/ipc/events.rs
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
// Events sent from jarvis-app to GUI
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
#[serde(tag = "event", rename_all = "snake_case")]
|
||||||
|
pub enum IpcEvent {
|
||||||
|
// Wake word detected, starting to listen
|
||||||
|
WakeWordDetected,
|
||||||
|
|
||||||
|
// Actively listening for command
|
||||||
|
Listening,
|
||||||
|
|
||||||
|
// Speech recognized
|
||||||
|
SpeechRecognized { text: String },
|
||||||
|
|
||||||
|
// LLM produced a reply for a free-form question
|
||||||
|
LlmReply { text: String },
|
||||||
|
|
||||||
|
// Command was executed
|
||||||
|
CommandExecuted { id: String, success: bool },
|
||||||
|
|
||||||
|
// Returned to idle state
|
||||||
|
Idle,
|
||||||
|
|
||||||
|
// Error occurred
|
||||||
|
Error { message: String },
|
||||||
|
|
||||||
|
// App started
|
||||||
|
Started,
|
||||||
|
|
||||||
|
// App is shutting down
|
||||||
|
Stopping,
|
||||||
|
|
||||||
|
// Pong response
|
||||||
|
Pong,
|
||||||
|
|
||||||
|
// request GUI to reveal/focus window
|
||||||
|
RevealWindow,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Actions sent from GUI to jarvis-app
|
||||||
|
#[derive(Clone, Debug, Deserialize)]
|
||||||
|
#[serde(tag = "action", rename_all = "snake_case")]
|
||||||
|
pub enum IpcAction {
|
||||||
|
// Request graceful shutdown
|
||||||
|
Stop,
|
||||||
|
|
||||||
|
// Reload commands from disk
|
||||||
|
ReloadCommands,
|
||||||
|
|
||||||
|
// Ping to check connection
|
||||||
|
Ping,
|
||||||
|
|
||||||
|
// Mute/unmute listening
|
||||||
|
SetMuted { muted: bool },
|
||||||
|
|
||||||
|
// Execute text command
|
||||||
|
TextCommand { text: String },
|
||||||
|
}
|
||||||
198
crates/jarvis-core/src/ipc/server.rs
Normal file
198
crates/jarvis-core/src/ipc/server.rs
Normal file
|
|
@ -0,0 +1,198 @@
|
||||||
|
use std::net::SocketAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use futures_util::{SinkExt, StreamExt};
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use parking_lot::RwLock;
|
||||||
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
use tokio_tungstenite::{accept_async, tungstenite::Message};
|
||||||
|
|
||||||
|
use super::events::{IpcAction, IpcEvent};
|
||||||
|
|
||||||
|
pub const IPC_PORT: u16 = 9712;
|
||||||
|
pub const IPC_ADDR: &str = "127.0.0.1";
|
||||||
|
|
||||||
|
static BROADCAST_TX: OnceCell<broadcast::Sender<IpcEvent>> = OnceCell::new();
|
||||||
|
static ACTION_HANDLER: OnceCell<Arc<RwLock<Option<Box<dyn Fn(IpcAction) + Send + Sync>>>>> = OnceCell::new();
|
||||||
|
|
||||||
|
// Initialize the IPC broadcast channel
|
||||||
|
pub fn init() -> broadcast::Sender<IpcEvent> {
|
||||||
|
if let Some(tx) = BROADCAST_TX.get() {
|
||||||
|
return tx.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
let (tx, _) = broadcast::channel::<IpcEvent>(32);
|
||||||
|
BROADCAST_TX.set(tx.clone()).ok();
|
||||||
|
ACTION_HANDLER.set(Arc::new(RwLock::new(None))).ok();
|
||||||
|
|
||||||
|
info!("IPC: Broadcast channel initialized");
|
||||||
|
tx
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send event to all connected clients
|
||||||
|
pub fn send(event: IpcEvent) {
|
||||||
|
if let Some(tx) = BROADCAST_TX.get() {
|
||||||
|
match tx.send(event.clone()) {
|
||||||
|
Ok(n) => {
|
||||||
|
if n > 0 {
|
||||||
|
debug!("IPC: Sent {:?} to {} client(s)", event, n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// no receivers, that's fine
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register handler for incoming actions from GUI
|
||||||
|
pub fn set_action_handler<F>(handler: F)
|
||||||
|
where
|
||||||
|
F: Fn(IpcAction) + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
if let Some(h) = ACTION_HANDLER.get() {
|
||||||
|
*h.write() = Some(Box::new(handler));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle_action(action: IpcAction) {
|
||||||
|
info!("IPC: Received action {:?}", action);
|
||||||
|
|
||||||
|
// handle ping internally
|
||||||
|
if matches!(action, IpcAction::Ping) {
|
||||||
|
send(IpcEvent::Pong);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// forward to registered handler
|
||||||
|
if let Some(handler_lock) = ACTION_HANDLER.get() {
|
||||||
|
let handler = handler_lock.read();
|
||||||
|
if let Some(ref h) = *handler {
|
||||||
|
h(action);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start the WebSocket server (blocking)
|
||||||
|
pub async fn start_server() {
|
||||||
|
let addr = format!("{}:{}", IPC_ADDR, IPC_PORT);
|
||||||
|
let socket_addr: SocketAddr = addr.parse().expect("Invalid IPC address");
|
||||||
|
|
||||||
|
let listener = match TcpListener::bind(&socket_addr).await {
|
||||||
|
Ok(l) => {
|
||||||
|
info!("IPC: WebSocket server listening on ws://{}", addr);
|
||||||
|
l
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("IPC: Failed to bind to {}: {}", addr, e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// notify that we're ready
|
||||||
|
send(IpcEvent::Started);
|
||||||
|
|
||||||
|
while let Ok((stream, peer_addr)) = listener.accept().await {
|
||||||
|
info!("IPC: Client connecting from {}", peer_addr);
|
||||||
|
|
||||||
|
let rx = BROADCAST_TX
|
||||||
|
.get()
|
||||||
|
.map(|tx| tx.subscribe())
|
||||||
|
.expect("IPC not initialized");
|
||||||
|
|
||||||
|
tokio::spawn(handle_client(stream, peer_addr, rx));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_client(
|
||||||
|
stream: TcpStream,
|
||||||
|
peer_addr: SocketAddr,
|
||||||
|
mut event_rx: broadcast::Receiver<IpcEvent>,
|
||||||
|
) {
|
||||||
|
let ws_stream = match accept_async(stream).await {
|
||||||
|
Ok(ws) => {
|
||||||
|
info!("IPC: Client connected: {}", peer_addr);
|
||||||
|
ws
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("IPC: WebSocket handshake failed for {}: {}", peer_addr, e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let (mut ws_tx, mut ws_rx) = ws_stream.split();
|
||||||
|
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
// forward events to client
|
||||||
|
event_result = event_rx.recv() => {
|
||||||
|
match event_result {
|
||||||
|
Ok(event) => {
|
||||||
|
let json = match serde_json::to_string(&event) {
|
||||||
|
Ok(j) => j,
|
||||||
|
Err(e) => {
|
||||||
|
error!("IPC: Failed to serialize event: {}", e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if ws_tx.send(Message::Text(json.into())).await.is_err() {
|
||||||
|
info!("IPC: Client {} disconnected (send failed)", peer_addr);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||||
|
warn!("IPC: Client {} lagged {} events", peer_addr, n);
|
||||||
|
}
|
||||||
|
Err(broadcast::error::RecvError::Closed) => {
|
||||||
|
info!("IPC: Broadcast channel closed");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// receive messages from client
|
||||||
|
msg_result = ws_rx.next() => {
|
||||||
|
match msg_result {
|
||||||
|
Some(Ok(Message::Text(text))) => {
|
||||||
|
match serde_json::from_str::<IpcAction>(&text) {
|
||||||
|
Ok(action) => handle_action(action),
|
||||||
|
Err(e) => {
|
||||||
|
warn!("IPC: Invalid action from {}: {} ({})", peer_addr, text, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(Ok(Message::Ping(data))) => {
|
||||||
|
if ws_tx.send(Message::Pong(data)).await.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(Ok(Message::Close(_))) => {
|
||||||
|
info!("IPC: Client {} sent close frame", peer_addr);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Some(Err(e)) => {
|
||||||
|
error!("IPC: Error receiving from {}: {}", peer_addr, e);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
info!("IPC: Client {} stream ended", peer_addr);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("IPC: Client disconnected: {}", peer_addr);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_clients() -> bool {
|
||||||
|
if let Some(tx) = BROADCAST_TX.get() {
|
||||||
|
tx.receiver_count() > 0
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
75
crates/jarvis-core/src/lib.rs
Normal file
75
crates/jarvis-core/src/lib.rs
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
use once_cell::sync::{Lazy, OnceCell};
|
||||||
|
use parking_lot::RwLock;
|
||||||
|
use std::{sync::Arc};
|
||||||
|
use platform_dirs::AppDirs;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
#[macro_use]
|
||||||
|
extern crate log;
|
||||||
|
|
||||||
|
pub mod time;
|
||||||
|
|
||||||
|
pub mod audio;
|
||||||
|
pub mod commands;
|
||||||
|
pub mod config;
|
||||||
|
pub mod db;
|
||||||
|
pub mod i18n;
|
||||||
|
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod listener;
|
||||||
|
|
||||||
|
pub mod recorder;
|
||||||
|
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod stt;
|
||||||
|
|
||||||
|
#[cfg(feature = "intent")]
|
||||||
|
pub mod intent;
|
||||||
|
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod slots;
|
||||||
|
|
||||||
|
pub mod models;
|
||||||
|
|
||||||
|
// re-exported from models/
|
||||||
|
pub use models::vosk_models;
|
||||||
|
pub use models::gliner_models;
|
||||||
|
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod audio_processing;
|
||||||
|
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod ipc;
|
||||||
|
|
||||||
|
pub mod voices;
|
||||||
|
|
||||||
|
pub mod audio_buffer;
|
||||||
|
|
||||||
|
#[cfg(feature = "lua")]
|
||||||
|
pub mod lua;
|
||||||
|
|
||||||
|
#[cfg(feature = "llm")]
|
||||||
|
pub mod llm;
|
||||||
|
|
||||||
|
// shared statics
|
||||||
|
// pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| std::env::current_dir().unwrap());
|
||||||
|
pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| {
|
||||||
|
std::env::current_exe()
|
||||||
|
.ok()
|
||||||
|
.and_then(|p| p.parent().map(|p| p.to_path_buf()))
|
||||||
|
.unwrap_or_else(|| std::env::current_dir().unwrap())
|
||||||
|
});
|
||||||
|
pub static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| APP_DIR.clone().join(config::SOUND_PATH));
|
||||||
|
pub static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
|
||||||
|
pub static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
|
||||||
|
pub static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
|
||||||
|
pub static DB: OnceCell<Arc<RwLock<db::structs::Settings>>> = OnceCell::new();
|
||||||
|
pub static COMMANDS_LIST: OnceCell<Vec<JCommandsList>> = OnceCell::new();
|
||||||
|
|
||||||
|
// re-exports
|
||||||
|
pub use commands::JCommandsList;
|
||||||
|
pub use config::structs::*;
|
||||||
|
pub use db::structs::Settings;
|
||||||
|
pub use db::SettingsManager;
|
||||||
|
|
||||||
|
// use crate::commands::{JComandsList, JCommand};
|
||||||
46
crates/jarvis-core/src/listener.rs
Normal file
46
crates/jarvis-core/src/listener.rs
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
mod rustpotter;
|
||||||
|
mod vosk;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
use crate::config::structs::WakeWordEngine;
|
||||||
|
|
||||||
|
use crate::DB;
|
||||||
|
|
||||||
|
static WAKE_WORD_ENGINE: OnceCell<WakeWordEngine> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), String> {
|
||||||
|
if WAKE_WORD_ENGINE.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let engine = DB.get().unwrap().read().wake_word_engine;
|
||||||
|
|
||||||
|
WAKE_WORD_ENGINE.set(engine)
|
||||||
|
.map_err(|_| "Wake word engine already set".to_string())?;
|
||||||
|
|
||||||
|
match engine {
|
||||||
|
WakeWordEngine::Porcupine => {
|
||||||
|
Err("Porcupine wake-word engine is not supported".to_string())
|
||||||
|
}
|
||||||
|
WakeWordEngine::Rustpotter => {
|
||||||
|
info!("Initializing Rustpotter wake-word engine.");
|
||||||
|
rustpotter::init()
|
||||||
|
.map_err(|_| "Failed to init Rustpotter".to_string())
|
||||||
|
}
|
||||||
|
WakeWordEngine::Vosk => {
|
||||||
|
info!("Initializing Vosk as wake-word engine.");
|
||||||
|
warn!("Using Vosk as wake-word engine is highly not recommended, because it's very slow for this task.");
|
||||||
|
vosk::init()
|
||||||
|
.map_err(|_| "Failed to init Vosk wake-word".to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
|
||||||
|
match WAKE_WORD_ENGINE.get()? {
|
||||||
|
WakeWordEngine::Porcupine => None,
|
||||||
|
WakeWordEngine::Rustpotter => rustpotter::data_callback(frame_buffer),
|
||||||
|
WakeWordEngine::Vosk => vosk::data_callback(frame_buffer),
|
||||||
|
}
|
||||||
|
}
|
||||||
56
crates/jarvis-core/src/listener/porcupine.rs
Normal file
56
crates/jarvis-core/src/listener/porcupine.rs
Normal file
|
|
@ -0,0 +1,56 @@
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use porcupine::{Porcupine, PorcupineBuilder};
|
||||||
|
|
||||||
|
use crate::config;
|
||||||
|
use crate::DB;
|
||||||
|
|
||||||
|
// store porcupine instance
|
||||||
|
static PORCUPINE: OnceCell<Porcupine> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), ()> {
|
||||||
|
let picovoice_api_key: String;
|
||||||
|
|
||||||
|
// retrieve picovoice api key
|
||||||
|
picovoice_api_key = DB.get().unwrap().api_keys.picovoice.clone();
|
||||||
|
if picovoice_api_key.trim().is_empty() {
|
||||||
|
warn!("Picovoice API key is not set.");
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// create porcupine instance with the given API key
|
||||||
|
match PorcupineBuilder::new_with_keyword_paths(
|
||||||
|
picovoice_api_key,
|
||||||
|
&[Path::new(config::KEYWORDS_PATH).join(config::DEFAULT_KEYWORD)],
|
||||||
|
)
|
||||||
|
.sensitivities(&[config::DEFAULT_SENSITIVITY]) // set sensitivity
|
||||||
|
.init()
|
||||||
|
{
|
||||||
|
Ok(pinstance) => {
|
||||||
|
// success
|
||||||
|
info!("Porcupine successfully initialized with the given API key.");
|
||||||
|
|
||||||
|
// store
|
||||||
|
PORCUPINE.set(pinstance);
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Porcupine failed to initialize, either API key is not valid or there is no internet connection.");
|
||||||
|
error!("Error details: {}", msg);
|
||||||
|
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
|
||||||
|
if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) {
|
||||||
|
if keyword_index >= 0 {
|
||||||
|
return Some(keyword_index);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
69
crates/jarvis-core/src/listener/rustpotter.rs
Normal file
69
crates/jarvis-core/src/listener/rustpotter.rs
Normal file
|
|
@ -0,0 +1,69 @@
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use rustpotter::Rustpotter;
|
||||||
|
|
||||||
|
use crate::config;
|
||||||
|
|
||||||
|
// store rustpotter instance
|
||||||
|
static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), ()> {
|
||||||
|
let rustpotter_config = config::RUSTPOTTER_DEFAULT_CONFIG;
|
||||||
|
|
||||||
|
// create rustpotter instance
|
||||||
|
match Rustpotter::new(&rustpotter_config) {
|
||||||
|
Ok(mut rinstance) => {
|
||||||
|
// success
|
||||||
|
// wake word files list
|
||||||
|
// @TODO. Make it configurable via GUI for custom user voice.
|
||||||
|
let rustpotter_wake_word_files: [&str; 1] = [
|
||||||
|
"resources/rustpotter/jarvis-default.rpw",
|
||||||
|
// "rustpotter/jarvis-community-1.rpw",
|
||||||
|
// "rustpotter/jarvis-community-2.rpw",
|
||||||
|
// "rustpotter/jarvis-community-3.rpw",
|
||||||
|
// "rustpotter/jarvis-community-4.rpw",
|
||||||
|
// "rustpotter/jarvis-community-5.rpw",
|
||||||
|
];
|
||||||
|
|
||||||
|
// load wake word files
|
||||||
|
for rpw in rustpotter_wake_word_files {
|
||||||
|
// @TODO: Change wakeword key to something else?
|
||||||
|
if let Err(e) = rinstance.add_wakeword_from_file(rpw, rpw) {
|
||||||
|
error!("Failed to load wakeword file '{}': {}", rpw, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// store
|
||||||
|
let _ = RUSTPOTTER.set(Mutex::new(rinstance));
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Rustpotter failed to initialize.\nError details: {}", msg);
|
||||||
|
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
|
||||||
|
let mut lock = RUSTPOTTER.get().unwrap().lock();
|
||||||
|
let rustpotter = lock.as_mut().unwrap();
|
||||||
|
// let detection = rustpotter.process_samples(frame_buffer.to_vec()); // @TODO. Temp crutch. Fix optimization issue, frame_buffer should not be copied to a new vector!
|
||||||
|
let detection = rustpotter.process_samples(frame_buffer);
|
||||||
|
|
||||||
|
// info!("Ruspotter data callback");
|
||||||
|
|
||||||
|
if let Some(detection) = detection {
|
||||||
|
if detection.score > config::RUSPOTTER_MIN_SCORE {
|
||||||
|
info!("Rustpotter detection info:\n{:?}", detection);
|
||||||
|
|
||||||
|
return Some(0);
|
||||||
|
} else {
|
||||||
|
info!("Rustpotter detection info:\n{:?}", detection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
76
crates/jarvis-core/src/listener/vosk.rs
Normal file
76
crates/jarvis-core/src/listener/vosk.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
use crate::{config, stt, i18n};
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), ()> {
|
||||||
|
Ok(()) // nothing to init for Vosk
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
|
||||||
|
if let Some((recognized, _confidence)) = stt::recognize_wake_word(frame_buffer) {
|
||||||
|
let recognized = recognized.trim().to_lowercase();
|
||||||
|
|
||||||
|
// skip unknown/empty
|
||||||
|
if recognized.is_empty() || recognized == "[unk]" {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Wake word candidate: '{}'", recognized);
|
||||||
|
|
||||||
|
// language-specific wake phrase
|
||||||
|
let lang = i18n::get_language();
|
||||||
|
let wake_phrases = config::get_wake_phrases(&lang);
|
||||||
|
|
||||||
|
// verify with seqdiff ratio
|
||||||
|
for word in recognized.split_whitespace() {
|
||||||
|
if word == "[unk]" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let word_chars: Vec<char> = word.chars().collect();
|
||||||
|
|
||||||
|
for wake_phrase in wake_phrases {
|
||||||
|
let wake_chars: Vec<char> = wake_phrase.chars().collect();
|
||||||
|
let similarity = seqdiff::ratio(&wake_chars, &word_chars);
|
||||||
|
|
||||||
|
if similarity >= config::VOSK_MIN_RATIO {
|
||||||
|
info!("Wake word match: '{}' ~ '{}' ({:.1}%)", word, wake_phrase, similarity);
|
||||||
|
return Some(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// info!("Similarity: {:.1}% ('{}' vs '{}')", similarity, recognized, config::VOSK_FETCH_PHRASE);
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
// @TODO. Make it better somehow (more accurate or with higher sensitivity).
|
||||||
|
// pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
|
||||||
|
// // recognize & convert to sequence
|
||||||
|
// let recognized_phrase = stt::recognize(&frame_buffer, true).unwrap_or("".into());
|
||||||
|
|
||||||
|
// if !recognized_phrase.trim().is_empty() {
|
||||||
|
// info!("Vosk wake-word debug info:");
|
||||||
|
// info!("rec: {}", recognized_phrase);
|
||||||
|
// let recognized_phrases = recognized_phrase.split_whitespace();
|
||||||
|
// for phrase in recognized_phrases {
|
||||||
|
// let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
|
||||||
|
|
||||||
|
// // compare
|
||||||
|
// let compare_ratio = seqdiff::ratio(
|
||||||
|
// &config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(),
|
||||||
|
// &recognized_phrase_chars,
|
||||||
|
// );
|
||||||
|
// info!("og phrase: {:?}", &config::VOSK_FETCH_PHRASE);
|
||||||
|
// info!("recognized phrase: {:?}", &recognized_phrase_chars);
|
||||||
|
// info!("compare ratio: {}", compare_ratio);
|
||||||
|
|
||||||
|
// if compare_ratio >= config::VOSK_MIN_RATIO {
|
||||||
|
// info!("Phrase activated.");
|
||||||
|
// return Some(0);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// None
|
||||||
|
// }
|
||||||
188
crates/jarvis-core/src/llm/client.rs
Normal file
188
crates/jarvis-core/src/llm/client.rs
Normal file
|
|
@ -0,0 +1,188 @@
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::env;
|
||||||
|
use thiserror::Error;
|
||||||
|
|
||||||
|
pub const DEFAULT_BASE_URL: &str = "https://api.groq.com/openai/v1";
|
||||||
|
pub const DEFAULT_MODEL: &str = "llama-3.3-70b-versatile";
|
||||||
|
pub const ENV_TOKEN: &str = "GROQ_TOKEN";
|
||||||
|
pub const ENV_BASE_URL: &str = "GROQ_BASE_URL";
|
||||||
|
pub const ENV_MODEL: &str = "GROQ_MODEL";
|
||||||
|
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
pub enum ConfigError {
|
||||||
|
#[error("missing environment variable: {0}")]
|
||||||
|
MissingEnv(&'static str),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Error)]
|
||||||
|
pub enum LlmError {
|
||||||
|
#[error("HTTP request failed: {0}")]
|
||||||
|
Http(#[from] reqwest::Error),
|
||||||
|
#[error("failed to parse response JSON: {0}")]
|
||||||
|
Deserialize(String),
|
||||||
|
#[error("API returned error (status {status}): {body}")]
|
||||||
|
Api { status: u16, body: String },
|
||||||
|
#[error("response contained no choices")]
|
||||||
|
EmptyResponse,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ChatMessage {
|
||||||
|
pub role: String,
|
||||||
|
pub content: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChatMessage {
|
||||||
|
pub fn user(content: impl Into<String>) -> Self {
|
||||||
|
Self { role: "user".to_string(), content: content.into() }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn system(content: impl Into<String>) -> Self {
|
||||||
|
Self { role: "system".to_string(), content: content.into() }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn assistant(content: impl Into<String>) -> Self {
|
||||||
|
Self { role: "assistant".to_string(), content: content.into() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
struct ChatRequest<'a> {
|
||||||
|
model: &'a str,
|
||||||
|
messages: &'a [ChatMessage],
|
||||||
|
max_tokens: u32,
|
||||||
|
temperature: f32,
|
||||||
|
top_p: f32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct ChatResponse {
|
||||||
|
choices: Vec<Choice>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct Choice {
|
||||||
|
message: ResponseMessage,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct ResponseMessage {
|
||||||
|
content: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct LlmClient {
|
||||||
|
base_url: String,
|
||||||
|
api_key: String,
|
||||||
|
model: String,
|
||||||
|
http: reqwest::blocking::Client,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LlmClient {
|
||||||
|
pub fn new(
|
||||||
|
base_url: impl Into<String>,
|
||||||
|
api_key: impl Into<String>,
|
||||||
|
model: impl Into<String>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
base_url: base_url.into(),
|
||||||
|
api_key: api_key.into(),
|
||||||
|
model: model.into(),
|
||||||
|
http: reqwest::blocking::Client::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_env() -> Result<Self, ConfigError> {
|
||||||
|
let api_key = env::var(ENV_TOKEN).map_err(|_| ConfigError::MissingEnv(ENV_TOKEN))?;
|
||||||
|
let base_url = env::var(ENV_BASE_URL).unwrap_or_else(|_| DEFAULT_BASE_URL.to_string());
|
||||||
|
let model = env::var(ENV_MODEL).unwrap_or_else(|_| DEFAULT_MODEL.to_string());
|
||||||
|
Ok(Self::new(base_url, api_key, model))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn model(&self) -> &str {
|
||||||
|
&self.model
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn complete(&self, messages: &[ChatMessage], max_tokens: u32) -> Result<String, LlmError> {
|
||||||
|
let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/'));
|
||||||
|
let body = ChatRequest {
|
||||||
|
model: &self.model,
|
||||||
|
messages,
|
||||||
|
max_tokens,
|
||||||
|
temperature: 0.7,
|
||||||
|
top_p: 1.0,
|
||||||
|
};
|
||||||
|
|
||||||
|
let resp = self
|
||||||
|
.http
|
||||||
|
.post(&url)
|
||||||
|
.bearer_auth(&self.api_key)
|
||||||
|
.json(&body)
|
||||||
|
.send()?;
|
||||||
|
|
||||||
|
let status = resp.status();
|
||||||
|
let text = resp.text()?;
|
||||||
|
|
||||||
|
if !status.is_success() {
|
||||||
|
return Err(LlmError::Api { status: status.as_u16(), body: text });
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed: ChatResponse =
|
||||||
|
serde_json::from_str(&text).map_err(|e| LlmError::Deserialize(e.to_string()))?;
|
||||||
|
parsed
|
||||||
|
.choices
|
||||||
|
.into_iter()
|
||||||
|
.next()
|
||||||
|
.map(|c| c.message.content)
|
||||||
|
.ok_or(LlmError::EmptyResponse)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parses_valid_chat_response() {
|
||||||
|
let raw = r#"{
|
||||||
|
"id": "chatcmpl-xyz",
|
||||||
|
"object": "chat.completion",
|
||||||
|
"model": "llama-3.3-70b-versatile",
|
||||||
|
"choices": [
|
||||||
|
{
|
||||||
|
"index": 0,
|
||||||
|
"message": { "role": "assistant", "content": "Hello there!" },
|
||||||
|
"finish_reason": "stop"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"usage": { "prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8 }
|
||||||
|
}"#;
|
||||||
|
|
||||||
|
let parsed: ChatResponse = serde_json::from_str(raw).expect("valid JSON should parse");
|
||||||
|
assert_eq!(parsed.choices.len(), 1);
|
||||||
|
assert_eq!(parsed.choices[0].message.content, "Hello there!");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn from_env_fails_when_token_missing() {
|
||||||
|
let prev = env::var(ENV_TOKEN).ok();
|
||||||
|
unsafe { env::remove_var(ENV_TOKEN); }
|
||||||
|
let result = LlmClient::from_env();
|
||||||
|
let err = match result {
|
||||||
|
Ok(_) => panic!("expected ConfigError when {ENV_TOKEN} is unset"),
|
||||||
|
Err(e) => e,
|
||||||
|
};
|
||||||
|
match err {
|
||||||
|
ConfigError::MissingEnv(name) => assert_eq!(name, ENV_TOKEN),
|
||||||
|
}
|
||||||
|
if let Some(v) = prev {
|
||||||
|
unsafe { env::set_var(ENV_TOKEN, v); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chat_message_helpers_set_role() {
|
||||||
|
assert_eq!(ChatMessage::user("hi").role, "user");
|
||||||
|
assert_eq!(ChatMessage::system("ctx").role, "system");
|
||||||
|
assert_eq!(ChatMessage::assistant("ok").role, "assistant");
|
||||||
|
}
|
||||||
|
}
|
||||||
138
crates/jarvis-core/src/llm/history.rs
Normal file
138
crates/jarvis-core/src/llm/history.rs
Normal file
|
|
@ -0,0 +1,138 @@
|
||||||
|
use super::client::ChatMessage;
|
||||||
|
|
||||||
|
pub struct ConversationHistory {
|
||||||
|
system: Option<ChatMessage>,
|
||||||
|
turns: Vec<ChatMessage>,
|
||||||
|
max_turns: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ConversationHistory {
|
||||||
|
pub fn new(system_prompt: impl Into<String>, max_turns: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
system: Some(ChatMessage::system(system_prompt)),
|
||||||
|
turns: Vec::new(),
|
||||||
|
max_turns: max_turns.max(1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn without_system(max_turns: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
system: None,
|
||||||
|
turns: Vec::new(),
|
||||||
|
max_turns: max_turns.max(1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn push_user(&mut self, content: impl Into<String>) {
|
||||||
|
self.turns.push(ChatMessage::user(content));
|
||||||
|
self.truncate();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn push_assistant(&mut self, content: impl Into<String>) {
|
||||||
|
self.turns.push(ChatMessage::assistant(content));
|
||||||
|
self.truncate();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn snapshot(&self) -> Vec<ChatMessage> {
|
||||||
|
let mut out = Vec::with_capacity(self.turns.len() + 1);
|
||||||
|
if let Some(s) = &self.system {
|
||||||
|
out.push(s.clone());
|
||||||
|
}
|
||||||
|
out.extend(self.turns.iter().cloned());
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn turns(&self) -> &[ChatMessage] {
|
||||||
|
&self.turns
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn clear(&mut self) {
|
||||||
|
self.turns.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn pop_last_user(&mut self) -> Option<ChatMessage> {
|
||||||
|
if matches!(self.turns.last(), Some(m) if m.role == "user") {
|
||||||
|
self.turns.pop()
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate(&mut self) {
|
||||||
|
if self.turns.len() > self.max_turns {
|
||||||
|
let drop = self.turns.len() - self.max_turns;
|
||||||
|
self.turns.drain(0..drop);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn evicts_oldest_turns_but_keeps_system_prompt() {
|
||||||
|
let mut h = ConversationHistory::new("you are jarvis", 4);
|
||||||
|
h.push_user("u1");
|
||||||
|
h.push_assistant("a1");
|
||||||
|
h.push_user("u2");
|
||||||
|
h.push_assistant("a2");
|
||||||
|
h.push_user("u3");
|
||||||
|
|
||||||
|
let snap = h.snapshot();
|
||||||
|
assert_eq!(snap[0].role, "system");
|
||||||
|
assert_eq!(snap[0].content, "you are jarvis");
|
||||||
|
assert_eq!(snap.len(), 1 + 4);
|
||||||
|
|
||||||
|
let contents: Vec<&str> = snap.iter().skip(1).map(|m| m.content.as_str()).collect();
|
||||||
|
assert_eq!(contents, vec!["a1", "u2", "a2", "u3"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn snapshot_with_no_system_returns_only_turns() {
|
||||||
|
let mut h = ConversationHistory::without_system(3);
|
||||||
|
h.push_user("hi");
|
||||||
|
h.push_assistant("hello");
|
||||||
|
let snap = h.snapshot();
|
||||||
|
assert_eq!(snap.len(), 2);
|
||||||
|
assert_eq!(snap[0].role, "user");
|
||||||
|
assert_eq!(snap[1].role, "assistant");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cap_of_zero_is_clamped_to_one() {
|
||||||
|
let mut h = ConversationHistory::new("sys", 0);
|
||||||
|
h.push_user("a");
|
||||||
|
h.push_user("b");
|
||||||
|
let snap = h.snapshot();
|
||||||
|
assert_eq!(snap.len(), 2);
|
||||||
|
assert_eq!(snap[0].role, "system");
|
||||||
|
assert_eq!(snap[1].content, "b");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pop_last_user_only_pops_a_trailing_user_turn() {
|
||||||
|
let mut h = ConversationHistory::new("sys", 4);
|
||||||
|
h.push_user("u1");
|
||||||
|
h.push_assistant("a1");
|
||||||
|
assert!(h.pop_last_user().is_none());
|
||||||
|
h.push_user("u2");
|
||||||
|
let popped = h.pop_last_user().expect("trailing user turn should pop");
|
||||||
|
assert_eq!(popped.role, "user");
|
||||||
|
assert_eq!(popped.content, "u2");
|
||||||
|
let snap = h.snapshot();
|
||||||
|
assert_eq!(snap.len(), 1 + 2);
|
||||||
|
assert_eq!(snap[2].content, "a1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn clear_removes_turns_but_not_system() {
|
||||||
|
let mut h = ConversationHistory::new("sys", 4);
|
||||||
|
h.push_user("u");
|
||||||
|
h.push_assistant("a");
|
||||||
|
h.clear();
|
||||||
|
let snap = h.snapshot();
|
||||||
|
assert_eq!(snap.len(), 1);
|
||||||
|
assert_eq!(snap[0].role, "system");
|
||||||
|
}
|
||||||
|
}
|
||||||
8
crates/jarvis-core/src/llm/mod.rs
Normal file
8
crates/jarvis-core/src/llm/mod.rs
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
mod client;
|
||||||
|
mod history;
|
||||||
|
|
||||||
|
pub use client::{
|
||||||
|
ChatMessage, ConfigError, LlmClient, LlmError,
|
||||||
|
DEFAULT_BASE_URL, DEFAULT_MODEL, ENV_BASE_URL, ENV_MODEL, ENV_TOKEN,
|
||||||
|
};
|
||||||
|
pub use history::ConversationHistory;
|
||||||
28
crates/jarvis-core/src/lua.rs
Normal file
28
crates/jarvis-core/src/lua.rs
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
mod engine;
|
||||||
|
mod sandbox;
|
||||||
|
mod error;
|
||||||
|
mod api;
|
||||||
|
|
||||||
|
mod structs;
|
||||||
|
pub use structs::*;
|
||||||
|
|
||||||
|
pub use engine::LuaEngine;
|
||||||
|
pub use sandbox::SandboxLevel;
|
||||||
|
pub use error::LuaError;
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
// Execute a Lua command script
|
||||||
|
pub fn execute(
|
||||||
|
script_path: &PathBuf,
|
||||||
|
context: CommandContext,
|
||||||
|
sandbox: SandboxLevel,
|
||||||
|
timeout: Duration,
|
||||||
|
) -> Result<CommandResult, LuaError> {
|
||||||
|
let engine = LuaEngine::new(sandbox)?;
|
||||||
|
engine.execute(script_path, context, timeout)
|
||||||
|
}
|
||||||
7
crates/jarvis-core/src/lua/api.rs
Normal file
7
crates/jarvis-core/src/lua/api.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
pub mod core;
|
||||||
|
pub mod audio;
|
||||||
|
pub mod context;
|
||||||
|
pub mod http;
|
||||||
|
pub mod fs;
|
||||||
|
pub mod state;
|
||||||
|
pub mod system;
|
||||||
37
crates/jarvis-core/src/lua/api/_new_api_module_example.rs
Normal file
37
crates/jarvis-core/src/lua/api/_new_api_module_example.rs
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
// QUICK HELP ON HOW TO ADD NEW LUA API MODULE
|
||||||
|
//
|
||||||
|
// # 1. DEFINE NEW API MODULE FILE
|
||||||
|
|
||||||
|
use mlua::{Lua, Table};
|
||||||
|
use crate::lua::error::LuaError;
|
||||||
|
|
||||||
|
pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> {
|
||||||
|
let mymodule = lua.create_table()?;
|
||||||
|
|
||||||
|
// add functions
|
||||||
|
let my_fn = lua.create_function(|_, arg: String| {
|
||||||
|
// implementation
|
||||||
|
Ok(format!("Result: {}", arg))
|
||||||
|
})?;
|
||||||
|
mymodule.set("my_function", my_fn)?;
|
||||||
|
|
||||||
|
jarvis.set("mymodule", mymodule)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// # 2. ADD NEW API MODULE TO mod.rs
|
||||||
|
|
||||||
|
pub mod mymodule;
|
||||||
|
|
||||||
|
|
||||||
|
// # 3. REGISTER NEW MODULE IN engine.rs
|
||||||
|
// in register_api()
|
||||||
|
api::mymodule::register(&self.lua, &jarvis)?;
|
||||||
|
|
||||||
|
// # 4. YOU CAN ALSO DEFINE ASYNC FUNCTIONS INSTEAD
|
||||||
|
let async_fn = lua.create_async_function(|_, url: String| async move {
|
||||||
|
let response = reqwest::get(&url).await
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
Ok(response.text().await.unwrap_or_default())
|
||||||
|
})?;
|
||||||
77
crates/jarvis-core/src/lua/api/audio.rs
Normal file
77
crates/jarvis-core/src/lua/api/audio.rs
Normal file
|
|
@ -0,0 +1,77 @@
|
||||||
|
// Audio Lua API: something sound related, apparently :3
|
||||||
|
|
||||||
|
use mlua::{Lua, Table};
|
||||||
|
use crate::voices::{self, Reaction};
|
||||||
|
|
||||||
|
pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> {
|
||||||
|
let audio = lua.create_table()?;
|
||||||
|
|
||||||
|
// jarvis.audio.play(reaction)
|
||||||
|
// reactions: "ok", "reply", "greet", "not_found", "error", "goodbye", "thanks"
|
||||||
|
let play_fn = lua.create_function(|_, reaction: String| {
|
||||||
|
let reaction_enum = match reaction.to_lowercase().as_str() {
|
||||||
|
"ok" => Reaction::Ok,
|
||||||
|
"reply" => Reaction::Reply,
|
||||||
|
"greet" => Reaction::Greet,
|
||||||
|
"not_found" => Reaction::NotFound,
|
||||||
|
"error" => Reaction::Error,
|
||||||
|
"goodbye" => Reaction::Goodbye,
|
||||||
|
"thanks" => Reaction::Thanks,
|
||||||
|
// "joke" => Reaction::Joke, NO PUN INTENDED :3
|
||||||
|
_ => {
|
||||||
|
log::warn!("[Lua] Unknown reaction: {}", reaction);
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
voices::play(reaction_enum);
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
audio.set("play", play_fn)?;
|
||||||
|
|
||||||
|
// jarvis.audio.play_ok()
|
||||||
|
let play_ok_fn = lua.create_function(|_, ()| {
|
||||||
|
voices::play_ok();
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
audio.set("play_ok", play_ok_fn)?;
|
||||||
|
|
||||||
|
// jarvis.audio.play_reply()
|
||||||
|
let play_reply_fn = lua.create_function(|_, ()| {
|
||||||
|
voices::play_reply();
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
audio.set("play_reply", play_reply_fn)?;
|
||||||
|
|
||||||
|
// jarvis.audio.play_error()
|
||||||
|
let play_error_fn = lua.create_function(|_, ()| {
|
||||||
|
voices::play_error();
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
audio.set("play_error", play_error_fn)?;
|
||||||
|
|
||||||
|
// jarvis.audio.play_not_found()
|
||||||
|
let play_not_found_fn = lua.create_function(|_, ()| {
|
||||||
|
voices::play_not_found();
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
audio.set("play_not_found", play_not_found_fn)?;
|
||||||
|
|
||||||
|
// jarvis.audio.play_greet()
|
||||||
|
let play_greet_fn = lua.create_function(|_, ()| {
|
||||||
|
voices::play_greet();
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
audio.set("play_greet", play_greet_fn)?;
|
||||||
|
|
||||||
|
// jarvis.audio.play_goodbye()
|
||||||
|
let play_goodbye_fn = lua.create_function(|_, ()| {
|
||||||
|
voices::play_goodbye();
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
audio.set("play_goodbye", play_goodbye_fn)?;
|
||||||
|
|
||||||
|
jarvis.set("audio", audio)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
45
crates/jarvis-core/src/lua/api/context.rs
Normal file
45
crates/jarvis-core/src/lua/api/context.rs
Normal file
|
|
@ -0,0 +1,45 @@
|
||||||
|
// Context Lua API: read-only command context
|
||||||
|
|
||||||
|
use mlua::{Lua, Table};
|
||||||
|
use crate::lua::{CommandContext};
|
||||||
|
|
||||||
|
use crate::commands::SlotValue;
|
||||||
|
|
||||||
|
pub fn register(lua: &Lua, jarvis: &Table, ctx: &CommandContext) -> mlua::Result<()> {
|
||||||
|
let context = lua.create_table()?;
|
||||||
|
|
||||||
|
// read-only context values
|
||||||
|
context.set("phrase", ctx.phrase.clone())?;
|
||||||
|
context.set("command_id", ctx.command_id.clone())?;
|
||||||
|
context.set("command_path", ctx.command_path.to_string_lossy().to_string())?;
|
||||||
|
context.set("language", ctx.language.clone())?;
|
||||||
|
|
||||||
|
// time info
|
||||||
|
let time = lua.create_table()?;
|
||||||
|
let now = chrono::Local::now();
|
||||||
|
time.set("year", now.format("%Y").to_string())?;
|
||||||
|
time.set("month", now.format("%m").to_string())?;
|
||||||
|
time.set("day", now.format("%d").to_string())?;
|
||||||
|
time.set("hour", now.format("%H").to_string())?;
|
||||||
|
time.set("minute", now.format("%M").to_string())?;
|
||||||
|
time.set("second", now.format("%S").to_string())?;
|
||||||
|
time.set("weekday", now.format("%A").to_string())?;
|
||||||
|
time.set("timestamp", now.timestamp())?;
|
||||||
|
context.set("time", time)?;
|
||||||
|
|
||||||
|
// slots
|
||||||
|
let slots_table = lua.create_table()?;
|
||||||
|
if let Some(ref slots) = ctx.slots {
|
||||||
|
for (name, value) in slots {
|
||||||
|
match value {
|
||||||
|
SlotValue::Text(t) => slots_table.set(name.as_str(), t.as_str())?,
|
||||||
|
SlotValue::Number(n) => slots_table.set(name.as_str(), *n)?,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
context.set("slots", slots_table)?;
|
||||||
|
|
||||||
|
jarvis.set("context", context)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
50
crates/jarvis-core/src/lua/api/core.rs
Normal file
50
crates/jarvis-core/src/lua/api/core.rs
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
// Core Lua API: log, sleep, print, etc.
|
||||||
|
|
||||||
|
use mlua::{Lua, Table, MultiValue};
|
||||||
|
|
||||||
|
pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> {
|
||||||
|
|
||||||
|
// @ jarvis.log(level, message)
|
||||||
|
// log something
|
||||||
|
let log_fn = lua.create_function(|_, (level, message): (String, String)| {
|
||||||
|
match level.to_lowercase().as_str() {
|
||||||
|
"debug" => log::debug!("[Lua] {}", message),
|
||||||
|
"info" => log::info!("[Lua] {}", message),
|
||||||
|
"warn" => log::warn!("[Lua] {}", message),
|
||||||
|
"error" => log::error!("[Lua] {}", message),
|
||||||
|
_ => log::info!("[Lua] {}", message),
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
jarvis.set("log", log_fn)?;
|
||||||
|
|
||||||
|
// @ jarvis.print(...)
|
||||||
|
// simple print
|
||||||
|
let print_fn = lua.create_function(|_, args: MultiValue| {
|
||||||
|
let parts: Vec<String> = args.iter()
|
||||||
|
.map(|v| format!("{:?}", v))
|
||||||
|
.collect();
|
||||||
|
log::info!("[Lua] {}", parts.join(" "));
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
jarvis.set("print", print_fn)?;
|
||||||
|
|
||||||
|
// @ jarvis.sleep(ms)
|
||||||
|
// ..zZZ
|
||||||
|
let sleep_fn = lua.create_function(|_, ms: u64| {
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(ms));
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
jarvis.set("sleep", sleep_fn)?;
|
||||||
|
|
||||||
|
// @ jarvis.speak(text)
|
||||||
|
// @TODO: update when TTS will be implemented
|
||||||
|
let speak_fn = lua.create_function(|_, text: String| {
|
||||||
|
log::info!("[Lua] SPEAK: {}", text);
|
||||||
|
// pass
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
jarvis.set("speak", speak_fn)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
212
crates/jarvis-core/src/lua/api/fs.rs
Normal file
212
crates/jarvis-core/src/lua/api/fs.rs
Normal file
|
|
@ -0,0 +1,212 @@
|
||||||
|
// File System Lua API: read, write, list (sandboxed)
|
||||||
|
|
||||||
|
use mlua::{Lua, Table};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
use crate::lua::sandbox::SandboxLevel;
|
||||||
|
|
||||||
|
pub fn register(
|
||||||
|
lua: &Lua,
|
||||||
|
jarvis: &Table,
|
||||||
|
command_path: &PathBuf,
|
||||||
|
sandbox: SandboxLevel,
|
||||||
|
) -> mlua::Result<()> {
|
||||||
|
let fs_table = lua.create_table()?;
|
||||||
|
|
||||||
|
let cmd_path = command_path.clone();
|
||||||
|
let sandbox_level = sandbox;
|
||||||
|
|
||||||
|
// jarvis.fs.read(path)
|
||||||
|
let cmd_path_read = cmd_path.clone();
|
||||||
|
let read_fn = lua.create_function(move |_, path: String| {
|
||||||
|
let full_path = resolve_path(&cmd_path_read, &path, sandbox_level)?;
|
||||||
|
|
||||||
|
fs::read_to_string(&full_path)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Read error: {}", e)))
|
||||||
|
})?;
|
||||||
|
fs_table.set("read", read_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.read_bytes(path)
|
||||||
|
let cmd_path_read_bytes = cmd_path.clone();
|
||||||
|
let read_bytes_fn = lua.create_function(move |lua, path: String| {
|
||||||
|
let full_path = resolve_path(&cmd_path_read_bytes, &path, sandbox_level)?;
|
||||||
|
|
||||||
|
let bytes = fs::read(&full_path)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Read error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(lua.create_string(&bytes)?)
|
||||||
|
})?;
|
||||||
|
fs_table.set("read_bytes", read_bytes_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.write(path, content)
|
||||||
|
let cmd_path_write = cmd_path.clone();
|
||||||
|
let write_fn = lua.create_function(move |_, (path, content): (String, String)| {
|
||||||
|
if !sandbox_level.allows_fs_write() {
|
||||||
|
return Err(mlua::Error::runtime("Write not allowed in this sandbox"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let full_path = resolve_path(&cmd_path_write, &path, sandbox_level)?;
|
||||||
|
|
||||||
|
// ensure parent directory exists
|
||||||
|
if let Some(parent) = full_path.parent() {
|
||||||
|
fs::create_dir_all(parent)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Create dir error: {}", e)))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
fs::write(&full_path, content)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Write error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
fs_table.set("write", write_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.append(path, content)
|
||||||
|
let cmd_path_append = cmd_path.clone();
|
||||||
|
let append_fn = lua.create_function(move |_, (path, content): (String, String)| {
|
||||||
|
if !sandbox_level.allows_fs_write() {
|
||||||
|
return Err(mlua::Error::runtime("Write not allowed in this sandbox"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let full_path = resolve_path(&cmd_path_append, &path, sandbox_level)?;
|
||||||
|
|
||||||
|
use std::io::Write;
|
||||||
|
let mut file = fs::OpenOptions::new()
|
||||||
|
.create(true)
|
||||||
|
.append(true)
|
||||||
|
.open(&full_path)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Open error: {}", e)))?;
|
||||||
|
|
||||||
|
file.write_all(content.as_bytes())
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Write error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
fs_table.set("append", append_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.exists(path)
|
||||||
|
let cmd_path_exists = cmd_path.clone();
|
||||||
|
let exists_fn = lua.create_function(move |_, path: String| {
|
||||||
|
let full_path = resolve_path(&cmd_path_exists, &path, sandbox_level)?;
|
||||||
|
Ok(full_path.exists())
|
||||||
|
})?;
|
||||||
|
fs_table.set("exists", exists_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.is_file(path)
|
||||||
|
let cmd_path_is_file = cmd_path.clone();
|
||||||
|
let is_file_fn = lua.create_function(move |_, path: String| {
|
||||||
|
let full_path = resolve_path(&cmd_path_is_file, &path, sandbox_level)?;
|
||||||
|
Ok(full_path.is_file())
|
||||||
|
})?;
|
||||||
|
fs_table.set("is_file", is_file_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.is_dir(path)
|
||||||
|
let cmd_path_is_dir = cmd_path.clone();
|
||||||
|
let is_dir_fn = lua.create_function(move |_, path: String| {
|
||||||
|
let full_path = resolve_path(&cmd_path_is_dir, &path, sandbox_level)?;
|
||||||
|
Ok(full_path.is_dir())
|
||||||
|
})?;
|
||||||
|
fs_table.set("is_dir", is_dir_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.list(path)
|
||||||
|
let cmd_path_list = cmd_path.clone();
|
||||||
|
let list_fn = lua.create_function(move |lua, path: Option<String>| {
|
||||||
|
let full_path = if let Some(p) = path {
|
||||||
|
resolve_path(&cmd_path_list, &p, sandbox_level)?
|
||||||
|
} else {
|
||||||
|
cmd_path_list.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = lua.create_table()?;
|
||||||
|
|
||||||
|
let entries = fs::read_dir(&full_path)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("List error: {}", e)))?;
|
||||||
|
|
||||||
|
let mut idx = 1;
|
||||||
|
for entry in entries {
|
||||||
|
if let Ok(entry) = entry {
|
||||||
|
let item = lua.create_table()?;
|
||||||
|
item.set("name", entry.file_name().to_string_lossy().to_string())?;
|
||||||
|
item.set("path", entry.path().to_string_lossy().to_string())?;
|
||||||
|
item.set("is_file", entry.path().is_file())?;
|
||||||
|
item.set("is_dir", entry.path().is_dir())?;
|
||||||
|
|
||||||
|
result.set(idx, item)?;
|
||||||
|
idx += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
})?;
|
||||||
|
fs_table.set("list", list_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.mkdir(path)
|
||||||
|
let cmd_path_mkdir = cmd_path.clone();
|
||||||
|
let mkdir_fn = lua.create_function(move |_, path: String| {
|
||||||
|
if !sandbox_level.allows_fs_write() {
|
||||||
|
return Err(mlua::Error::runtime("Write not allowed in this sandbox"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let full_path = resolve_path(&cmd_path_mkdir, &path, sandbox_level)?;
|
||||||
|
|
||||||
|
fs::create_dir_all(&full_path)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Mkdir error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
fs_table.set("mkdir", mkdir_fn)?;
|
||||||
|
|
||||||
|
// jarvis.fs.remove(path)
|
||||||
|
let cmd_path_remove = cmd_path.clone();
|
||||||
|
let remove_fn = lua.create_function(move |_, path: String| {
|
||||||
|
if !sandbox_level.allows_fs_write() {
|
||||||
|
return Err(mlua::Error::runtime("Write not allowed in this sandbox"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let full_path = resolve_path(&cmd_path_remove, &path, sandbox_level)?;
|
||||||
|
|
||||||
|
if full_path.is_dir() {
|
||||||
|
fs::remove_dir_all(&full_path)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Remove error: {}", e)))?;
|
||||||
|
} else {
|
||||||
|
fs::remove_file(&full_path)
|
||||||
|
.map_err(|e| mlua::Error::runtime(format!("Remove error: {}", e)))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
fs_table.set("remove", remove_fn)?;
|
||||||
|
|
||||||
|
jarvis.set("fs", fs_table)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve path relative to command folder, with sandbox checks
|
||||||
|
fn resolve_path(command_path: &PathBuf, path: &str, sandbox: SandboxLevel) -> mlua::Result<PathBuf> {
|
||||||
|
let path = Path::new(path);
|
||||||
|
|
||||||
|
// if absolute path, check sandbox allows it
|
||||||
|
if path.is_absolute() {
|
||||||
|
if !sandbox.allows_expanded_paths() {
|
||||||
|
return Err(mlua::Error::runtime("Absolute paths not allowed in this sandbox"));
|
||||||
|
}
|
||||||
|
return Ok(path.to_path_buf());
|
||||||
|
}
|
||||||
|
|
||||||
|
// relative path - resolve against command folder
|
||||||
|
let resolved = command_path.join(path);
|
||||||
|
|
||||||
|
// canonicalize to resolve ../ etc and check it's still within command folder
|
||||||
|
let canonical = resolved.canonicalize()
|
||||||
|
.unwrap_or_else(|_| resolved.clone());
|
||||||
|
|
||||||
|
let cmd_canonical = command_path.canonicalize()
|
||||||
|
.unwrap_or_else(|_| command_path.clone());
|
||||||
|
|
||||||
|
if !sandbox.allows_expanded_paths() && !canonical.starts_with(&cmd_canonical) {
|
||||||
|
return Err(mlua::Error::runtime("Path escapes command folder"));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(resolved)
|
||||||
|
}
|
||||||
226
crates/jarvis-core/src/lua/api/http.rs
Normal file
226
crates/jarvis-core/src/lua/api/http.rs
Normal file
|
|
@ -0,0 +1,226 @@
|
||||||
|
// HTTP Lua API: GET, POST, JSON requests
|
||||||
|
|
||||||
|
use mlua::{Lua, Table, Value};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
pub fn register(lua: &Lua, jarvis: &Table) -> mlua::Result<()> {
|
||||||
|
let http = lua.create_table()?;
|
||||||
|
|
||||||
|
// jarvis.http.get(url, headers?)
|
||||||
|
let get_fn = lua.create_function(|lua, (url, headers): (String, Option<Table>)| {
|
||||||
|
http_request(lua, "GET", &url, None, headers)
|
||||||
|
})?;
|
||||||
|
http.set("get", get_fn)?;
|
||||||
|
|
||||||
|
// jarvis.http.post(url, body, headers?)
|
||||||
|
let post_fn = lua.create_function(|lua, (url, body, headers): (String, Option<String>, Option<Table>)| {
|
||||||
|
http_request(lua, "POST", &url, body, headers)
|
||||||
|
})?;
|
||||||
|
http.set("post", post_fn)?;
|
||||||
|
|
||||||
|
// jarvis.http.post_json(url, data, headers?)
|
||||||
|
let post_json_fn = lua.create_function(|lua, (url, data, headers): (String, Table, Option<Table>)| {
|
||||||
|
// convert Lua table to JSON string
|
||||||
|
let json_value = table_to_json(lua, data)?;
|
||||||
|
let body = serde_json::to_string(&json_value)
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
// add content-type header
|
||||||
|
let mut header_map: HashMap<String, String> = HashMap::new();
|
||||||
|
header_map.insert("Content-Type".to_string(), "application/json".to_string());
|
||||||
|
|
||||||
|
if let Some(h) = headers {
|
||||||
|
for pair in h.pairs::<String, String>() {
|
||||||
|
if let Ok((k, v)) = pair {
|
||||||
|
header_map.insert(k, v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
http_request_with_headers(lua, "POST", &url, Some(body), header_map)
|
||||||
|
})?;
|
||||||
|
http.set("post_json", post_json_fn)?;
|
||||||
|
|
||||||
|
// jarvis.http.json(url) - GET + parse JSON
|
||||||
|
let json_fn = lua.create_function(|lua, url: String| {
|
||||||
|
let client = reqwest::blocking::Client::builder()
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
let response = client.get(&url)
|
||||||
|
.send()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
if response.status().is_success() {
|
||||||
|
let json: serde_json::Value = response.json()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
json_to_lua(lua, json)
|
||||||
|
} else {
|
||||||
|
Ok(Value::Nil)
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
http.set("json", json_fn)?;
|
||||||
|
|
||||||
|
jarvis.set("http", http)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn http_request(
|
||||||
|
lua: &Lua,
|
||||||
|
method: &str,
|
||||||
|
url: &str,
|
||||||
|
body: Option<String>,
|
||||||
|
headers: Option<Table>,
|
||||||
|
) -> mlua::Result<Table> {
|
||||||
|
let header_map: HashMap<String, String> = if let Some(h) = headers {
|
||||||
|
h.pairs::<String, String>()
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.collect()
|
||||||
|
} else {
|
||||||
|
HashMap::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
http_request_with_headers(lua, method, url, body, header_map)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn http_request_with_headers(
|
||||||
|
lua: &Lua,
|
||||||
|
method: &str,
|
||||||
|
url: &str,
|
||||||
|
body: Option<String>,
|
||||||
|
headers: HashMap<String, String>,
|
||||||
|
) -> mlua::Result<Table> {
|
||||||
|
let client = reqwest::blocking::Client::builder()
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
let mut request = match method {
|
||||||
|
"POST" => client.post(url),
|
||||||
|
"PUT" => client.put(url),
|
||||||
|
"DELETE" => client.delete(url),
|
||||||
|
_ => client.get(url),
|
||||||
|
};
|
||||||
|
|
||||||
|
for (k, v) in headers {
|
||||||
|
request = request.header(&k, &v);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(b) = body {
|
||||||
|
request = request.body(b);
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = lua.create_table()?;
|
||||||
|
|
||||||
|
match request.send() {
|
||||||
|
Ok(response) => {
|
||||||
|
result.set("ok", response.status().is_success())?;
|
||||||
|
result.set("status", response.status().as_u16())?;
|
||||||
|
|
||||||
|
// get headers
|
||||||
|
let headers_table = lua.create_table()?;
|
||||||
|
for (name, value) in response.headers() {
|
||||||
|
if let Ok(v) = value.to_str() {
|
||||||
|
headers_table.set(name.as_str(), v)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.set("headers", headers_table)?;
|
||||||
|
|
||||||
|
// get body
|
||||||
|
match response.text() {
|
||||||
|
Ok(text) => result.set("body", text)?,
|
||||||
|
Err(e) => result.set("body", format!("Error reading body: {}", e))?,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
result.set("ok", false)?;
|
||||||
|
result.set("status", 0)?;
|
||||||
|
result.set("error", e.to_string())?;
|
||||||
|
result.set("body", "")?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert Lua table to serde_json::Value
|
||||||
|
fn table_to_json(lua: &Lua, table: Table) -> mlua::Result<serde_json::Value> {
|
||||||
|
use serde_json::{Value as JsonValue, Map};
|
||||||
|
|
||||||
|
// check if it's an array (sequential integer keys starting from 1)
|
||||||
|
let is_array = table.clone().pairs::<i64, Value>()
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.enumerate()
|
||||||
|
.all(|(i, (k, _))| k == (i + 1) as i64);
|
||||||
|
|
||||||
|
if is_array && table.len()? > 0 {
|
||||||
|
let arr: Vec<JsonValue> = table.sequence_values::<Value>()
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.map(|v| lua_to_json(lua, v))
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
Ok(JsonValue::Array(arr))
|
||||||
|
} else {
|
||||||
|
let mut map = Map::new();
|
||||||
|
for pair in table.pairs::<String, Value>() {
|
||||||
|
let (k, v) = pair?;
|
||||||
|
map.insert(k, lua_to_json(lua, v)?);
|
||||||
|
}
|
||||||
|
Ok(JsonValue::Object(map))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert Lua Value to serde_json::Value
|
||||||
|
fn lua_to_json(lua: &Lua, value: Value) -> mlua::Result<serde_json::Value> {
|
||||||
|
use serde_json::{Value as JsonValue, Number};
|
||||||
|
|
||||||
|
match value {
|
||||||
|
Value::Nil => Ok(JsonValue::Null),
|
||||||
|
Value::Boolean(b) => Ok(JsonValue::Bool(b)),
|
||||||
|
Value::Integer(i) => Ok(JsonValue::Number(Number::from(i))),
|
||||||
|
Value::Number(n) => {
|
||||||
|
Number::from_f64(n)
|
||||||
|
.map(JsonValue::Number)
|
||||||
|
.ok_or_else(|| mlua::Error::runtime("Invalid float"))
|
||||||
|
}
|
||||||
|
Value::String(s) => Ok(JsonValue::String(s.to_str()?.to_string())),
|
||||||
|
Value::Table(t) => table_to_json(lua, t),
|
||||||
|
_ => Err(mlua::Error::runtime("Unsupported type for JSON")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert serde_json::Value to Lua Value
|
||||||
|
fn json_to_lua(lua: &Lua, json: serde_json::Value) -> mlua::Result<Value> {
|
||||||
|
use serde_json::Value as JsonValue;
|
||||||
|
|
||||||
|
match json {
|
||||||
|
JsonValue::Null => Ok(Value::Nil),
|
||||||
|
JsonValue::Bool(b) => Ok(Value::Boolean(b)),
|
||||||
|
JsonValue::Number(n) => {
|
||||||
|
if let Some(i) = n.as_i64() {
|
||||||
|
Ok(Value::Integer(i))
|
||||||
|
} else if let Some(f) = n.as_f64() {
|
||||||
|
Ok(Value::Number(f))
|
||||||
|
} else {
|
||||||
|
Ok(Value::Nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
JsonValue::String(s) => Ok(Value::String(lua.create_string(&s)?)),
|
||||||
|
JsonValue::Array(arr) => {
|
||||||
|
let table = lua.create_table()?;
|
||||||
|
for (i, v) in arr.into_iter().enumerate() {
|
||||||
|
table.set(i + 1, json_to_lua(lua, v)?)?;
|
||||||
|
}
|
||||||
|
Ok(Value::Table(table))
|
||||||
|
}
|
||||||
|
JsonValue::Object(map) => {
|
||||||
|
let table = lua.create_table()?;
|
||||||
|
for (k, v) in map {
|
||||||
|
table.set(k, json_to_lua(lua, v)?)?;
|
||||||
|
}
|
||||||
|
Ok(Value::Table(table))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
0
crates/jarvis-core/src/lua/api/settings.rs
Normal file
0
crates/jarvis-core/src/lua/api/settings.rs
Normal file
184
crates/jarvis-core/src/lua/api/state.rs
Normal file
184
crates/jarvis-core/src/lua/api/state.rs
Normal file
|
|
@ -0,0 +1,184 @@
|
||||||
|
// State Lua API: persistent key-value storage per command
|
||||||
|
|
||||||
|
use mlua::{Lua, Table, Result, Value};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::fs;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
const STATE_FILE: &str = ".state.json";
|
||||||
|
|
||||||
|
pub fn register(lua: &Lua, jarvis: &Table, command_path: &PathBuf) -> mlua::Result<()> {
|
||||||
|
let state = lua.create_table()?;
|
||||||
|
let state_path = command_path.join(STATE_FILE);
|
||||||
|
|
||||||
|
// jarvis.state.get(key)
|
||||||
|
let state_path_get = state_path.clone();
|
||||||
|
let get_fn = lua.create_function(move |lua, key: String| {
|
||||||
|
let data = load_state(&state_path_get);
|
||||||
|
|
||||||
|
if let Some(value) = data.get(&key) {
|
||||||
|
json_to_lua_value(lua, value.clone())
|
||||||
|
} else {
|
||||||
|
Ok(Value::Nil)
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
state.set("get", get_fn)?;
|
||||||
|
|
||||||
|
// jarvis.state.set(key, value)
|
||||||
|
let state_path_set = state_path.clone();
|
||||||
|
let set_fn = lua.create_function(move |_, (key, value): (String, Value)| {
|
||||||
|
let mut data = load_state(&state_path_set);
|
||||||
|
|
||||||
|
let json_value = lua_to_json_value(value)?;
|
||||||
|
data.insert(key, json_value);
|
||||||
|
|
||||||
|
save_state(&state_path_set, &data)?;
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
state.set("set", set_fn)?;
|
||||||
|
|
||||||
|
// jarvis.state.delete(key)
|
||||||
|
let state_path_delete = state_path.clone();
|
||||||
|
let delete_fn = lua.create_function(move |_, key: String| {
|
||||||
|
let mut data = load_state(&state_path_delete);
|
||||||
|
let existed = data.remove(&key).is_some();
|
||||||
|
save_state(&state_path_delete, &data)?;
|
||||||
|
Ok(existed)
|
||||||
|
})?;
|
||||||
|
state.set("delete", delete_fn)?;
|
||||||
|
|
||||||
|
// jarvis.state.clear()
|
||||||
|
let state_path_clear = state_path.clone();
|
||||||
|
let clear_fn = lua.create_function(move |_, ()| {
|
||||||
|
let data: HashMap<String, serde_json::Value> = HashMap::new();
|
||||||
|
save_state(&state_path_clear, &data)?;
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
state.set("clear", clear_fn)?;
|
||||||
|
|
||||||
|
// jarvis.state.keys()
|
||||||
|
let state_path_keys = state_path.clone();
|
||||||
|
let keys_fn = lua.create_function(move |lua, ()| {
|
||||||
|
let data = load_state(&state_path_keys);
|
||||||
|
let table = lua.create_table()?;
|
||||||
|
|
||||||
|
for (i, key) in data.keys().enumerate() {
|
||||||
|
table.set(i + 1, key.clone())?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(table)
|
||||||
|
})?;
|
||||||
|
state.set("keys", keys_fn)?;
|
||||||
|
|
||||||
|
// jarvis.state.all()
|
||||||
|
let state_path_all = state_path.clone();
|
||||||
|
let all_fn = lua.create_function(move |lua, ()| {
|
||||||
|
let data = load_state(&state_path_all);
|
||||||
|
let table = lua.create_table()?;
|
||||||
|
|
||||||
|
for (key, value) in data {
|
||||||
|
let lua_value = json_to_lua_value(lua, value)?;
|
||||||
|
table.set(key, lua_value)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(table)
|
||||||
|
})?;
|
||||||
|
state.set("all", all_fn)?;
|
||||||
|
|
||||||
|
jarvis.set("state", state)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_state(path: &PathBuf) -> HashMap<String, serde_json::Value> {
|
||||||
|
if !path.exists() {
|
||||||
|
return HashMap::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
fs::read_to_string(path)
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| serde_json::from_str(&s).ok())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save_state(path: &PathBuf, data: &HashMap<String, serde_json::Value>) -> mlua::Result<()> {
|
||||||
|
let json = serde_json::to_string_pretty(data)
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
fs::write(path, json)
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lua_to_json_value(value: Value) -> mlua::Result<serde_json::Value> {
|
||||||
|
use serde_json::Value as JsonValue;
|
||||||
|
|
||||||
|
match value {
|
||||||
|
Value::Nil => Ok(JsonValue::Null),
|
||||||
|
Value::Boolean(b) => Ok(JsonValue::Bool(b)),
|
||||||
|
Value::Integer(i) => Ok(JsonValue::Number(i.into())),
|
||||||
|
Value::Number(n) => {
|
||||||
|
serde_json::Number::from_f64(n)
|
||||||
|
.map(JsonValue::Number)
|
||||||
|
.ok_or_else(|| mlua::Error::runtime("Invalid float"))
|
||||||
|
}
|
||||||
|
Value::String(s) => Ok(JsonValue::String(s.to_str()?.to_string())),
|
||||||
|
Value::Table(t) => {
|
||||||
|
// check if array
|
||||||
|
let is_array = t.clone().pairs::<i64, Value>()
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.enumerate()
|
||||||
|
.all(|(i, (k, _))| k == (i + 1) as i64);
|
||||||
|
|
||||||
|
if is_array && t.len().unwrap_or(0) > 0 {
|
||||||
|
let arr: Vec<JsonValue> = t.sequence_values::<Value>()
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.map(lua_to_json_value)
|
||||||
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
Ok(JsonValue::Array(arr))
|
||||||
|
} else {
|
||||||
|
let mut map = serde_json::Map::new();
|
||||||
|
for pair in t.pairs::<String, Value>() {
|
||||||
|
let (k, v) = pair?;
|
||||||
|
map.insert(k, lua_to_json_value(v)?);
|
||||||
|
}
|
||||||
|
Ok(JsonValue::Object(map))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => Err(mlua::Error::runtime("Unsupported type for state")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn json_to_lua_value(lua: &Lua, json: serde_json::Value) -> mlua::Result<Value> {
|
||||||
|
use serde_json::Value as JsonValue;
|
||||||
|
|
||||||
|
match json {
|
||||||
|
JsonValue::Null => Ok(Value::Nil),
|
||||||
|
JsonValue::Bool(b) => Ok(Value::Boolean(b)),
|
||||||
|
JsonValue::Number(n) => {
|
||||||
|
if let Some(i) = n.as_i64() {
|
||||||
|
Ok(Value::Integer(i))
|
||||||
|
} else if let Some(f) = n.as_f64() {
|
||||||
|
Ok(Value::Number(f))
|
||||||
|
} else {
|
||||||
|
Ok(Value::Nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
JsonValue::String(s) => Ok(Value::String(lua.create_string(&s)?)),
|
||||||
|
JsonValue::Array(arr) => {
|
||||||
|
let table = lua.create_table()?;
|
||||||
|
for (i, v) in arr.into_iter().enumerate() {
|
||||||
|
table.set(i + 1, json_to_lua_value(lua, v)?)?;
|
||||||
|
}
|
||||||
|
Ok(Value::Table(table))
|
||||||
|
}
|
||||||
|
JsonValue::Object(map) => {
|
||||||
|
let table = lua.create_table()?;
|
||||||
|
for (k, v) in map {
|
||||||
|
table.set(k, json_to_lua_value(lua, v)?)?;
|
||||||
|
}
|
||||||
|
Ok(Value::Table(table))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
240
crates/jarvis-core/src/lua/api/system.rs
Normal file
240
crates/jarvis-core/src/lua/api/system.rs
Normal file
|
|
@ -0,0 +1,240 @@
|
||||||
|
// System Lua API: exec, open, clipboard, notify
|
||||||
|
|
||||||
|
use mlua::{Lua, Table};
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
use crate::lua::sandbox::SandboxLevel;
|
||||||
|
|
||||||
|
pub fn register(lua: &Lua, jarvis: &Table, sandbox: SandboxLevel) -> mlua::Result<()> {
|
||||||
|
let system = lua.create_table()?;
|
||||||
|
|
||||||
|
// jarvis.system.open(url_or_path) - always available
|
||||||
|
let open_fn = lua.create_function(|_, target: String| {
|
||||||
|
let result = if cfg!(target_os = "windows") {
|
||||||
|
Command::new("cmd")
|
||||||
|
.args(["/C", "start", "", &target])
|
||||||
|
.spawn()
|
||||||
|
} else if cfg!(target_os = "macos") {
|
||||||
|
Command::new("open")
|
||||||
|
.arg(&target)
|
||||||
|
.spawn()
|
||||||
|
} else {
|
||||||
|
Command::new("xdg-open")
|
||||||
|
.arg(&target)
|
||||||
|
.spawn()
|
||||||
|
};
|
||||||
|
|
||||||
|
match result {
|
||||||
|
Ok(_) => Ok(true),
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!("[Lua] Failed to open {}: {}", target, e);
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
system.set("open", open_fn)?;
|
||||||
|
|
||||||
|
// jarvis.system.exec(cmd, args?) - only in full sandbox
|
||||||
|
if sandbox.allows_exec() {
|
||||||
|
let exec_fn = lua.create_function(|lua, (cmd, args): (String, Option<Table>)| {
|
||||||
|
let mut command = if cfg!(target_os = "windows") {
|
||||||
|
let mut c = Command::new("cmd");
|
||||||
|
c.args(["/C", &cmd]);
|
||||||
|
c
|
||||||
|
} else {
|
||||||
|
let mut c = Command::new("sh");
|
||||||
|
c.args(["-c", &cmd]);
|
||||||
|
c
|
||||||
|
};
|
||||||
|
|
||||||
|
// add extra args if provided
|
||||||
|
if let Some(args_table) = args {
|
||||||
|
for pair in args_table.sequence_values::<String>() {
|
||||||
|
if let Ok(arg) = pair {
|
||||||
|
command.arg(arg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let output = command.output()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
let result = lua.create_table()?;
|
||||||
|
result.set("success", output.status.success())?;
|
||||||
|
result.set("code", output.status.code().unwrap_or(-1))?;
|
||||||
|
result.set("stdout", String::from_utf8_lossy(&output.stdout).to_string())?;
|
||||||
|
result.set("stderr", String::from_utf8_lossy(&output.stderr).to_string())?;
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
})?;
|
||||||
|
system.set("exec", exec_fn)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// jarvis.system.notify(title, message) - always available
|
||||||
|
let notify_fn = lua.create_function(|_, (title, message): (String, String)| {
|
||||||
|
log::info!("[Lua] NOTIFY: {} - {}", title, message);
|
||||||
|
|
||||||
|
// platform-specific notification
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
{
|
||||||
|
use winrt_notification::{Toast, Duration as ToastDuration};
|
||||||
|
|
||||||
|
if let Err(e) = Toast::new(Toast::POWERSHELL_APP_ID)
|
||||||
|
.title(&title)
|
||||||
|
.text1(&message)
|
||||||
|
.duration(ToastDuration::Short)
|
||||||
|
.show()
|
||||||
|
{
|
||||||
|
log::warn!("[Lua] Failed to show toast notification: {}", e);
|
||||||
|
// fallback to msg.exe
|
||||||
|
let _ = Command::new("msg")
|
||||||
|
.args(["*", "/time:10", &format!("{}: {}", title, message)])
|
||||||
|
.spawn();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
{
|
||||||
|
let _ = Command::new("notify-send")
|
||||||
|
.args([&title, &message])
|
||||||
|
.spawn();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
{
|
||||||
|
let script = format!(
|
||||||
|
r#"display notification "{}" with title "{}""#,
|
||||||
|
message.replace("\"", "\\\""),
|
||||||
|
title.replace("\"", "\\\"")
|
||||||
|
);
|
||||||
|
let _ = Command::new("osascript")
|
||||||
|
.args(["-e", &script])
|
||||||
|
.spawn();
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
system.set("notify", notify_fn)?;
|
||||||
|
|
||||||
|
// jarvis.system.clipboard - subtable
|
||||||
|
let clipboard = lua.create_table()?;
|
||||||
|
|
||||||
|
// jarvis.system.clipboard.get() - always available
|
||||||
|
let clipboard_get_fn = lua.create_function(|_, ()| {
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
{
|
||||||
|
let output = Command::new("powershell")
|
||||||
|
.args(["-Command", "Get-Clipboard"])
|
||||||
|
.output()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
{
|
||||||
|
let output = Command::new("xclip")
|
||||||
|
.args(["-selection", "clipboard", "-o"])
|
||||||
|
.output()
|
||||||
|
.or_else(|_| {
|
||||||
|
Command::new("xsel")
|
||||||
|
.args(["--clipboard", "--output"])
|
||||||
|
.output()
|
||||||
|
})
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(String::from_utf8_lossy(&output.stdout).to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
{
|
||||||
|
let output = Command::new("pbpaste")
|
||||||
|
.output()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(String::from_utf8_lossy(&output.stdout).to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(any(target_os = "windows", target_os = "linux", target_os = "macos")))]
|
||||||
|
{
|
||||||
|
Err(mlua::Error::runtime("Clipboard not supported on this platform"))
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
clipboard.set("get", clipboard_get_fn)?;
|
||||||
|
|
||||||
|
// jarvis.system.clipboard.set(text) - only in full sandbox
|
||||||
|
if sandbox.allows_clipboard_write() {
|
||||||
|
let clipboard_set_fn = lua.create_function(|_, text: String| {
|
||||||
|
#[cfg(target_os = "windows")]
|
||||||
|
{
|
||||||
|
let script = format!("Set-Clipboard -Value '{}'", text.replace("'", "''"));
|
||||||
|
Command::new("powershell")
|
||||||
|
.args(["-Command", &script])
|
||||||
|
.output()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
{
|
||||||
|
use std::io::Write;
|
||||||
|
let mut child = Command::new("xclip")
|
||||||
|
.args(["-selection", "clipboard"])
|
||||||
|
.stdin(std::process::Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.or_else(|_| {
|
||||||
|
Command::new("xsel")
|
||||||
|
.args(["--clipboard", "--input"])
|
||||||
|
.stdin(std::process::Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
})
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
if let Some(stdin) = child.stdin.as_mut() {
|
||||||
|
stdin.write_all(text.as_bytes())
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
{
|
||||||
|
use std::io::Write;
|
||||||
|
let mut child = Command::new("pbcopy")
|
||||||
|
.stdin(std::process::Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
|
||||||
|
if let Some(stdin) = child.stdin.as_mut() {
|
||||||
|
stdin.write_all(text.as_bytes())
|
||||||
|
.map_err(|e| mlua::Error::runtime(e.to_string()))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
})?;
|
||||||
|
clipboard.set("set", clipboard_set_fn)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
system.set("clipboard", clipboard)?;
|
||||||
|
|
||||||
|
// jarvis.system.env(name) - get environment variable (always available)
|
||||||
|
let env_fn = lua.create_function(|_, name: String| {
|
||||||
|
Ok(std::env::var(&name).ok())
|
||||||
|
})?;
|
||||||
|
system.set("env", env_fn)?;
|
||||||
|
|
||||||
|
// jarvis.system.platform - read-only string
|
||||||
|
let platform = if cfg!(target_os = "windows") {
|
||||||
|
"windows"
|
||||||
|
} else if cfg!(target_os = "macos") {
|
||||||
|
"macos"
|
||||||
|
} else if cfg!(target_os = "linux") {
|
||||||
|
"linux"
|
||||||
|
} else {
|
||||||
|
"unknown"
|
||||||
|
};
|
||||||
|
system.set("platform", platform)?;
|
||||||
|
|
||||||
|
jarvis.set("system", system)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
169
crates/jarvis-core/src/lua/engine.rs
Normal file
169
crates/jarvis-core/src/lua/engine.rs
Normal file
|
|
@ -0,0 +1,169 @@
|
||||||
|
use mlua::{Lua, Value, StdLib};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::Duration;
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
use super::sandbox::SandboxLevel;
|
||||||
|
use super::error::LuaError;
|
||||||
|
use super::{CommandContext, CommandResult};
|
||||||
|
use super::api;
|
||||||
|
|
||||||
|
pub struct LuaEngine {
|
||||||
|
lua: Lua,
|
||||||
|
sandbox: SandboxLevel,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
impl LuaEngine {
|
||||||
|
pub fn new(sandbox: SandboxLevel) -> Result<Self, LuaError> {
|
||||||
|
// select which standard libraries to load based on sandbox access level
|
||||||
|
let std_libs = match sandbox {
|
||||||
|
SandboxLevel::Minimal => {
|
||||||
|
StdLib::TABLE | StdLib::STRING | StdLib::MATH
|
||||||
|
}
|
||||||
|
SandboxLevel::Standard => {
|
||||||
|
StdLib::TABLE | StdLib::STRING | StdLib::MATH | StdLib::UTF8
|
||||||
|
}
|
||||||
|
SandboxLevel::Full => {
|
||||||
|
StdLib::TABLE | StdLib::STRING | StdLib::MATH | StdLib::UTF8 | StdLib::OS
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let lua = Lua::new_with(std_libs, mlua::LuaOptions::default())
|
||||||
|
.map_err(|e| LuaError::InitError(e.to_string()))?;
|
||||||
|
|
||||||
|
// remove dangerous globals regardless of sandbox
|
||||||
|
{
|
||||||
|
let globals = lua.globals();
|
||||||
|
|
||||||
|
// always remove these
|
||||||
|
let _ = globals.set("loadfile", Value::Nil);
|
||||||
|
let _ = globals.set("dofile", Value::Nil);
|
||||||
|
let _ = globals.set("load", Value::Nil);
|
||||||
|
let _ = globals.set("loadstring", Value::Nil);
|
||||||
|
|
||||||
|
// remove io unless full sandbox
|
||||||
|
if !matches!(sandbox, SandboxLevel::Full) {
|
||||||
|
let _ = globals.set("io", Value::Nil);
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove os.execute, os.exit, os.setlocale even in full mode
|
||||||
|
// for SECURITY REASONS!!!
|
||||||
|
if matches!(sandbox, SandboxLevel::Full) {
|
||||||
|
if let Ok(os) = globals.get::<mlua::Table>("os") {
|
||||||
|
let _ = os.set("execute", Value::Nil);
|
||||||
|
let _ = os.set("exit", Value::Nil);
|
||||||
|
let _ = os.set("remove", Value::Nil);
|
||||||
|
let _ = os.set("rename", Value::Nil);
|
||||||
|
let _ = os.set("setlocale", Value::Nil);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Self { lua, sandbox })
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register all jarvis APIs
|
||||||
|
fn register_api(&self, context: &CommandContext) -> Result<(), LuaError> {
|
||||||
|
let globals = self.lua.globals();
|
||||||
|
|
||||||
|
// main jarvis table
|
||||||
|
let jarvis = self.lua.create_table()
|
||||||
|
.map_err(|e| LuaError::InitError(e.to_string()))?;
|
||||||
|
|
||||||
|
// always register core APIs
|
||||||
|
api::core::register(&self.lua, &jarvis)?;
|
||||||
|
api::audio::register(&self.lua, &jarvis)?;
|
||||||
|
api::context::register(&self.lua, &jarvis, context)?;
|
||||||
|
|
||||||
|
// sandbox-controlled APIs
|
||||||
|
if self.sandbox.allows_http() {
|
||||||
|
api::http::register(&self.lua, &jarvis)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.sandbox.allows_state() {
|
||||||
|
api::state::register(&self.lua, &jarvis, &context.command_path)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.sandbox.allows_fs() {
|
||||||
|
api::fs::register(&self.lua, &jarvis, &context.command_path, self.sandbox)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
api::system::register(&self.lua, &jarvis, self.sandbox)?;
|
||||||
|
|
||||||
|
globals.set("jarvis", jarvis)
|
||||||
|
.map_err(|e| LuaError::InitError(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Main LUA executor
|
||||||
|
pub fn execute(
|
||||||
|
&self,
|
||||||
|
script_path: &PathBuf,
|
||||||
|
context: CommandContext,
|
||||||
|
timeout: Duration,
|
||||||
|
) -> Result<CommandResult, LuaError> {
|
||||||
|
// register APIs
|
||||||
|
self.register_api(&context)?;
|
||||||
|
|
||||||
|
// load script
|
||||||
|
let script_content = fs::read_to_string(script_path)
|
||||||
|
.map_err(|e| LuaError::LoadError(format!("{}: {}", script_path.display(), e)))?;
|
||||||
|
|
||||||
|
let script_name = script_path.file_name()
|
||||||
|
.unwrap()
|
||||||
|
.to_string_lossy()
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
|
||||||
|
// set up timeout hook
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
self.lua.set_hook(mlua::HookTriggers {
|
||||||
|
every_nth_instruction: Some(1000),
|
||||||
|
..Default::default()
|
||||||
|
}, move |_lua, _debug| {
|
||||||
|
if start.elapsed() > timeout {
|
||||||
|
Err(mlua::Error::runtime("Script timeout"))
|
||||||
|
} else {
|
||||||
|
Ok(mlua::VmState::Continue)
|
||||||
|
}
|
||||||
|
}).map_err(|e| LuaError::InitError(e.to_string()))?;
|
||||||
|
|
||||||
|
// execute script
|
||||||
|
let result = self.lua.load(&script_content)
|
||||||
|
.set_name(&script_name)
|
||||||
|
.eval::<Value>();
|
||||||
|
|
||||||
|
// remove hook
|
||||||
|
let _ = self.lua.remove_hook();
|
||||||
|
|
||||||
|
// result
|
||||||
|
match result {
|
||||||
|
Ok(value) => Ok(self.parse_result(value)),
|
||||||
|
Err(e) => {
|
||||||
|
if e.to_string().contains("timeout") {
|
||||||
|
Err(LuaError::Timeout)
|
||||||
|
} else {
|
||||||
|
Err(LuaError::RuntimeError(e.to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse Lua return value into CommandResult
|
||||||
|
fn parse_result(&self, value: Value) -> CommandResult {
|
||||||
|
match value {
|
||||||
|
// return { chain = false }
|
||||||
|
Value::Table(t) => {
|
||||||
|
let chain = t.get::<bool>("chain").unwrap_or(true);
|
||||||
|
CommandResult { chain }
|
||||||
|
}
|
||||||
|
// return false (shorthand for no chain)
|
||||||
|
Value::Boolean(chain) => CommandResult { chain },
|
||||||
|
// return nil or no return = chain
|
||||||
|
_ => CommandResult::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
49
crates/jarvis-core/src/lua/error.rs
Normal file
49
crates/jarvis-core/src/lua/error.rs
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
use std::fmt;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum LuaError {
|
||||||
|
// Failed to create Lua VM
|
||||||
|
InitError(String),
|
||||||
|
|
||||||
|
// Failed to load script file
|
||||||
|
LoadError(String),
|
||||||
|
|
||||||
|
// Script execution error
|
||||||
|
RuntimeError(String),
|
||||||
|
|
||||||
|
// Script exceeded timeout
|
||||||
|
Timeout,
|
||||||
|
|
||||||
|
// Sandbox violation
|
||||||
|
SandboxViolation(String),
|
||||||
|
|
||||||
|
// IO error
|
||||||
|
IoError(std::io::Error),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for LuaError {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
match self {
|
||||||
|
LuaError::InitError(msg) => write!(f, "Lua init error: {}", msg),
|
||||||
|
LuaError::LoadError(msg) => write!(f, "Script load error: {}", msg),
|
||||||
|
LuaError::RuntimeError(msg) => write!(f, "Runtime error: {}", msg),
|
||||||
|
LuaError::Timeout => write!(f, "Script timeout"),
|
||||||
|
LuaError::SandboxViolation(msg) => write!(f, "Sandbox violation: {}", msg),
|
||||||
|
LuaError::IoError(e) => write!(f, "IO error: {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for LuaError {}
|
||||||
|
|
||||||
|
impl From<mlua::Error> for LuaError {
|
||||||
|
fn from(e: mlua::Error) -> Self {
|
||||||
|
LuaError::RuntimeError(e.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<std::io::Error> for LuaError {
|
||||||
|
fn from(e: std::io::Error) -> Self {
|
||||||
|
LuaError::IoError(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
65
crates/jarvis-core/src/lua/sandbox.rs
Normal file
65
crates/jarvis-core/src/lua/sandbox.rs
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
// Sandbox level controlling what APIs are available
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub enum SandboxLevel {
|
||||||
|
// Minimal: only core APIs (log, speak, audio, context)
|
||||||
|
Minimal,
|
||||||
|
|
||||||
|
// Standard: + http, state, fs (command folder only)
|
||||||
|
Standard,
|
||||||
|
|
||||||
|
// Full: + system.exec, expanded fs access
|
||||||
|
Full,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SandboxLevel {
|
||||||
|
pub fn from_str(s: &str) -> Self {
|
||||||
|
match s.to_lowercase().as_str() {
|
||||||
|
"minimal" => SandboxLevel::Minimal,
|
||||||
|
"full" => SandboxLevel::Full,
|
||||||
|
_ => SandboxLevel::Standard,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can use HTTP API
|
||||||
|
pub fn allows_http(&self) -> bool {
|
||||||
|
matches!(self, SandboxLevel::Standard | SandboxLevel::Full)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can use persistent state API
|
||||||
|
pub fn allows_state(&self) -> bool {
|
||||||
|
matches!(self, SandboxLevel::Standard | SandboxLevel::Full)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can use file system API
|
||||||
|
pub fn allows_fs(&self) -> bool {
|
||||||
|
matches!(self, SandboxLevel::Standard | SandboxLevel::Full)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can write files
|
||||||
|
pub fn allows_fs_write(&self) -> bool {
|
||||||
|
matches!(self, SandboxLevel::Standard | SandboxLevel::Full)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can execute system commands
|
||||||
|
pub fn allows_exec(&self) -> bool {
|
||||||
|
matches!(self, SandboxLevel::Full)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can access clipboard write
|
||||||
|
pub fn allows_clipboard_write(&self) -> bool {
|
||||||
|
matches!(self, SandboxLevel::Full)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can access paths outside command folder
|
||||||
|
pub fn allows_expanded_paths(&self) -> bool {
|
||||||
|
matches!(self, SandboxLevel::Full)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for SandboxLevel {
|
||||||
|
fn default() -> Self {
|
||||||
|
SandboxLevel::Standard
|
||||||
|
}
|
||||||
|
}
|
||||||
35
crates/jarvis-core/src/lua/structs.rs
Normal file
35
crates/jarvis-core/src/lua/structs.rs
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
use std::{collections::HashMap, path::PathBuf};
|
||||||
|
|
||||||
|
use crate::commands::SlotValue;
|
||||||
|
|
||||||
|
// Context passed to Lua scripts
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct CommandContext {
|
||||||
|
// The phrase that triggered the command
|
||||||
|
pub phrase: String,
|
||||||
|
|
||||||
|
// Command ID
|
||||||
|
pub command_id: String,
|
||||||
|
|
||||||
|
// Path to command folder
|
||||||
|
pub command_path: PathBuf,
|
||||||
|
|
||||||
|
// Current language
|
||||||
|
pub language: String,
|
||||||
|
|
||||||
|
// Slots
|
||||||
|
pub slots: Option<HashMap<String, SlotValue>>
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result returned from Lua script execution
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct CommandResult {
|
||||||
|
// Whether to continue chaining commands
|
||||||
|
pub chain: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for CommandResult {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self { chain: true }
|
||||||
|
}
|
||||||
|
}
|
||||||
112
crates/jarvis-core/src/lua/tests.rs
Normal file
112
crates/jarvis-core/src/lua/tests.rs
Normal file
|
|
@ -0,0 +1,112 @@
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::lua::{CommandContext, LuaError, SandboxLevel, execute};
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::Duration;
|
||||||
|
use tempfile::tempdir;
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
fn create_test_context(cmd_path: PathBuf) -> CommandContext {
|
||||||
|
CommandContext {
|
||||||
|
phrase: "test phrase".to_string(),
|
||||||
|
command_id: "test_cmd".to_string(),
|
||||||
|
command_path: cmd_path,
|
||||||
|
language: "en".to_string(),
|
||||||
|
slots: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_minimal_sandbox() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let script_path = dir.path().join("test.lua");
|
||||||
|
|
||||||
|
fs::write(&script_path, r#"
|
||||||
|
jarvis.log("info", "test log")
|
||||||
|
return { chain = false }
|
||||||
|
"#).unwrap();
|
||||||
|
|
||||||
|
let context = create_test_context(dir.path().to_path_buf());
|
||||||
|
let result = execute(
|
||||||
|
&script_path,
|
||||||
|
context,
|
||||||
|
SandboxLevel::Minimal,
|
||||||
|
Duration::from_secs(5),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(result.is_ok());
|
||||||
|
assert_eq!(result.unwrap().chain, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_state_persistence() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let script_path = dir.path().join("test.lua");
|
||||||
|
|
||||||
|
// first run - set state
|
||||||
|
fs::write(&script_path, r#"
|
||||||
|
jarvis.state.set("key", "value")
|
||||||
|
return true
|
||||||
|
"#).unwrap();
|
||||||
|
|
||||||
|
let context = create_test_context(dir.path().to_path_buf());
|
||||||
|
execute(&script_path, context, SandboxLevel::Standard, Duration::from_secs(5)).unwrap();
|
||||||
|
|
||||||
|
// second run - read state
|
||||||
|
fs::write(&script_path, r#"
|
||||||
|
local val = jarvis.state.get("key")
|
||||||
|
if val == "value" then
|
||||||
|
return true
|
||||||
|
else
|
||||||
|
error("State not persisted")
|
||||||
|
end
|
||||||
|
"#).unwrap();
|
||||||
|
|
||||||
|
let context = create_test_context(dir.path().to_path_buf());
|
||||||
|
let result = execute(&script_path, context, SandboxLevel::Standard, Duration::from_secs(5));
|
||||||
|
|
||||||
|
assert!(result.is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_timeout() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let script_path = dir.path().join("test.lua");
|
||||||
|
|
||||||
|
fs::write(&script_path, r#"
|
||||||
|
while true do end
|
||||||
|
"#).unwrap();
|
||||||
|
|
||||||
|
let context = create_test_context(dir.path().to_path_buf());
|
||||||
|
let result = execute(
|
||||||
|
&script_path,
|
||||||
|
context,
|
||||||
|
SandboxLevel::Minimal,
|
||||||
|
Duration::from_millis(100),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(matches!(result, Err(LuaError::Timeout)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sandbox_fs_escape() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let script_path = dir.path().join("test.lua");
|
||||||
|
|
||||||
|
fs::write(&script_path, r#"
|
||||||
|
local ok, err = pcall(function()
|
||||||
|
jarvis.fs.read("../../../etc/passwd")
|
||||||
|
end)
|
||||||
|
if ok then
|
||||||
|
error("Should have been blocked")
|
||||||
|
end
|
||||||
|
return true
|
||||||
|
"#).unwrap();
|
||||||
|
|
||||||
|
let context = create_test_context(dir.path().to_path_buf());
|
||||||
|
let result = execute(&script_path, context, SandboxLevel::Standard, Duration::from_secs(5));
|
||||||
|
|
||||||
|
assert!(result.is_ok());
|
||||||
|
}
|
||||||
|
}
|
||||||
67
crates/jarvis-core/src/models.rs
Normal file
67
crates/jarvis-core/src/models.rs
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
mod registry;
|
||||||
|
mod catalog;
|
||||||
|
pub mod structs;
|
||||||
|
pub mod loaders;
|
||||||
|
|
||||||
|
pub mod vosk_models;
|
||||||
|
pub mod gliner_models;
|
||||||
|
|
||||||
|
// re-export loaders
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub use loaders::embedding;
|
||||||
|
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub use loaders::gliner;
|
||||||
|
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub use loaders::ort_model;
|
||||||
|
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub use loaders::intent_classifier;
|
||||||
|
|
||||||
|
#[cfg(feature = "vosk")]
|
||||||
|
pub use loaders::vosk;
|
||||||
|
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
pub use loaders::nnnoiseless;
|
||||||
|
|
||||||
|
pub use registry::ModelRegistry;
|
||||||
|
pub use structs::{Task, ModelDef, BackendOption};
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
use crate::APP_DIR;
|
||||||
|
|
||||||
|
pub const MODELS_PATH: &str = "resources/models";
|
||||||
|
|
||||||
|
static REGISTRY: OnceCell<ModelRegistry> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), String> {
|
||||||
|
if REGISTRY.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let registry = ModelRegistry::new();
|
||||||
|
|
||||||
|
let models_dir = APP_DIR.join(MODELS_PATH);
|
||||||
|
let models = catalog::scan_models(&models_dir);
|
||||||
|
info!("Found {} model(s) in {:?}", models.len(), models_dir);
|
||||||
|
registry.set_catalog(models);
|
||||||
|
|
||||||
|
REGISTRY.set(registry)
|
||||||
|
.map_err(|_| "Models registry already initialized".to_string())?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn registry() -> &'static ModelRegistry {
|
||||||
|
REGISTRY.get().expect("Models registry not initialized - call models::init() first")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_options(task: Task) -> Vec<BackendOption> {
|
||||||
|
registry().with_catalog(|models| catalog::get_options(task, models))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_valid_backend(task: Task, backend_id: &str) -> bool {
|
||||||
|
registry().with_catalog(|models| catalog::is_valid_backend(task, backend_id, models))
|
||||||
|
}
|
||||||
140
crates/jarvis-core/src/models/catalog.rs
Normal file
140
crates/jarvis-core/src/models/catalog.rs
Normal file
|
|
@ -0,0 +1,140 @@
|
||||||
|
use std::fs;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use super::structs::{Task, ModelDef, BackendOption};
|
||||||
|
|
||||||
|
// scan the models directory for folders containing model.toml
|
||||||
|
pub fn scan_models(models_dir: &Path) -> Vec<ModelDef> {
|
||||||
|
let mut models = Vec::new();
|
||||||
|
|
||||||
|
if !models_dir.exists() {
|
||||||
|
warn!("Models directory not found: {:?}", models_dir);
|
||||||
|
return models;
|
||||||
|
}
|
||||||
|
|
||||||
|
let entries = match fs::read_dir(models_dir) {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Failed to read models dir: {}", e);
|
||||||
|
return models;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let path = entry.path();
|
||||||
|
if !path.is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let toml_path = path.join("model.toml");
|
||||||
|
if !toml_path.exists() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
match load_model_def(&toml_path, &path) {
|
||||||
|
Ok(def) => {
|
||||||
|
info!("Found model: {} ({}) - tasks: {:?}", def.name, def.id, def.tasks);
|
||||||
|
models.push(def);
|
||||||
|
}
|
||||||
|
Err(e) => warn!("Failed to load model from {:?}: {}", path, e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
models
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_model_def(toml_path: &Path, model_dir: &Path) -> Result<ModelDef, String> {
|
||||||
|
let content = fs::read_to_string(toml_path)
|
||||||
|
.map_err(|e| format!("read error: {}", e))?;
|
||||||
|
|
||||||
|
let parsed: ModelToml = toml::from_str(&content)
|
||||||
|
.map_err(|e| format!("parse error: {}", e))?;
|
||||||
|
|
||||||
|
let mut def = parsed.model;
|
||||||
|
def.path = model_dir.to_path_buf();
|
||||||
|
|
||||||
|
Ok(def)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
struct ModelToml {
|
||||||
|
model: ModelDef,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Code backends per task
|
||||||
|
pub fn code_backends(task: Task) -> Vec<BackendOption> {
|
||||||
|
match task {
|
||||||
|
Task::Intent => vec![
|
||||||
|
BackendOption {
|
||||||
|
id: "intent-classifier".into(),
|
||||||
|
name: "Intent Classifier".into(),
|
||||||
|
model_id: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
Task::Slots => vec![],
|
||||||
|
Task::Vad => vec![
|
||||||
|
BackendOption {
|
||||||
|
id: "energy".into(),
|
||||||
|
name: "Energy-based".into(),
|
||||||
|
model_id: None,
|
||||||
|
},
|
||||||
|
BackendOption {
|
||||||
|
id: "nnnoiseless".into(),
|
||||||
|
name: "Nnnoiseless".into(),
|
||||||
|
model_id: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
Task::NoiseSuppression => vec![
|
||||||
|
BackendOption {
|
||||||
|
id: "nnnoiseless".into(),
|
||||||
|
name: "Nnnoiseless".into(),
|
||||||
|
model_id: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
Task::Stt => vec![
|
||||||
|
BackendOption {
|
||||||
|
id: "vosk".into(),
|
||||||
|
name: "Vosk".into(),
|
||||||
|
model_id: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// get all available options for a task:
|
||||||
|
// "none" first, then code backends, then AI models from catalog
|
||||||
|
pub fn get_options(task: Task, models: &[ModelDef]) -> Vec<BackendOption> {
|
||||||
|
let mut options = vec![
|
||||||
|
BackendOption {
|
||||||
|
id: "none".into(),
|
||||||
|
name: "Disabled".into(),
|
||||||
|
model_id: None,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
options.extend(code_backends(task));
|
||||||
|
|
||||||
|
for model in models {
|
||||||
|
if model.tasks.contains(&task) {
|
||||||
|
options.push(BackendOption {
|
||||||
|
id: model.id.clone(),
|
||||||
|
name: model.name.clone(),
|
||||||
|
model_id: Some(model.id.clone()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
options
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_valid_backend(task: Task, backend_id: &str, models: &[ModelDef]) -> bool {
|
||||||
|
if backend_id == "none" {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if code_backends(task).iter().any(|b| b.id == backend_id) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
models.iter().any(|m| m.id == backend_id && m.tasks.contains(&task))
|
||||||
|
}
|
||||||
130
crates/jarvis-core/src/models/gliner_models.rs
Normal file
130
crates/jarvis-core/src/models/gliner_models.rs
Normal file
|
|
@ -0,0 +1,130 @@
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::fs;
|
||||||
|
use crate::APP_DIR;
|
||||||
|
|
||||||
|
const GLINER_DIRS: &[&str] = &["gliner_small-v2.1", "gliner_multi-v2.1"];
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct GlinerModelVariant {
|
||||||
|
// type id stored in settings, e.g. "int8", "fp16", "full"
|
||||||
|
pub value: String,
|
||||||
|
// shown in dropdown, e.g. "int8 (174MB / 332MB)"
|
||||||
|
pub display_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// scan both model dirs and return a deduplicated list of model types
|
||||||
|
pub fn scan_gliner_variants() -> Vec<GlinerModelVariant> {
|
||||||
|
let base = APP_DIR.join("resources").join("models");
|
||||||
|
|
||||||
|
// collect: type -> { dir_name -> size_mb }
|
||||||
|
let mut types: HashMap<String, HashMap<String, u64>> = HashMap::new();
|
||||||
|
|
||||||
|
for dir_name in GLINER_DIRS {
|
||||||
|
let onnx_dir = base.join(dir_name).join("onnx");
|
||||||
|
if !onnx_dir.exists() { continue; }
|
||||||
|
|
||||||
|
let entries = match fs::read_dir(&onnx_dir) {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let path = entry.path();
|
||||||
|
let file_name = match path.file_name().and_then(|n| n.to_str()) {
|
||||||
|
Some(n) if n.ends_with(".onnx") => n.to_string(),
|
||||||
|
_ => continue,
|
||||||
|
};
|
||||||
|
|
||||||
|
let variant_type = file_name_to_type(&file_name);
|
||||||
|
let size_mb = fs::metadata(&path).map(|m| m.len()).unwrap_or(0) / (1024 * 1024);
|
||||||
|
|
||||||
|
types.entry(variant_type)
|
||||||
|
.or_default()
|
||||||
|
.insert(dir_name.to_string(), size_mb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut result: Vec<GlinerModelVariant> = types.into_iter().map(|(variant, sizes)| {
|
||||||
|
let size_str = build_size_string(&sizes);
|
||||||
|
let label = if variant == "full" { "Full".to_string() } else { variant.clone() };
|
||||||
|
GlinerModelVariant {
|
||||||
|
display_name: format!("{} ({})", label, size_str),
|
||||||
|
value: variant,
|
||||||
|
}
|
||||||
|
}).collect();
|
||||||
|
|
||||||
|
// sort: full first, then alphabetically
|
||||||
|
result.sort_by(|a, b| {
|
||||||
|
let a_full = a.value == "full";
|
||||||
|
let b_full = b.value == "full";
|
||||||
|
b_full.cmp(&a_full).then_with(|| a.value.cmp(&b.value))
|
||||||
|
});
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
// "model.onnx" -> "full", "model_int8.onnx" -> "int8"
|
||||||
|
fn file_name_to_type(name: &str) -> String {
|
||||||
|
let stem = name.strip_suffix(".onnx").unwrap_or(name);
|
||||||
|
if stem == "model" {
|
||||||
|
"full".to_string()
|
||||||
|
} else if let Some(variant) = stem.strip_prefix("model_") {
|
||||||
|
variant.to_string()
|
||||||
|
} else {
|
||||||
|
stem.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// build size display: "174MB" if only one dir, "small: 174MB / multi: 332MB" if both
|
||||||
|
fn build_size_string(sizes: &HashMap<String, u64>) -> String {
|
||||||
|
if sizes.len() == 1 {
|
||||||
|
let (dir, mb) = sizes.iter().next().unwrap();
|
||||||
|
let short = short_dir_name(dir);
|
||||||
|
return format!("{}: {}MB", short, mb);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut parts: Vec<String> = Vec::new();
|
||||||
|
for dir_name in GLINER_DIRS {
|
||||||
|
if let Some(mb) = sizes.get(*dir_name) {
|
||||||
|
parts.push(format!("{}: {}MB", short_dir_name(dir_name), mb));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parts.join(" / ")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn short_dir_name(dir: &str) -> &str {
|
||||||
|
if dir.contains("small") { "small" }
|
||||||
|
else if dir.contains("multi") { "multi" }
|
||||||
|
else { dir }
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolve variant type + language into actual file path
|
||||||
|
// returns (model_dir_path, onnx_file_name) or None
|
||||||
|
pub fn resolve_model(variant: &str, language: &str) -> Option<(std::path::PathBuf, String)> {
|
||||||
|
let base = APP_DIR.join("resources").join("models");
|
||||||
|
let file_name = type_to_file_name(variant);
|
||||||
|
|
||||||
|
// pick dir based on language
|
||||||
|
let preferred: &[&str] = match language {
|
||||||
|
"en" => &["gliner_small-v2.1", "gliner_multi-v2.1"],
|
||||||
|
_ => &["gliner_multi-v2.1", "gliner_small-v2.1"],
|
||||||
|
};
|
||||||
|
|
||||||
|
for dir_name in preferred {
|
||||||
|
let path = base.join(dir_name).join("onnx").join(&file_name);
|
||||||
|
if path.exists() {
|
||||||
|
return Some((base.join(dir_name), file_name));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
// "full" -> "model.onnx", "int8" -> "model_int8.onnx"
|
||||||
|
fn type_to_file_name(variant: &str) -> String {
|
||||||
|
if variant == "full" || variant.is_empty() {
|
||||||
|
"model.onnx".to_string()
|
||||||
|
} else {
|
||||||
|
format!("model_{}.onnx", variant)
|
||||||
|
}
|
||||||
|
}
|
||||||
47
crates/jarvis-core/src/models/loaders/embedding.rs
Normal file
47
crates/jarvis-core/src/models/loaders/embedding.rs
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
// fastembed embedding model (all-MiniLM-L6-v2, paraphrase-multilingual, etc.)
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use fastembed::{TextEmbedding, UserDefinedEmbeddingModel, TokenizerFiles, Pooling, QuantizationMode, OutputKey};
|
||||||
|
|
||||||
|
use crate::models::registry::ModelRegistry;
|
||||||
|
|
||||||
|
pub struct EmbeddingModel {
|
||||||
|
pub embedding: Mutex<TextEmbedding>,
|
||||||
|
}
|
||||||
|
|
||||||
|
// fastembed uses ORT internally which is thread-safe
|
||||||
|
unsafe impl Send for EmbeddingModel {}
|
||||||
|
unsafe impl Sync for EmbeddingModel {}
|
||||||
|
|
||||||
|
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<EmbeddingModel>, String> {
|
||||||
|
registry.get_or_load::<EmbeddingModel>(model_id, |def| {
|
||||||
|
let model_dir = &def.path;
|
||||||
|
|
||||||
|
info!("Loading embedding model from: {}", model_dir.display());
|
||||||
|
|
||||||
|
let user_model = UserDefinedEmbeddingModel {
|
||||||
|
onnx_file: std::fs::read(model_dir.join("model.onnx"))
|
||||||
|
.map_err(|e| format!("Failed to read model.onnx: {}", e))?,
|
||||||
|
tokenizer_files: TokenizerFiles {
|
||||||
|
tokenizer_file: std::fs::read(model_dir.join("tokenizer.json"))
|
||||||
|
.map_err(|e| format!("Failed to read tokenizer.json: {}", e))?,
|
||||||
|
config_file: std::fs::read(model_dir.join("config.json"))
|
||||||
|
.map_err(|e| format!("Failed to read config.json: {}", e))?,
|
||||||
|
special_tokens_map_file: std::fs::read(model_dir.join("special_tokens_map.json"))
|
||||||
|
.map_err(|e| format!("Failed to read special_tokens_map.json: {}", e))?,
|
||||||
|
tokenizer_config_file: std::fs::read(model_dir.join("tokenizer_config.json"))
|
||||||
|
.map_err(|e| format!("Failed to read tokenizer_config.json: {}", e))?,
|
||||||
|
},
|
||||||
|
pooling: Some(Pooling::Mean),
|
||||||
|
quantization: QuantizationMode::None,
|
||||||
|
output_key: Some(OutputKey::ByName("last_hidden_state")),
|
||||||
|
};
|
||||||
|
|
||||||
|
let model = TextEmbedding::try_new_from_user_defined(user_model, Default::default())
|
||||||
|
.map_err(|e| format!("Failed to load embedding model: {}", e))?;
|
||||||
|
|
||||||
|
info!("Embedding model loaded: {}", def.name);
|
||||||
|
Ok(EmbeddingModel { embedding: Mutex::new(model) })
|
||||||
|
})
|
||||||
|
}
|
||||||
51
crates/jarvis-core/src/models/loaders/gliner.rs
Normal file
51
crates/jarvis-core/src/models/loaders/gliner.rs
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
// GLiNER model for named entity recognition / slot extraction
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use regex::Regex;
|
||||||
|
use tokenizers::Tokenizer;
|
||||||
|
|
||||||
|
use crate::models::registry::ModelRegistry;
|
||||||
|
|
||||||
|
const WORD_REGEX: &str = r"\w+(?:[-_]\w+)*|\S";
|
||||||
|
|
||||||
|
pub struct GlinerModel {
|
||||||
|
pub session: Mutex<ort::session::Session>,
|
||||||
|
pub tokenizer: Tokenizer,
|
||||||
|
pub splitter: Regex,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe impl Send for GlinerModel {}
|
||||||
|
unsafe impl Sync for GlinerModel {}
|
||||||
|
|
||||||
|
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<GlinerModel>, String> {
|
||||||
|
registry.get_or_load::<GlinerModel>(model_id, |def| {
|
||||||
|
let model_dir = &def.path;
|
||||||
|
|
||||||
|
// GLiNER models keep onnx files in an "onnx" subfolder
|
||||||
|
let onnx_dir = model_dir.join("onnx");
|
||||||
|
let model_path = if onnx_dir.exists() {
|
||||||
|
onnx_dir.join("model.onnx")
|
||||||
|
} else {
|
||||||
|
model_dir.join("model.onnx")
|
||||||
|
};
|
||||||
|
|
||||||
|
let tokenizer_path = model_dir.join("tokenizer.json");
|
||||||
|
|
||||||
|
info!("Loading GLiNER model from: {}", model_dir.display());
|
||||||
|
|
||||||
|
let session = ort::session::Session::builder()
|
||||||
|
.map_err(|e| format!("Failed to create ORT session builder: {}", e))?
|
||||||
|
.commit_from_file(&model_path)
|
||||||
|
.map_err(|e| format!("Failed to load ONNX model: {}", e))?;
|
||||||
|
|
||||||
|
let tokenizer = Tokenizer::from_file(&tokenizer_path)
|
||||||
|
.map_err(|e| format!("Failed to load tokenizer: {}", e))?;
|
||||||
|
|
||||||
|
let splitter = Regex::new(WORD_REGEX)
|
||||||
|
.map_err(|e| format!("Failed to compile word regex: {}", e))?;
|
||||||
|
|
||||||
|
info!("GLiNER model loaded: {}", def.name);
|
||||||
|
Ok(GlinerModel { session: Mutex::new(session), tokenizer, splitter })
|
||||||
|
})
|
||||||
|
}
|
||||||
30
crates/jarvis-core/src/models/loaders/intent_classifier.rs
Normal file
30
crates/jarvis-core/src/models/loaders/intent_classifier.rs
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
// intent-classifier crate wrapper
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use intent_classifier::IntentClassifier;
|
||||||
|
|
||||||
|
use crate::models::registry::ModelRegistry;
|
||||||
|
|
||||||
|
pub struct IntentClassifierModel {
|
||||||
|
pub classifier: IntentClassifier,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe impl Send for IntentClassifierModel {}
|
||||||
|
unsafe impl Sync for IntentClassifierModel {}
|
||||||
|
|
||||||
|
// init is async (IntentClassifier::new().await), so we create it
|
||||||
|
// outside the registry and insert it after
|
||||||
|
pub async fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<IntentClassifierModel>, String> {
|
||||||
|
if let Some(existing) = registry.get::<IntentClassifierModel>(model_id) {
|
||||||
|
info!("IntentClassifier '{}' already loaded, reusing", model_id);
|
||||||
|
return Ok(existing);
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Initializing IntentClassifier...");
|
||||||
|
|
||||||
|
let classifier = IntentClassifier::new().await
|
||||||
|
.map_err(|e| format!("Failed to init IntentClassifier: {}", e))?;
|
||||||
|
|
||||||
|
info!("IntentClassifier initialized");
|
||||||
|
Ok(registry.insert(model_id, IntentClassifierModel { classifier }))
|
||||||
|
}
|
||||||
12
crates/jarvis-core/src/models/loaders/mod.rs
Normal file
12
crates/jarvis-core/src/models/loaders/mod.rs
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod embedding;
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod gliner;
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod ort_model;
|
||||||
|
#[cfg(feature = "jarvis_app")]
|
||||||
|
pub mod intent_classifier;
|
||||||
|
#[cfg(feature = "vosk")]
|
||||||
|
pub mod vosk;
|
||||||
|
#[cfg(feature = "nnnoiseless")]
|
||||||
|
pub mod nnnoiseless;
|
||||||
110
crates/jarvis-core/src/models/loaders/nnnoiseless.rs
Normal file
110
crates/jarvis-core/src/models/loaders/nnnoiseless.rs
Normal file
|
|
@ -0,0 +1,110 @@
|
||||||
|
// nnnoiseless - used for both noise suppression and VAD.
|
||||||
|
// each consumer needs its own DenoiseState (stateful per-stream),
|
||||||
|
// so this doesn't go through the registry. just centralizes creation.
|
||||||
|
|
||||||
|
use nnnoiseless::DenoiseState;
|
||||||
|
use crate::config;
|
||||||
|
|
||||||
|
// noise suppression instance
|
||||||
|
pub struct NnnoiselessNS {
|
||||||
|
state: Box<DenoiseState<'static>>,
|
||||||
|
buffer: Vec<f32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NnnoiselessNS {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
state: DenoiseState::new(),
|
||||||
|
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn process(&mut self, input: &[i16]) -> Vec<i16> {
|
||||||
|
self.buffer.extend(input.iter().map(|&s| s as f32));
|
||||||
|
|
||||||
|
let frame_size = config::NNNOISELESS_FRAME_SIZE;
|
||||||
|
let full_frames = self.buffer.len() / frame_size;
|
||||||
|
|
||||||
|
if full_frames == 0 {
|
||||||
|
return input.to_vec();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut output: Vec<i16> = Vec::with_capacity(full_frames * frame_size);
|
||||||
|
let mut input_frame = [0.0f32; 480];
|
||||||
|
let mut output_frame = [0.0f32; 480];
|
||||||
|
|
||||||
|
let consumed = full_frames * frame_size;
|
||||||
|
for i in 0..full_frames {
|
||||||
|
let offset = i * frame_size;
|
||||||
|
input_frame.copy_from_slice(&self.buffer[offset..offset + frame_size]);
|
||||||
|
|
||||||
|
let _ = self.state.process_frame(&mut output_frame, &input_frame);
|
||||||
|
|
||||||
|
for &sample in &output_frame {
|
||||||
|
let clamped = sample.clamp(i16::MIN as f32, i16::MAX as f32);
|
||||||
|
output.push(clamped as i16);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keep leftover samples (single drain at the end)
|
||||||
|
self.buffer.drain(..consumed);
|
||||||
|
|
||||||
|
output
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset(&mut self) {
|
||||||
|
self.buffer.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// VAD instance
|
||||||
|
pub struct NnnoiselessVAD {
|
||||||
|
state: Box<DenoiseState<'static>>,
|
||||||
|
buffer: Vec<f32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NnnoiselessVAD {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
state: DenoiseState::new(),
|
||||||
|
buffer: Vec::with_capacity(config::NNNOISELESS_FRAME_SIZE * 2),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn detect(&mut self, input: &[i16]) -> (bool, f32) {
|
||||||
|
self.buffer.extend(input.iter().map(|&s| s as f32));
|
||||||
|
|
||||||
|
let frame_size = config::NNNOISELESS_FRAME_SIZE;
|
||||||
|
let full_frames = self.buffer.len() / frame_size;
|
||||||
|
|
||||||
|
if full_frames == 0 {
|
||||||
|
return (true, 0.5);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut total_vad = 0.0f32;
|
||||||
|
let mut input_frame = [0.0f32; 480];
|
||||||
|
let mut output_frame = [0.0f32; 480];
|
||||||
|
|
||||||
|
let consumed = full_frames * frame_size;
|
||||||
|
for i in 0..full_frames {
|
||||||
|
let offset = i * frame_size;
|
||||||
|
input_frame.copy_from_slice(&self.buffer[offset..offset + frame_size]);
|
||||||
|
|
||||||
|
let vad_prob = self.state.process_frame(&mut output_frame, &input_frame);
|
||||||
|
total_vad += vad_prob;
|
||||||
|
}
|
||||||
|
|
||||||
|
// single drain
|
||||||
|
self.buffer.drain(..consumed);
|
||||||
|
|
||||||
|
let avg_vad = total_vad / full_frames as f32;
|
||||||
|
let is_voice = avg_vad >= config::VAD_NNNOISELESS_THRESHOLD;
|
||||||
|
|
||||||
|
(is_voice, avg_vad)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset(&mut self) {
|
||||||
|
self.state = DenoiseState::new();
|
||||||
|
self.buffer.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
44
crates/jarvis-core/src/models/loaders/ort_model.rs
Normal file
44
crates/jarvis-core/src/models/loaders/ort_model.rs
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
// generic ORT model - session + optional tokenizer.
|
||||||
|
// for models like BERT (tiny, distil, mini) that can serve
|
||||||
|
// multiple tasks (intent, NER, text classification, etc.)
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
use tokenizers::Tokenizer;
|
||||||
|
|
||||||
|
use crate::models::registry::ModelRegistry;
|
||||||
|
|
||||||
|
pub struct OrtModel {
|
||||||
|
pub session: Mutex<ort::session::Session>,
|
||||||
|
pub tokenizer: Option<Tokenizer>,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe impl Send for OrtModel {}
|
||||||
|
unsafe impl Sync for OrtModel {}
|
||||||
|
|
||||||
|
pub fn load(registry: &ModelRegistry, model_id: &str) -> Result<Arc<OrtModel>, String> {
|
||||||
|
registry.get_or_load::<OrtModel>(model_id, |def| {
|
||||||
|
let model_dir = &def.path;
|
||||||
|
let onnx_path = model_dir.join("model.onnx");
|
||||||
|
|
||||||
|
info!("Loading ORT model from: {}", model_dir.display());
|
||||||
|
|
||||||
|
let session = ort::session::Session::builder()
|
||||||
|
.map_err(|e| format!("ORT session builder error: {}", e))?
|
||||||
|
.commit_from_file(&onnx_path)
|
||||||
|
.map_err(|e| format!("Failed to load ONNX model '{}': {}", onnx_path.display(), e))?;
|
||||||
|
|
||||||
|
let tokenizer_path = model_dir.join("tokenizer.json");
|
||||||
|
let tokenizer = if tokenizer_path.exists() {
|
||||||
|
Some(
|
||||||
|
Tokenizer::from_file(&tokenizer_path)
|
||||||
|
.map_err(|e| format!("Failed to load tokenizer: {}", e))?
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
info!("ORT model loaded: {}", def.name);
|
||||||
|
Ok(OrtModel { session: Mutex::new(session), tokenizer })
|
||||||
|
})
|
||||||
|
}
|
||||||
33
crates/jarvis-core/src/models/loaders/vosk.rs
Normal file
33
crates/jarvis-core/src/models/loaders/vosk.rs
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
// vosk speech recognition model
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use vosk::Model;
|
||||||
|
|
||||||
|
use crate::models::registry::ModelRegistry;
|
||||||
|
|
||||||
|
pub struct VoskModel {
|
||||||
|
pub model: Model,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe impl Send for VoskModel {}
|
||||||
|
unsafe impl Sync for VoskModel {}
|
||||||
|
|
||||||
|
// load a vosk model by path through the registry.
|
||||||
|
// vosk models aren't in the catalog (they use their own directory structure),
|
||||||
|
// so we pass the path directly and use model_id for dedup.
|
||||||
|
// @ToDo: Consider moving to catalog
|
||||||
|
pub fn load(registry: &ModelRegistry, model_id: &str, model_path: &str) -> Result<Arc<VoskModel>, String> {
|
||||||
|
// check if already loaded
|
||||||
|
if let Some(existing) = registry.get::<VoskModel>(model_id) {
|
||||||
|
info!("Vosk model '{}' already loaded, reusing", model_id);
|
||||||
|
return Ok(existing);
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Loading Vosk model from: {}", model_path);
|
||||||
|
|
||||||
|
let model = Model::new(model_path)
|
||||||
|
.ok_or_else(|| format!("Failed to load Vosk model from: {}", model_path))?;
|
||||||
|
|
||||||
|
info!("Vosk model loaded: {}", model_id);
|
||||||
|
Ok(registry.insert(model_id, VoskModel { model }))
|
||||||
|
}
|
||||||
108
crates/jarvis-core/src/models/registry.rs
Normal file
108
crates/jarvis-core/src/models/registry.rs
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
use std::any::Any;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use parking_lot::{Mutex, RwLock};
|
||||||
|
|
||||||
|
use super::structs::ModelDef;
|
||||||
|
|
||||||
|
// central model registry. loads models once and shares them between components.
|
||||||
|
// completely type-agnostic
|
||||||
|
pub struct ModelRegistry {
|
||||||
|
loaded: Mutex<HashMap<String, Arc<dyn Any + Send + Sync>>>,
|
||||||
|
catalog: RwLock<Vec<ModelDef>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ModelRegistry {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
loaded: Mutex::new(HashMap::new()),
|
||||||
|
catalog: RwLock::new(Vec::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_catalog(&self, defs: Vec<ModelDef>) {
|
||||||
|
*self.catalog.write() = defs;
|
||||||
|
}
|
||||||
|
|
||||||
|
// read access to catalog without cloning the whole vec
|
||||||
|
pub fn with_catalog<R>(&self, f: impl FnOnce(&[ModelDef]) -> R) -> R {
|
||||||
|
f(&self.catalog.read())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_model_def(&self, id: &str) -> Option<ModelDef> {
|
||||||
|
self.catalog.read().iter().find(|m| m.id == id).cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
// get a loaded model, downcasted to the expected type
|
||||||
|
pub fn get<T: 'static + Send + Sync>(&self, id: &str) -> Option<Arc<T>> {
|
||||||
|
self.loaded.lock()
|
||||||
|
.get(id)?
|
||||||
|
.clone()
|
||||||
|
.downcast::<T>()
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
// get or load a model. if two components request the same id,
|
||||||
|
// the model only loads once.
|
||||||
|
//
|
||||||
|
// the lock is released before calling the loader to avoid deadlocks
|
||||||
|
// if the loader tries to load a dependency through the registry.
|
||||||
|
pub fn get_or_load<T: 'static + Send + Sync>(
|
||||||
|
&self,
|
||||||
|
id: &str,
|
||||||
|
loader: impl FnOnce(&ModelDef) -> Result<T, String>,
|
||||||
|
) -> Result<Arc<T>, String> {
|
||||||
|
// fast path: already loaded
|
||||||
|
if let Some(existing) = self.get::<T>(id) {
|
||||||
|
info!("Model '{}' already loaded, reusing", id);
|
||||||
|
return Ok(existing);
|
||||||
|
}
|
||||||
|
|
||||||
|
// grab model def (releases catalog lock immediately)
|
||||||
|
let def = self.get_model_def(id)
|
||||||
|
.ok_or_else(|| format!("Model '{}' not found in catalog", id))?;
|
||||||
|
|
||||||
|
// run loader without holding any lock
|
||||||
|
info!("Loading model '{}' from {:?}...", id, def.path);
|
||||||
|
let model = loader(&def)?;
|
||||||
|
let arc = Arc::new(model);
|
||||||
|
|
||||||
|
// insert (check again in case another thread loaded it meanwhile)
|
||||||
|
let mut map = self.loaded.lock();
|
||||||
|
if let Some(existing) = map.get(id) {
|
||||||
|
if let Ok(typed) = existing.clone().downcast::<T>() {
|
||||||
|
info!("Model '{}' was loaded by another thread, reusing", id);
|
||||||
|
return Ok(typed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
map.insert(id.to_string(), arc.clone());
|
||||||
|
info!("Model '{}' loaded and registered", id);
|
||||||
|
|
||||||
|
Ok(arc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// insert a model directly (for models not in the catalog,
|
||||||
|
// or loaded through non-standard means like async init)
|
||||||
|
pub fn insert<T: 'static + Send + Sync>(&self, id: &str, model: T) -> Arc<T> {
|
||||||
|
let arc = Arc::new(model);
|
||||||
|
self.loaded.lock().insert(id.to_string(), arc.clone());
|
||||||
|
arc
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn unload(&self, id: &str) -> bool {
|
||||||
|
let removed = self.loaded.lock().remove(id).is_some();
|
||||||
|
if removed {
|
||||||
|
info!("Model '{}' unloaded from registry", id);
|
||||||
|
}
|
||||||
|
removed
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_loaded(&self, id: &str) -> bool {
|
||||||
|
self.loaded.lock().contains_key(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn loaded_ids(&self) -> Vec<String> {
|
||||||
|
self.loaded.lock().keys().cloned().collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
38
crates/jarvis-core/src/models/structs.rs
Normal file
38
crates/jarvis-core/src/models/structs.rs
Normal file
|
|
@ -0,0 +1,38 @@
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use serde::{Serialize, Deserialize};
|
||||||
|
|
||||||
|
// tasks that components can request a backend for
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum Task {
|
||||||
|
Intent,
|
||||||
|
Slots,
|
||||||
|
Vad,
|
||||||
|
NoiseSuppression,
|
||||||
|
Stt,
|
||||||
|
}
|
||||||
|
|
||||||
|
// metadata about a model, parsed from model.toml on disk
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct ModelDef {
|
||||||
|
pub id: String,
|
||||||
|
pub name: String,
|
||||||
|
pub tasks: Vec<Task>,
|
||||||
|
|
||||||
|
#[serde(default)]
|
||||||
|
pub description: String,
|
||||||
|
|
||||||
|
// set at runtime after scanning the folder
|
||||||
|
#[serde(skip)]
|
||||||
|
pub path: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
// a selectable option for a task (shown in UI / stored in settings)
|
||||||
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
pub struct BackendOption {
|
||||||
|
pub id: String,
|
||||||
|
pub name: String,
|
||||||
|
// if Some, this option uses a model from the registry.
|
||||||
|
// if None, it's a code-only backend (like energy VAD) or disabled.
|
||||||
|
pub model_id: Option<String>,
|
||||||
|
}
|
||||||
113
crates/jarvis-core/src/models/vosk_models.rs
Normal file
113
crates/jarvis-core/src/models/vosk_models.rs
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
use std::fs;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use crate::{APP_DIR, config};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct VoskModelInfo {
|
||||||
|
pub name: String, // folder name: "vosk-model-small-ru-0.22"
|
||||||
|
pub path: PathBuf, // full path
|
||||||
|
pub language: String, // extracted from name: "ru"
|
||||||
|
pub size: String, // "small", "large", etc.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scan for available Vosk models
|
||||||
|
pub fn scan_vosk_models() -> Vec<VoskModelInfo> {
|
||||||
|
let models_dir = {
|
||||||
|
APP_DIR.join(config::VOSK_MODELS_PATH)
|
||||||
|
};
|
||||||
|
let mut models = Vec::new();
|
||||||
|
|
||||||
|
info!("VOSK MODELS DIR: {}", models_dir.display());
|
||||||
|
|
||||||
|
if !models_dir.exists() {
|
||||||
|
warn!("Vosk models directory not found: {}", models_dir.display());
|
||||||
|
return models;
|
||||||
|
}
|
||||||
|
|
||||||
|
let entries = match fs::read_dir(models_dir) {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Failed to read vosk models directory: {}", e);
|
||||||
|
return models;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
let entry = match entry {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
|
||||||
|
let path = entry.path();
|
||||||
|
|
||||||
|
// must be a directory
|
||||||
|
if !path.is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// check if it looks like a vosk model (has am/conf/graph folders or similar)
|
||||||
|
if !is_vosk_model(&path) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let name = path.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let (language, size) = parse_model_name(&name);
|
||||||
|
|
||||||
|
models.push(VoskModelInfo {
|
||||||
|
name,
|
||||||
|
path,
|
||||||
|
language,
|
||||||
|
size,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
models.sort_by(|a, b| a.name.cmp(&b.name));
|
||||||
|
models
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if directory looks like a Vosk model
|
||||||
|
fn is_vosk_model(path: &Path) -> bool {
|
||||||
|
// vosk models typically have these subdirectories
|
||||||
|
path.join("am").exists() ||
|
||||||
|
path.join("conf").exists() ||
|
||||||
|
path.join("graph").exists() ||
|
||||||
|
path.join("ivector").exists()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract language and size from model name
|
||||||
|
// e.g., "vosk-model-small-ru-0.22" -> ("ru", "small")
|
||||||
|
fn parse_model_name(name: &str) -> (String, String) {
|
||||||
|
let parts: Vec<&str> = name.split('-').collect();
|
||||||
|
|
||||||
|
let mut language = String::from("unknown");
|
||||||
|
let mut size = String::from("unknown");
|
||||||
|
|
||||||
|
// look for common size indicators
|
||||||
|
for part in &parts {
|
||||||
|
match *part {
|
||||||
|
"small" | "big" | "large" | "lgraph" => size = part.to_string(),
|
||||||
|
// language codes are usually 2 letters
|
||||||
|
s if s.len() == 2 && s.chars().all(|c| c.is_alphabetic()) => {
|
||||||
|
language = s.to_string();
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
(language, size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get model path by name
|
||||||
|
pub fn get_model_path(model_name: &str) -> Option<PathBuf> {
|
||||||
|
let path = Path::new(config::VOSK_MODELS_PATH).join(model_name);
|
||||||
|
if path.exists() && path.is_dir() {
|
||||||
|
Some(path)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
171
crates/jarvis-core/src/recorder.rs
Normal file
171
crates/jarvis-core/src/recorder.rs
Normal file
|
|
@ -0,0 +1,171 @@
|
||||||
|
mod pvrecorder;
|
||||||
|
|
||||||
|
// mod cpal;
|
||||||
|
// mod portaudio;
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
use crate::{config, config::structs::RecorderType, DB};
|
||||||
|
|
||||||
|
static RECORDER_TYPE: OnceCell<RecorderType> = OnceCell::new();
|
||||||
|
static FRAME_LENGTH: OnceCell<u32> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), ()> {
|
||||||
|
// set default recorder type
|
||||||
|
// @TODO. Make it configurable?
|
||||||
|
RECORDER_TYPE.set(config::DEFAULT_RECORDER_TYPE).unwrap();
|
||||||
|
|
||||||
|
// some info
|
||||||
|
info!("Loading recorder ...");
|
||||||
|
info!("Available audio_devices are:\n{:?}", get_audio_devices());
|
||||||
|
|
||||||
|
// load given recorder
|
||||||
|
match RECORDER_TYPE.get().unwrap() {
|
||||||
|
RecorderType::PvRecorder => {
|
||||||
|
// Init Pv Recorder
|
||||||
|
info!("Initializing PvRecorder recording backend.");
|
||||||
|
FRAME_LENGTH.set(512u32).unwrap(); // pvrecorder requires frame buffer of 512
|
||||||
|
let selected_microphone = get_selected_microphone_index();
|
||||||
|
match pvrecorder::init_microphone(
|
||||||
|
selected_microphone,
|
||||||
|
FRAME_LENGTH.get().unwrap().to_owned(),
|
||||||
|
) {
|
||||||
|
false => {
|
||||||
|
error!("Recorder initialization failed.");
|
||||||
|
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
info!(
|
||||||
|
"Recorder initialization success. Listening to microphone ({}): {}",
|
||||||
|
selected_microphone,
|
||||||
|
get_audio_device_name(selected_microphone)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RecorderType::PortAudio => {
|
||||||
|
// Init PortAudio
|
||||||
|
info!("Initializing PortAudio recording backend");
|
||||||
|
todo!();
|
||||||
|
// match portaudio::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
|
||||||
|
// false => {
|
||||||
|
// // Switch to PortAudio recorder
|
||||||
|
// error!("PortAudio audio backend failed.");
|
||||||
|
// },
|
||||||
|
// _ => ()
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
RecorderType::Cpal => {
|
||||||
|
// Init CPAL
|
||||||
|
info!("Initializing CPAL recording backend");
|
||||||
|
todo!();
|
||||||
|
// match cpal::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
|
||||||
|
// false => {
|
||||||
|
// // Switch to CPAL recorder
|
||||||
|
// error!("CPAL audio backend failed.");
|
||||||
|
// },
|
||||||
|
// _ => ()
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn read_microphone(frame_buffer: &mut [i16]) {
|
||||||
|
match RECORDER_TYPE.get().unwrap() {
|
||||||
|
RecorderType::PvRecorder => {
|
||||||
|
pvrecorder::read_microphone(frame_buffer);
|
||||||
|
}
|
||||||
|
RecorderType::PortAudio => {
|
||||||
|
todo!();
|
||||||
|
// portaudio::read_microphone(frame_buffer);
|
||||||
|
}
|
||||||
|
RecorderType::Cpal => {
|
||||||
|
// cpal::read_microphone(frame_buffer);
|
||||||
|
panic!("Cpal should be used via callback assignment");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start_recording() -> Result<(), ()> {
|
||||||
|
match RECORDER_TYPE.get().unwrap() {
|
||||||
|
RecorderType::PvRecorder => {
|
||||||
|
return pvrecorder::start_recording(
|
||||||
|
get_selected_microphone_index(),
|
||||||
|
FRAME_LENGTH.get().unwrap().to_owned(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
RecorderType::PortAudio => {
|
||||||
|
todo!();
|
||||||
|
// portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
|
||||||
|
}
|
||||||
|
RecorderType::Cpal => {
|
||||||
|
todo!();
|
||||||
|
// cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stop_recording() -> Result<(), ()> {
|
||||||
|
match RECORDER_TYPE.get().unwrap() {
|
||||||
|
RecorderType::PvRecorder => pvrecorder::stop_recording(),
|
||||||
|
RecorderType::PortAudio => {
|
||||||
|
todo!();
|
||||||
|
// portaudio::stop_recording();
|
||||||
|
}
|
||||||
|
RecorderType::Cpal => {
|
||||||
|
todo!();
|
||||||
|
// cpal::stop_recording();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_selected_microphone_index() -> i32 {
|
||||||
|
let idx = DB.get().unwrap().read().microphone;
|
||||||
|
|
||||||
|
if idx > 0 {
|
||||||
|
// validate that this microphone is actually in the list
|
||||||
|
let devices = get_audio_devices();
|
||||||
|
if (idx as usize) >= devices.len() {
|
||||||
|
warn!("Microphone index {} not found ({} available), falling back to default",
|
||||||
|
idx, devices.len());
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
idx
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_audio_devices() -> Vec<String> {
|
||||||
|
match RECORDER_TYPE.get() {
|
||||||
|
Some(RecorderType::PvRecorder) => pvrecorder::list_audio_devices(),
|
||||||
|
Some(RecorderType::PortAudio) => {
|
||||||
|
todo!();
|
||||||
|
}
|
||||||
|
Some(RecorderType::Cpal) => {
|
||||||
|
todo!();
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
// not initialized yet, default to pvrecorder
|
||||||
|
pvrecorder::list_audio_devices()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_audio_device_name(idx: i32) -> String {
|
||||||
|
match RECORDER_TYPE.get() {
|
||||||
|
Some(RecorderType::PvRecorder) => pvrecorder::get_audio_device_name(idx),
|
||||||
|
Some(RecorderType::PortAudio) => {
|
||||||
|
todo!();
|
||||||
|
}
|
||||||
|
Some(RecorderType::Cpal) => {
|
||||||
|
todo!();
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
// not initialized yet, default to pvrecorder
|
||||||
|
pvrecorder::get_audio_device_name(idx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
191
crates/jarvis-core/src/recorder/cpal.rs
Normal file
191
crates/jarvis-core/src/recorder/cpal.rs
Normal file
|
|
@ -0,0 +1,191 @@
|
||||||
|
/*
|
||||||
|
Abandoned temporary.
|
||||||
|
Problems with frame size.
|
||||||
|
*/
|
||||||
|
|
||||||
|
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||||||
|
use cpal::{BufferSize, StreamConfig, SampleRate, Host, Device, Stream, SampleFormat};
|
||||||
|
use log::{info, warn, error};
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
|
||||||
|
|
||||||
|
use crate::tauri_commands::cpal_data_callback;
|
||||||
|
|
||||||
|
static HOST: OnceCell<Host> = OnceCell::new();
|
||||||
|
thread_local!(static RECORDER: OnceCell<ArcSwap<Stream>> = OnceCell::new());
|
||||||
|
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
|
||||||
|
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
|
||||||
|
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
|
||||||
|
|
||||||
|
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
|
||||||
|
// init host & frame buffer for the callback
|
||||||
|
if HOST.get().is_none() {
|
||||||
|
HOST.set(cpal::default_host());
|
||||||
|
|
||||||
|
// FRAME_BUFFER.set(Mutex::new(vec![0; FRAME_LENGTH.load(Ordering::SeqCst) as usize]));
|
||||||
|
}
|
||||||
|
|
||||||
|
// init microphone
|
||||||
|
RECORDER.with(|recorder| {
|
||||||
|
match recorder.get().is_none() {
|
||||||
|
true => {
|
||||||
|
if let Some(device) = get_device(device_index as usize) {
|
||||||
|
// store
|
||||||
|
recorder.set(ArcSwap::from_pointee(create_stream(device, frame_length)));
|
||||||
|
|
||||||
|
// remember current configuration
|
||||||
|
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
|
||||||
|
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
|
||||||
|
|
||||||
|
// success
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
false => {
|
||||||
|
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
|
||||||
|
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
|
||||||
|
||
|
||||||
|
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
|
||||||
|
warn!("Selected microphone or frame length was changed, re-initializing ...");
|
||||||
|
// initialize again with new device index
|
||||||
|
if IS_RECORDING.load(Ordering::SeqCst) {
|
||||||
|
stop_recording();
|
||||||
|
}
|
||||||
|
|
||||||
|
// remember new configuration
|
||||||
|
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
|
||||||
|
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
|
||||||
|
|
||||||
|
if let Some(device) = get_device(device_index as usize) {
|
||||||
|
// store
|
||||||
|
recorder.get().unwrap().store(Arc::new(create_stream(device, frame_length)));
|
||||||
|
|
||||||
|
// success
|
||||||
|
return true
|
||||||
|
} else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// success
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_stream(device: Device, frame_length: u32) -> Stream {
|
||||||
|
// get default input stream config
|
||||||
|
// let default_config = device.default_input_config().unwrap();
|
||||||
|
|
||||||
|
// create config for the stream
|
||||||
|
// let config: StreamConfig = StreamConfig {
|
||||||
|
// channels: default_config.channels(),
|
||||||
|
// sample_rate: SampleRate(16000),
|
||||||
|
// buffer_size: BufferSize::Fixed(frame_length)
|
||||||
|
// };
|
||||||
|
|
||||||
|
let config = device
|
||||||
|
.default_input_config()
|
||||||
|
.expect("Failed to load default input config");
|
||||||
|
|
||||||
|
let channels = config.channels();
|
||||||
|
|
||||||
|
let err_fn = move |err| {
|
||||||
|
eprintln!("an error occurred on stream: {}", err);
|
||||||
|
};
|
||||||
|
|
||||||
|
match config.sample_format() {
|
||||||
|
SampleFormat::F32 => device.build_input_stream(
|
||||||
|
&config.into(),
|
||||||
|
move |data: &[f32], info| {
|
||||||
|
cpal_data_callback(data, channels);
|
||||||
|
},
|
||||||
|
err_fn,
|
||||||
|
None
|
||||||
|
),
|
||||||
|
SampleFormat::U16 => device.build_input_stream(
|
||||||
|
&config.into(),
|
||||||
|
move |data: &[u16], info| {
|
||||||
|
cpal_data_callback(data, channels);
|
||||||
|
},
|
||||||
|
err_fn,
|
||||||
|
None
|
||||||
|
),
|
||||||
|
SampleFormat::I16 => device.build_input_stream(
|
||||||
|
&config.into(),
|
||||||
|
move |data: &[i16], info| {
|
||||||
|
cpal_data_callback(data, channels);
|
||||||
|
},
|
||||||
|
err_fn,
|
||||||
|
None
|
||||||
|
),
|
||||||
|
_ => todo!()
|
||||||
|
}.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
|
||||||
|
let mut result = Vec::with_capacity(input_data.len() / 2);
|
||||||
|
result.extend(
|
||||||
|
input_data
|
||||||
|
.chunks_exact(2)
|
||||||
|
.map(|chunk| chunk[0] / 2 + chunk[1] / 2),
|
||||||
|
);
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_device(device_index: usize) -> Option<Device> {
|
||||||
|
if let Some(device) = HOST.get().unwrap().input_devices().expect("Get devices error ...").nth(device_index) {
|
||||||
|
Some(device)
|
||||||
|
} else {
|
||||||
|
if let Some(default) = HOST.get().unwrap().default_input_device() {
|
||||||
|
Some(default)
|
||||||
|
} else {
|
||||||
|
error!("No default input device ...");
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start_recording(device_index: i32, frame_length: u32) {
|
||||||
|
// ensure microphone is initialized
|
||||||
|
init_microphone(device_index, frame_length);
|
||||||
|
|
||||||
|
// start recording
|
||||||
|
RECORDER.with(|recorder| {
|
||||||
|
match recorder.get().unwrap().load().play() {
|
||||||
|
Err(msg) => {
|
||||||
|
error!("[CPAL] Audio stream PLAY error ... {:?}", msg);
|
||||||
|
},
|
||||||
|
_ => ()
|
||||||
|
};
|
||||||
|
|
||||||
|
IS_RECORDING.store(true, Ordering::SeqCst);
|
||||||
|
info!("START recording from microphone ...");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stop_recording() {
|
||||||
|
// ensure microphone is initialized
|
||||||
|
RECORDER.with(|recorder| {
|
||||||
|
if recorder.get().is_some() && IS_RECORDING.load(Ordering::SeqCst) {
|
||||||
|
// pause instead of stop
|
||||||
|
match recorder.get().unwrap().load().pause() {
|
||||||
|
Err(msg) => {
|
||||||
|
error!("[CPAL] Audio stream PAUSE error ... {:?}", msg);
|
||||||
|
},
|
||||||
|
_ => ()
|
||||||
|
};
|
||||||
|
|
||||||
|
IS_RECORDING.store(false, Ordering::SeqCst);
|
||||||
|
info!("STOP recording from microphone ...");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
205
crates/jarvis-core/src/recorder/portaudio.rs
Normal file
205
crates/jarvis-core/src/recorder/portaudio.rs
Normal file
|
|
@ -0,0 +1,205 @@
|
||||||
|
/*
|
||||||
|
Abandoned temporary.
|
||||||
|
*/
|
||||||
|
|
||||||
|
use portaudio as pa;
|
||||||
|
use pa::{DeviceIndex, Stream};
|
||||||
|
use log::{info, warn, error};
|
||||||
|
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
|
||||||
|
|
||||||
|
thread_local!(static RECORDER: OnceCell<ArcSwap<Mutex<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>>>> = OnceCell::new());
|
||||||
|
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
|
||||||
|
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
|
||||||
|
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
|
||||||
|
|
||||||
|
const CHANNELS: i32 = 1;
|
||||||
|
const SAMPLE_RATE: f64 = 16_000.0;
|
||||||
|
|
||||||
|
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
|
||||||
|
RECORDER.with(|r| {
|
||||||
|
match r.get().is_none() {
|
||||||
|
true => {
|
||||||
|
match create_stream(device_index, frame_length) {
|
||||||
|
Ok(stream) => {
|
||||||
|
// store
|
||||||
|
r.set(ArcSwap::from_pointee(Mutex::new(stream)));
|
||||||
|
|
||||||
|
// remember current configuration
|
||||||
|
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
|
||||||
|
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
|
||||||
|
|
||||||
|
// success
|
||||||
|
true
|
||||||
|
},
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
|
||||||
|
|
||||||
|
// fail
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
_ => {
|
||||||
|
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
|
||||||
|
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
|
||||||
|
||
|
||||||
|
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
|
||||||
|
warn!("Selected microphone or frame length was changed, re-initializing ...");
|
||||||
|
// initialize again with new device index
|
||||||
|
if IS_RECORDING.load(Ordering::SeqCst) {
|
||||||
|
// RECORDER.get().unwrap().load().stop().expect("Failed to start audio recording!");
|
||||||
|
stop_recording();
|
||||||
|
}
|
||||||
|
|
||||||
|
// store
|
||||||
|
match create_stream(device_index, frame_length) {
|
||||||
|
Ok(stream) => {
|
||||||
|
// store new stream
|
||||||
|
r.get().unwrap().store(Arc::new(Mutex::new(stream)));
|
||||||
|
|
||||||
|
// remember new configuration
|
||||||
|
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
|
||||||
|
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
|
||||||
|
|
||||||
|
// success
|
||||||
|
return true
|
||||||
|
},
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
|
||||||
|
|
||||||
|
// fail
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// success
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_stream(device_index: i32, frame_length: u32) -> Result<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>, pa::Error> {
|
||||||
|
let pa_recorder: Result<pa::PortAudio, pa::Error> = pa::PortAudio::new();
|
||||||
|
|
||||||
|
match pa_recorder {
|
||||||
|
Ok(pa) => {
|
||||||
|
let input_settings = match get_input_settings(DeviceIndex(device_index as u32), &pa, SAMPLE_RATE, frame_length, CHANNELS) {
|
||||||
|
Ok(settings) => settings,
|
||||||
|
Err(error) => panic!("{}", String::from(error))
|
||||||
|
};
|
||||||
|
|
||||||
|
// Construct a stream with input and output sample types of i16
|
||||||
|
match pa.open_blocking_stream(input_settings) {
|
||||||
|
Ok(strm) => Ok(strm),
|
||||||
|
Err(error) => panic!("{}", error.to_string()),
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(msg) => Err(msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_input_latency(audio_port: &pa::PortAudio, input_index: pa::DeviceIndex) -> Result<f64, String>
|
||||||
|
{
|
||||||
|
let input_device_information = audio_port.device_info(input_index).or_else(|error| Err(String::from(format!("{}", error))));
|
||||||
|
Ok(input_device_information.unwrap().default_low_input_latency)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_input_stream_parameters(input_index: pa::DeviceIndex, latency: f64, channels: i32) -> Result<pa::StreamParameters<i16>, String>
|
||||||
|
{
|
||||||
|
const INTERLEAVED: bool = true;
|
||||||
|
Ok(pa::StreamParameters::<i16>::new(input_index, channels, INTERLEAVED, latency))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_input_settings(input_index: pa::DeviceIndex, audio_port: &pa::PortAudio, sample_rate: f64, frames: u32, channels: i32) -> Result<pa::InputStreamSettings<i16>, String>
|
||||||
|
{
|
||||||
|
Ok(
|
||||||
|
pa::InputStreamSettings::new(
|
||||||
|
(get_input_stream_parameters(
|
||||||
|
input_index,
|
||||||
|
(get_input_latency(
|
||||||
|
&audio_port,
|
||||||
|
input_index,
|
||||||
|
))?,
|
||||||
|
channels
|
||||||
|
))?,
|
||||||
|
sample_rate,
|
||||||
|
frames,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// We'll use this function to wait for read/write availability.
|
||||||
|
fn wait_for_stream<F>(f: F, name: &str) -> u32
|
||||||
|
where
|
||||||
|
F: Fn() -> Result<pa::StreamAvailable, pa::error::Error>,
|
||||||
|
{
|
||||||
|
loop {
|
||||||
|
match f() {
|
||||||
|
Ok(available) => match available {
|
||||||
|
pa::StreamAvailable::Frames(frames) => return frames as u32,
|
||||||
|
pa::StreamAvailable::InputOverflowed => println!("Input stream has overflowed"),
|
||||||
|
pa::StreamAvailable::OutputUnderflowed => {
|
||||||
|
println!("Output stream has underflowed")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(err) => panic!(
|
||||||
|
"An error occurred while waiting for the {} stream: {}",
|
||||||
|
name, err
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn read_microphone(frame_buffer: &mut [i16]) {
|
||||||
|
// ensure microphone is initialized
|
||||||
|
RECORDER.with(|r| {
|
||||||
|
if r.get().is_some() {
|
||||||
|
let cell = r.get().unwrap().load();
|
||||||
|
let mut lock = cell.lock();
|
||||||
|
let stream = lock.as_mut().unwrap();
|
||||||
|
|
||||||
|
// read to frame buffer
|
||||||
|
let in_frames = wait_for_stream(|| stream.read_available(), "Read");
|
||||||
|
|
||||||
|
if in_frames > 0 {
|
||||||
|
// let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
|
||||||
|
// println!("Read {:?} frames from the input stream.", in_frames);
|
||||||
|
|
||||||
|
let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
|
||||||
|
println!("Read: {} (required {})", input_samples.len(), frame_buffer.len());
|
||||||
|
frame_buffer.copy_from_slice(input_samples.chunks(frame_buffer.len()).last().unwrap());
|
||||||
|
}
|
||||||
|
// r.get().unwrap().load().read(frame_buffer).expect("Failed to read audio frame");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start_recording(device_index: i32, frame_length: u32) {
|
||||||
|
// ensure microphone is initialized
|
||||||
|
init_microphone(device_index, frame_length);
|
||||||
|
|
||||||
|
// start recording
|
||||||
|
RECORDER.with(|r| {
|
||||||
|
r.get().unwrap().load().lock().unwrap().start().expect("Failed to start audio recording!");
|
||||||
|
IS_RECORDING.store(true, Ordering::SeqCst);
|
||||||
|
info!("START recording from microphone ...");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stop_recording() {
|
||||||
|
RECORDER.with(|r| {
|
||||||
|
if r.get().is_some() && IS_RECORDING.load(Ordering::SeqCst) {
|
||||||
|
// stop recording
|
||||||
|
let pa = r.get().unwrap().load();
|
||||||
|
r.get().unwrap().load().lock().unwrap().stop().expect("Failed to stop audio recording!");
|
||||||
|
IS_RECORDING.store(false, Ordering::SeqCst);
|
||||||
|
info!("STOP recording from microphone ...");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
136
crates/jarvis-core/src/recorder/pvrecorder.rs
Normal file
136
crates/jarvis-core/src/recorder/pvrecorder.rs
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use pv_recorder::{PvRecorder, PvRecorderBuilder};
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
|
static RECORDER: OnceCell<PvRecorder> = OnceCell::new();
|
||||||
|
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
|
||||||
|
|
||||||
|
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
|
||||||
|
if RECORDER.get().is_some() {
|
||||||
|
return true; // already initialized
|
||||||
|
}
|
||||||
|
|
||||||
|
// initialize
|
||||||
|
let pv_recorder = PvRecorderBuilder::new(frame_length as i32)
|
||||||
|
.device_index(device_index)
|
||||||
|
// .frame_length(frame_length as i32)
|
||||||
|
.init();
|
||||||
|
|
||||||
|
match pv_recorder {
|
||||||
|
Ok(pv) => {
|
||||||
|
// store
|
||||||
|
let _ = RECORDER.set(pv);
|
||||||
|
|
||||||
|
// success
|
||||||
|
true
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Failed to initialize pvrecorder.\nError details: {:?}", msg);
|
||||||
|
|
||||||
|
// fail
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn read_microphone(frame_buffer: &mut [i16]) {
|
||||||
|
// ensure microphone is initialized
|
||||||
|
if RECORDER.get().is_some() {
|
||||||
|
// read to frame buffer
|
||||||
|
|
||||||
|
let frame = RECORDER.get().unwrap().read();
|
||||||
|
|
||||||
|
match frame {
|
||||||
|
Ok(f) => {
|
||||||
|
frame_buffer.copy_from_slice(f.as_slice());
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
// @TODO: Fix? PvRecorder always wait for PCM buffer size of 512.
|
||||||
|
error!("Failed to read audio frame. {:?}", msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start_recording(device_index: i32, frame_length: u32) -> Result<(), ()> {
|
||||||
|
// ensure microphone is initialized
|
||||||
|
init_microphone(device_index, frame_length);
|
||||||
|
|
||||||
|
// start recording
|
||||||
|
match RECORDER.get().unwrap().start() {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("START recording from microphone ...");
|
||||||
|
|
||||||
|
// change recording state
|
||||||
|
IS_RECORDING.store(true, Ordering::SeqCst);
|
||||||
|
|
||||||
|
// success
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Failed to START audio recording: {}", msg);
|
||||||
|
|
||||||
|
// fail
|
||||||
|
Err(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stop_recording() -> Result<(), ()> {
|
||||||
|
// ensure microphone is initialized & recording is in process
|
||||||
|
if RECORDER.get().is_some() && IS_RECORDING.load(Ordering::SeqCst) {
|
||||||
|
// stop recording
|
||||||
|
match RECORDER.get().unwrap().stop() {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("STOP recording from microphone ...");
|
||||||
|
|
||||||
|
// change recording state
|
||||||
|
IS_RECORDING.store(false, Ordering::SeqCst);
|
||||||
|
|
||||||
|
// success
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Err(msg) => {
|
||||||
|
error!("Failed to STOP audio recording: {}", msg);
|
||||||
|
|
||||||
|
// fail
|
||||||
|
return Err(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(()) // if already stopped or not yet initialized
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_audio_devices() -> Vec<String> {
|
||||||
|
let audio_devices = PvRecorderBuilder::default().get_available_devices();
|
||||||
|
match audio_devices {
|
||||||
|
Ok(audio_devices) => audio_devices,
|
||||||
|
Err(err) => {
|
||||||
|
error!("Failed to get audio devices: {}", err);
|
||||||
|
Vec::new()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_audio_device_name(idx: i32) -> String {
|
||||||
|
if idx == -1 {
|
||||||
|
return String::from("System Default");
|
||||||
|
}
|
||||||
|
|
||||||
|
let audio_devices = list_audio_devices();
|
||||||
|
let mut first_device: String = String::new();
|
||||||
|
|
||||||
|
for (_idx, device) in audio_devices.iter().enumerate() {
|
||||||
|
if idx as usize == _idx {
|
||||||
|
return device.to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
if _idx == 0 {
|
||||||
|
first_device = device.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// return first device as default, if none were matched
|
||||||
|
first_device
|
||||||
|
}
|
||||||
58
crates/jarvis-core/src/slots.rs
Normal file
58
crates/jarvis-core/src/slots.rs
Normal file
|
|
@ -0,0 +1,58 @@
|
||||||
|
mod gliner;
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
use crate::commands::{SlotDefinition, SlotValue};
|
||||||
|
use crate::{models, DB};
|
||||||
|
|
||||||
|
static BACKEND: OnceCell<String> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), String> {
|
||||||
|
if BACKEND.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let backend = DB.get()
|
||||||
|
.map(|db| db.read().slots_backend.clone())
|
||||||
|
.unwrap_or_else(|| "none".to_string());
|
||||||
|
|
||||||
|
BACKEND.set(backend.clone()).map_err(|_| "Slot backend already set")?;
|
||||||
|
|
||||||
|
match backend.as_str() {
|
||||||
|
"none" => {
|
||||||
|
info!("Slot extraction disabled");
|
||||||
|
}
|
||||||
|
// any model ID is treated as a GLiNER model for now
|
||||||
|
model_id => {
|
||||||
|
info!("Initializing GLiNER slot extraction with model '{}'.", model_id);
|
||||||
|
let model = models::gliner::load(models::registry(), model_id)?;
|
||||||
|
gliner::init_with_model(model)?;
|
||||||
|
info!("GLiNER slot extraction initialized.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extract(
|
||||||
|
text: &str,
|
||||||
|
slots: &HashMap<String, SlotDefinition>,
|
||||||
|
) -> HashMap<String, SlotValue> {
|
||||||
|
if slots.is_empty() {
|
||||||
|
return HashMap::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
match BACKEND.get().map(|s| s.as_str()).unwrap_or("none") {
|
||||||
|
"none" => HashMap::new(),
|
||||||
|
_ => {
|
||||||
|
match gliner::extract(text, slots) {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(e) => {
|
||||||
|
error!("GLiNER slot extraction failed: {}", e);
|
||||||
|
HashMap::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
395
crates/jarvis-core/src/slots/gliner.rs
Normal file
395
crates/jarvis-core/src/slots/gliner.rs
Normal file
|
|
@ -0,0 +1,395 @@
|
||||||
|
// BASED ON: gline-rs crate source code
|
||||||
|
// https://github.com/fbilhaut/gline-rs
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use ndarray::Array;
|
||||||
|
use ort::value::Tensor;
|
||||||
|
|
||||||
|
use crate::commands::{SlotDefinition, SlotValue};
|
||||||
|
use crate::models::gliner::GlinerModel;
|
||||||
|
|
||||||
|
static MODEL: OnceCell<Arc<GlinerModel>> = OnceCell::new();
|
||||||
|
|
||||||
|
// GLiNER defaults
|
||||||
|
const THRESHOLD: f32 = 0.3;
|
||||||
|
const MAX_WIDTH: usize = 12;
|
||||||
|
const MAX_LENGTH: usize = 512;
|
||||||
|
const MIN_CONFIDENCE: f32 = 0.4;
|
||||||
|
|
||||||
|
pub fn init_with_model(model: Arc<GlinerModel>) -> Result<(), String> {
|
||||||
|
MODEL.set(model).map_err(|_| "GLiNER model already initialized".to_string())?;
|
||||||
|
info!("GLiNER slot extraction ready");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// word splitting
|
||||||
|
|
||||||
|
struct WordToken<'a> {
|
||||||
|
start: usize,
|
||||||
|
end: usize,
|
||||||
|
text: &'a str,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn split_words<'a>(text: &'a str, model: &GlinerModel, limit: Option<usize>) -> Vec<WordToken<'a>> {
|
||||||
|
let mut tokens = Vec::new();
|
||||||
|
for m in model.splitter.find_iter(text) {
|
||||||
|
tokens.push(WordToken {
|
||||||
|
start: m.start(),
|
||||||
|
end: m.end(),
|
||||||
|
text: m.as_str(),
|
||||||
|
});
|
||||||
|
if let Some(lim) = limit {
|
||||||
|
if tokens.len() >= lim { break; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tokens
|
||||||
|
}
|
||||||
|
|
||||||
|
// prompt construction
|
||||||
|
//
|
||||||
|
// GLiNER prompt format:
|
||||||
|
// [<<ENT>>, label1_w1, label1_w2, <<ENT>>, label2_w1, ..., <<SEP>>, word1, word2, ..., wordN]
|
||||||
|
|
||||||
|
fn build_prompt(entities: &[&str], words: &[WordToken]) -> (Vec<String>, usize) {
|
||||||
|
let mut prompt = Vec::with_capacity(entities.len() * 2 + 1 + words.len());
|
||||||
|
|
||||||
|
for entity in entities {
|
||||||
|
prompt.push("<<ENT>>".to_string());
|
||||||
|
prompt.push(entity.to_string());
|
||||||
|
}
|
||||||
|
prompt.push("<<SEP>>".to_string());
|
||||||
|
|
||||||
|
let entities_len = prompt.len();
|
||||||
|
|
||||||
|
for w in words {
|
||||||
|
prompt.push(w.text.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
(prompt, entities_len)
|
||||||
|
}
|
||||||
|
|
||||||
|
// encoding
|
||||||
|
|
||||||
|
struct EncodedBatch {
|
||||||
|
input_ids: ndarray::Array2<i64>,
|
||||||
|
attention_masks: ndarray::Array2<i64>,
|
||||||
|
word_masks: ndarray::Array2<i64>,
|
||||||
|
text_lengths: ndarray::Array2<i64>,
|
||||||
|
num_words: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_single(
|
||||||
|
model: &GlinerModel,
|
||||||
|
entities: &[&str],
|
||||||
|
words: &[WordToken],
|
||||||
|
) -> Result<EncodedBatch, String> {
|
||||||
|
let (prompt, ent_len) = build_prompt(entities, words);
|
||||||
|
let text_word_count = words.len();
|
||||||
|
|
||||||
|
let mut word_encodings: Vec<Vec<u32>> = Vec::with_capacity(prompt.len());
|
||||||
|
let mut total_tokens: usize = 2; // BOS + EOS
|
||||||
|
let mut entity_tokens: usize = 0;
|
||||||
|
|
||||||
|
for (pos, word) in prompt.iter().enumerate() {
|
||||||
|
let encoding = model.tokenizer.encode(word.as_str(), false)
|
||||||
|
.map_err(|e| format!("Tokenizer encode error: {}", e))?;
|
||||||
|
let ids = encoding.get_ids().to_vec();
|
||||||
|
total_tokens += ids.len();
|
||||||
|
if pos < ent_len {
|
||||||
|
entity_tokens += ids.len();
|
||||||
|
}
|
||||||
|
word_encodings.push(ids);
|
||||||
|
}
|
||||||
|
|
||||||
|
let text_offset = entity_tokens + 1;
|
||||||
|
|
||||||
|
if log::log_enabled!(log::Level::Debug) {
|
||||||
|
debug!("GLiNER prompt ({} total, ent_len={}, text_offset={}):", prompt.len(), ent_len, text_offset);
|
||||||
|
for (i, (word, enc)) in prompt.iter().zip(word_encodings.iter()).enumerate() {
|
||||||
|
debug!(" [{}]{} '{}' -> {:?}", i, if i < ent_len { " ENT" } else { " TXT" }, word, enc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut input_ids = Array::zeros((1, total_tokens));
|
||||||
|
let mut attention_masks = Array::zeros((1, total_tokens));
|
||||||
|
let mut word_masks = Array::zeros((1, total_tokens));
|
||||||
|
|
||||||
|
let mut idx: usize = 0;
|
||||||
|
let mut word_id: i64 = 0;
|
||||||
|
|
||||||
|
// BOS
|
||||||
|
input_ids[[0, idx]] = 1;
|
||||||
|
attention_masks[[0, idx]] = 1;
|
||||||
|
idx += 1;
|
||||||
|
|
||||||
|
for word_enc in word_encodings.iter() {
|
||||||
|
for (token_idx, &token_id) in word_enc.iter().enumerate() {
|
||||||
|
input_ids[[0, idx]] = token_id as i64;
|
||||||
|
attention_masks[[0, idx]] = 1;
|
||||||
|
if idx >= text_offset && token_idx == 0 {
|
||||||
|
word_masks[[0, idx]] = word_id;
|
||||||
|
}
|
||||||
|
idx += 1;
|
||||||
|
}
|
||||||
|
if idx >= text_offset {
|
||||||
|
word_id += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EOS
|
||||||
|
input_ids[[0, idx]] = 2;
|
||||||
|
attention_masks[[0, idx]] = 1;
|
||||||
|
|
||||||
|
let mut text_lengths = Array::zeros((1, 1));
|
||||||
|
text_lengths[[0, 0]] = (text_word_count + 1) as i64;
|
||||||
|
|
||||||
|
if log::log_enabled!(log::Level::Debug) {
|
||||||
|
debug!("GLiNER input_ids: {:?}", input_ids.as_slice().unwrap());
|
||||||
|
debug!("GLiNER word_masks: {:?}", word_masks.as_slice().unwrap());
|
||||||
|
debug!("GLiNER text_lengths: {}", text_word_count);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(EncodedBatch {
|
||||||
|
input_ids,
|
||||||
|
attention_masks,
|
||||||
|
word_masks,
|
||||||
|
text_lengths,
|
||||||
|
num_words: text_word_count + 1,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// span tensors
|
||||||
|
|
||||||
|
fn make_span_tensors(num_words: usize, max_width: usize) -> (ndarray::Array3<i64>, ndarray::Array2<bool>) {
|
||||||
|
let num_spans = num_words * max_width;
|
||||||
|
|
||||||
|
let mut span_idx = Array::zeros((1, num_spans, 2));
|
||||||
|
let mut span_mask = Array::from_elem((1, num_spans), false);
|
||||||
|
|
||||||
|
for start in 0..num_words {
|
||||||
|
let remaining = num_words - start;
|
||||||
|
let actual_max = max_width.min(remaining);
|
||||||
|
for width in 0..actual_max {
|
||||||
|
let dim = start * max_width + width;
|
||||||
|
span_idx[[0, dim, 0]] = start as i64;
|
||||||
|
span_idx[[0, dim, 1]] = (start + width) as i64;
|
||||||
|
span_mask[[0, dim]] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
(span_idx, span_mask)
|
||||||
|
}
|
||||||
|
|
||||||
|
// decode + greedy search
|
||||||
|
|
||||||
|
fn sigmoid(x: f32) -> f32 {
|
||||||
|
1.0 / (1.0 + (-x).exp())
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Entity {
|
||||||
|
text: String,
|
||||||
|
label: String,
|
||||||
|
prob: f32,
|
||||||
|
start: usize,
|
||||||
|
end: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_and_search(
|
||||||
|
logits_data: &[f32],
|
||||||
|
logits_shape: &[usize],
|
||||||
|
words: &[WordToken],
|
||||||
|
text: &str,
|
||||||
|
entities: &[&str],
|
||||||
|
max_width: usize,
|
||||||
|
threshold: f32,
|
||||||
|
) -> Vec<Entity> {
|
||||||
|
let num_tokens = words.len();
|
||||||
|
|
||||||
|
let dim_mw = logits_shape.get(2).copied().unwrap_or(0);
|
||||||
|
let dim_e = logits_shape.get(3).copied().unwrap_or(0);
|
||||||
|
|
||||||
|
let mut spans: Vec<Entity> = Vec::new();
|
||||||
|
|
||||||
|
for start in 1..=num_tokens {
|
||||||
|
let max_end = (start + max_width).min(num_tokens + 1);
|
||||||
|
for end in start..max_end {
|
||||||
|
let width = end - start;
|
||||||
|
for (class_idx, &entity_label) in entities.iter().enumerate() {
|
||||||
|
let flat_idx = start * dim_mw * dim_e + width * dim_e + class_idx;
|
||||||
|
if flat_idx >= logits_data.len() { continue; }
|
||||||
|
|
||||||
|
let raw_score = logits_data[flat_idx];
|
||||||
|
let prob = sigmoid(raw_score);
|
||||||
|
if prob >= threshold {
|
||||||
|
let w_start = start - 1;
|
||||||
|
let w_end = end - 1;
|
||||||
|
let start_offset = words[w_start].start;
|
||||||
|
let end_offset = words[w_end].end;
|
||||||
|
let span_text = text[start_offset..end_offset].to_string();
|
||||||
|
spans.push(Entity {
|
||||||
|
text: span_text,
|
||||||
|
label: entity_label.to_string(),
|
||||||
|
prob,
|
||||||
|
start: start_offset,
|
||||||
|
end: end_offset,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spans.sort_unstable_by(|a, b| (a.start, a.end).cmp(&(b.start, b.end)));
|
||||||
|
greedy_flat(spans)
|
||||||
|
}
|
||||||
|
|
||||||
|
// takes ownership, filters in place - no cloning
|
||||||
|
fn greedy_flat(mut spans: Vec<Entity>) -> Vec<Entity> {
|
||||||
|
if spans.len() <= 1 {
|
||||||
|
return spans;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut keep = vec![false; spans.len()];
|
||||||
|
let mut prev = 0usize;
|
||||||
|
|
||||||
|
for next in 1..spans.len() {
|
||||||
|
let no_overlap = spans[next].start >= spans[prev].end
|
||||||
|
|| spans[prev].start >= spans[next].end;
|
||||||
|
|
||||||
|
if no_overlap {
|
||||||
|
keep[prev] = true;
|
||||||
|
prev = next;
|
||||||
|
} else if spans[prev].prob < spans[next].prob {
|
||||||
|
prev = next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
keep[prev] = true;
|
||||||
|
|
||||||
|
let mut idx = 0;
|
||||||
|
spans.retain(|_| { let k = keep[idx]; idx += 1; k });
|
||||||
|
spans
|
||||||
|
}
|
||||||
|
|
||||||
|
// public extract API
|
||||||
|
|
||||||
|
pub fn extract(
|
||||||
|
text: &str,
|
||||||
|
slots: &HashMap<String, SlotDefinition>,
|
||||||
|
) -> Result<HashMap<String, SlotValue>, String> {
|
||||||
|
let model = MODEL.get().ok_or("GLiNER not initialized")?;
|
||||||
|
|
||||||
|
let mut label_to_slots: HashMap<&str, Vec<&str>> = HashMap::new();
|
||||||
|
for (slot_name, def) in slots {
|
||||||
|
if !def.entity.is_empty() {
|
||||||
|
label_to_slots
|
||||||
|
.entry(def.entity.as_str())
|
||||||
|
.or_default()
|
||||||
|
.push(slot_name.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if label_to_slots.is_empty() {
|
||||||
|
return Ok(HashMap::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let labels: Vec<&str> = label_to_slots.keys().copied().collect();
|
||||||
|
|
||||||
|
debug!("GLiNER extract: text='{}', labels={:?}", text, labels);
|
||||||
|
|
||||||
|
let words = split_words(text, &model, Some(MAX_LENGTH));
|
||||||
|
if words.is_empty() {
|
||||||
|
return Ok(HashMap::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let encoded = encode_single(&model, &labels, &words)?;
|
||||||
|
|
||||||
|
let (span_idx, span_mask) = make_span_tensors(encoded.num_words, MAX_WIDTH);
|
||||||
|
|
||||||
|
let t_input_ids = Tensor::from_array(encoded.input_ids).map_err(|e| format!("tensor: {}", e))?;
|
||||||
|
let t_attn = Tensor::from_array(encoded.attention_masks).map_err(|e| format!("tensor: {}", e))?;
|
||||||
|
let t_words = Tensor::from_array(encoded.word_masks).map_err(|e| format!("tensor: {}", e))?;
|
||||||
|
let t_lengths = Tensor::from_array(encoded.text_lengths).map_err(|e| format!("tensor: {}", e))?;
|
||||||
|
let t_span_idx = Tensor::from_array(span_idx).map_err(|e| format!("tensor: {}", e))?;
|
||||||
|
let t_span_mask = Tensor::from_array(span_mask).map_err(|e| format!("tensor: {}", e))?;
|
||||||
|
|
||||||
|
let mut session = model.session.lock();
|
||||||
|
let outputs = session.run(
|
||||||
|
ort::inputs! {
|
||||||
|
"input_ids" => t_input_ids,
|
||||||
|
"attention_mask" => t_attn,
|
||||||
|
"words_mask" => t_words,
|
||||||
|
"text_lengths" => t_lengths,
|
||||||
|
"span_idx" => t_span_idx,
|
||||||
|
"span_mask" => t_span_mask,
|
||||||
|
}
|
||||||
|
).map_err(|e| format!("ort inference error: {}", e))?;
|
||||||
|
|
||||||
|
let (shape, logits_data) = outputs["logits"]
|
||||||
|
.try_extract_tensor::<f32>()
|
||||||
|
.map_err(|e| format!("Failed to extract logits: {}", e))?;
|
||||||
|
|
||||||
|
let logits_shape: Vec<usize> = shape.iter().map(|&d| d as usize).collect();
|
||||||
|
|
||||||
|
// debug dump - gated so sigmoid/loop don't run in release
|
||||||
|
if log::log_enabled!(log::Level::Debug) {
|
||||||
|
debug!("GLiNER logits shape: {:?}, data len: {}", logits_shape, logits_data.len());
|
||||||
|
let max_logit = logits_data.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
|
||||||
|
debug!("GLiNER max logit: {:.4}, sigmoid: {:.4}", max_logit, sigmoid(max_logit));
|
||||||
|
|
||||||
|
let num_words = logits_shape.get(1).copied().unwrap_or(0);
|
||||||
|
let dim_mw = logits_shape.get(2).copied().unwrap_or(0);
|
||||||
|
let dim_e = logits_shape.get(3).copied().unwrap_or(0);
|
||||||
|
for start in 0..num_words {
|
||||||
|
for width in 0..dim_mw.min(num_words - start) {
|
||||||
|
for class_idx in 0..dim_e {
|
||||||
|
let flat_idx = start * dim_mw * dim_e + width * dim_e + class_idx;
|
||||||
|
if flat_idx < logits_data.len() {
|
||||||
|
let score = logits_data[flat_idx];
|
||||||
|
let prob = sigmoid(score);
|
||||||
|
if prob > 0.05 {
|
||||||
|
let end = start + width;
|
||||||
|
let w_start = if start < words.len() { words[start].text } else { "?" };
|
||||||
|
let w_end = if end < words.len() { words[end].text } else { "?" };
|
||||||
|
debug!(" span[{}..{}] '{}'->'{}' label={} score={:.3} prob={:.3}",
|
||||||
|
start, end, w_start, w_end, labels.get(class_idx).unwrap_or(&"?"), score, prob);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let entities = decode_and_search(
|
||||||
|
logits_data, &logits_shape, &words, text, &labels, MAX_WIDTH, THRESHOLD,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut result = HashMap::new();
|
||||||
|
|
||||||
|
for entity in &entities {
|
||||||
|
if entity.prob < MIN_CONFIDENCE {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!("GLiNER entity: '{}' -> '{}' ({:.1}%)",
|
||||||
|
entity.text, entity.label, entity.prob * 100.0);
|
||||||
|
|
||||||
|
if let Some(slot_names) = label_to_slots.get(entity.label.as_str()) {
|
||||||
|
for &slot_name in slot_names {
|
||||||
|
if !result.contains_key(slot_name) {
|
||||||
|
let value = parse_slot_value(&entity.text);
|
||||||
|
result.insert(slot_name.to_string(), value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_slot_value(text: &str) -> SlotValue {
|
||||||
|
if let Ok(n) = text.parse::<f64>() {
|
||||||
|
return SlotValue::Number(n);
|
||||||
|
}
|
||||||
|
SlotValue::Text(text.to_string())
|
||||||
|
}
|
||||||
42
crates/jarvis-core/src/stt.rs
Normal file
42
crates/jarvis-core/src/stt.rs
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
#[cfg(feature = "vosk")]
|
||||||
|
mod vosk;
|
||||||
|
|
||||||
|
use crate::config;
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
|
||||||
|
use crate::config::structs::SpeechToTextEngine;
|
||||||
|
pub use self::vosk::init_vosk;
|
||||||
|
pub use self::vosk::recognize_wake_word;
|
||||||
|
pub use self::vosk::recognize_speech;
|
||||||
|
pub use self::vosk::reset_speech_recognizer;
|
||||||
|
pub use self::vosk::reset_wake_recognizer;
|
||||||
|
pub use self::vosk::finalize_speech;
|
||||||
|
|
||||||
|
static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init() -> Result<(), String> {
|
||||||
|
if STT_TYPE.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
STT_TYPE.set(config::DEFAULT_SPEECH_TO_TEXT_ENGINE)
|
||||||
|
.map_err(|_| "STT type already set".to_string())?;
|
||||||
|
|
||||||
|
match STT_TYPE.get().unwrap() {
|
||||||
|
SpeechToTextEngine::Vosk => {
|
||||||
|
info!("Initializing Vosk STT backend.");
|
||||||
|
vosk::init_vosk()?;
|
||||||
|
info!("STT backend initialized.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
|
||||||
|
if include_partial {
|
||||||
|
vosk::recognize_wake_word(data).map(|(text, _)| text)
|
||||||
|
} else {
|
||||||
|
vosk::recognize_speech(data)
|
||||||
|
}
|
||||||
|
}
|
||||||
156
crates/jarvis-core/src/stt/vosk.rs
Normal file
156
crates/jarvis-core/src/stt/vosk.rs
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
|
use vosk::{DecodingState, Recognizer};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use parking_lot::Mutex;
|
||||||
|
|
||||||
|
use crate::{vosk_models, i18n, config, models};
|
||||||
|
use crate::models::vosk::VoskModel;
|
||||||
|
use crate::DB;
|
||||||
|
|
||||||
|
// the model Arc keeps the vosk::Model alive for the recognizers
|
||||||
|
static VOSK_MODEL: OnceCell<Arc<VoskModel>> = OnceCell::new();
|
||||||
|
static WAKE_RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
|
||||||
|
static SPEECH_RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
|
||||||
|
|
||||||
|
pub fn init_vosk() -> Result<(), String> {
|
||||||
|
if VOSK_MODEL.get().is_some() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let model_path = get_configured_model_path()?;
|
||||||
|
let model_id = format!("vosk:{}", model_path.display());
|
||||||
|
|
||||||
|
// load through registry (shared if anything else needs the same model)
|
||||||
|
let vosk = models::vosk::load(
|
||||||
|
models::registry(),
|
||||||
|
&model_id,
|
||||||
|
model_path.to_str().unwrap(),
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// language-specific wake grammar
|
||||||
|
let lang = i18n::get_language();
|
||||||
|
let wake_grammar = config::get_wake_grammar(&lang);
|
||||||
|
info!("Wake grammar for '{}': {:?}", lang, wake_grammar);
|
||||||
|
|
||||||
|
let mut wake_recognizer = Recognizer::new_with_grammar(&vosk.model, 16000.0, wake_grammar)
|
||||||
|
.ok_or("Failed to create wake word recognizer")?;
|
||||||
|
|
||||||
|
wake_recognizer.set_max_alternatives(1);
|
||||||
|
|
||||||
|
let mut speech_recognizer = Recognizer::new(&vosk.model, 16000.0)
|
||||||
|
.ok_or("Failed to create speech recognizer")?;
|
||||||
|
|
||||||
|
speech_recognizer.set_max_alternatives(config::VOSK_SPEECH_RECOGNIZER_MAX_ALTERNATIVES);
|
||||||
|
speech_recognizer.set_words(config::VOSK_SPEECH_RECOGNIZER_WORDS);
|
||||||
|
speech_recognizer.set_partial_words(config::VOSK_SPEECH_PARTIAL_WORDS);
|
||||||
|
|
||||||
|
VOSK_MODEL.set(vosk).map_err(|_| "Model already set")?;
|
||||||
|
WAKE_RECOGNIZER.set(Mutex::new(wake_recognizer)).map_err(|_| "Wake recognizer already set")?;
|
||||||
|
SPEECH_RECOGNIZER.set(Mutex::new(speech_recognizer)).map_err(|_| "Speech recognizer already set")?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub fn recognize_wake_word(data: &[i16]) -> Option<(String, f32)> {
|
||||||
|
let mut recognizer = WAKE_RECOGNIZER.get()?.lock();
|
||||||
|
|
||||||
|
match recognizer.accept_waveform(data) {
|
||||||
|
Ok(DecodingState::Running) => {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
Ok(DecodingState::Finalized) => {
|
||||||
|
let result = recognizer.result();
|
||||||
|
|
||||||
|
if let Some(alternatives) = result.multiple() {
|
||||||
|
if let Some(best) = alternatives.alternatives.first() {
|
||||||
|
if !best.text.is_empty() {
|
||||||
|
return Some((best.text.to_string(), best.confidence));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub fn recognize_speech(data: &[i16]) -> Option<String> {
|
||||||
|
let mut recognizer = SPEECH_RECOGNIZER.get()?.lock();
|
||||||
|
|
||||||
|
match recognizer.accept_waveform(data) {
|
||||||
|
Ok(DecodingState::Finalized) => {
|
||||||
|
recognizer.result()
|
||||||
|
.multiple()
|
||||||
|
.and_then(|m| m.alternatives.first().map(|a| a.text.to_string()))
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub fn reset_speech_recognizer() {
|
||||||
|
if let Some(recognizer) = SPEECH_RECOGNIZER.get() {
|
||||||
|
recognizer.lock().reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn finalize_speech() -> Option<String> {
|
||||||
|
let mut recognizer = SPEECH_RECOGNIZER.get()?.lock();
|
||||||
|
let result = recognizer.final_result()
|
||||||
|
.multiple()
|
||||||
|
.and_then(|m| m.alternatives.first().map(|a| a.text.to_string()))
|
||||||
|
.filter(|s| !s.trim().is_empty());
|
||||||
|
recognizer.reset();
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset_wake_recognizer() {
|
||||||
|
if let Some(recognizer) = WAKE_RECOGNIZER.get() {
|
||||||
|
recognizer.lock().reset();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_configured_model_path() -> Result<std::path::PathBuf, String> {
|
||||||
|
// try to get from settings
|
||||||
|
if let Some(db) = DB.get() {
|
||||||
|
let settings = db.read();
|
||||||
|
if !settings.vosk_model.is_empty() {
|
||||||
|
if let Some(path) = vosk_models::get_model_path(&settings.vosk_model) {
|
||||||
|
return Ok(path);
|
||||||
|
}
|
||||||
|
warn!("Configured Vosk model '{}' not found, falling back to auto-detect", settings.vosk_model);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// auto-detect: prefer model matching current language
|
||||||
|
let available = vosk_models::scan_vosk_models();
|
||||||
|
let language = i18n::get_language();
|
||||||
|
|
||||||
|
let lang_code = match language.as_str() {
|
||||||
|
"ru" => "ru",
|
||||||
|
"en" => "us",
|
||||||
|
"ua" => "uk",
|
||||||
|
other => other,
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(matched) = available.iter().find(|m| m.language == lang_code) {
|
||||||
|
info!("Auto-detected Vosk model for '{}': {}", language, matched.name);
|
||||||
|
return Ok(matched.path.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(first) = available.first() {
|
||||||
|
info!("Auto-detected Vosk model (no language match): {}", first.name);
|
||||||
|
return Ok(first.path.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
// fallback to legacy path
|
||||||
|
let legacy_path = std::path::Path::new(config::VOSK_MODEL_PATH);
|
||||||
|
if legacy_path.exists() {
|
||||||
|
return Ok(legacy_path.to_path_buf());
|
||||||
|
}
|
||||||
|
|
||||||
|
Err("No Vosk models found".into())
|
||||||
|
}
|
||||||
2
crates/jarvis-core/src/time.rs
Normal file
2
crates/jarvis-core/src/time.rs
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
pub mod structs;
|
||||||
|
pub use structs::*;
|
||||||
21
crates/jarvis-core/src/time/structs.rs
Normal file
21
crates/jarvis-core/src/time/structs.rs
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
use chrono::Timelike;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub enum TimeOfDay {
|
||||||
|
Morning, // 5:00 - 11:59
|
||||||
|
Day, // 12:00 - 16:59
|
||||||
|
Evening, // 17:00 - 21:59
|
||||||
|
Night, // 22:00 - 4:59
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TimeOfDay {
|
||||||
|
pub fn now() -> Self {
|
||||||
|
let hour = chrono::Local::now().hour();
|
||||||
|
match hour {
|
||||||
|
5..=11 => TimeOfDay::Morning,
|
||||||
|
12..=16 => TimeOfDay::Day,
|
||||||
|
17..=21 => TimeOfDay::Evening,
|
||||||
|
_ => TimeOfDay::Night,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue