intent recognition added (first implementation)
This commit is contained in:
parent
0c28304840
commit
5f1f9ce321
20 changed files with 625 additions and 255 deletions
|
|
@ -16,6 +16,9 @@ winit = "0.30"
|
|||
image.workspace = true
|
||||
platform-dirs.workspace = true
|
||||
rand.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
tokio = { version = "1", features = ["rt-multi-thread"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies.winit]
|
||||
version = "0.30"
|
||||
|
|
@ -26,4 +29,4 @@ winapi = { version = "0.3", features = ["winuser"] }
|
|||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
gtk = "0.18"
|
||||
glib = "0.18"
|
||||
glib = "0.21.5"
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
use std::time::SystemTime;
|
||||
|
||||
use jarvis_core::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST};
|
||||
use jarvis_core::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST, intent};
|
||||
use rand::prelude::*;
|
||||
|
||||
pub fn start() -> Result<(), ()> {
|
||||
|
|
@ -9,6 +9,7 @@ pub fn start() -> Result<(), ()> {
|
|||
}
|
||||
|
||||
fn main_loop() -> Result<(), ()> {
|
||||
let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime");
|
||||
let mut start: SystemTime;
|
||||
let sounds_directory = audio::get_sound_directory().unwrap();
|
||||
let frame_length: usize = 512; // default for every wake-word engine
|
||||
|
|
@ -34,7 +35,7 @@ fn main_loop() -> Result<(), ()> {
|
|||
|
||||
// recognize wake-word
|
||||
match listener::data_callback(&frame_buffer) {
|
||||
Some(keyword_index) => {
|
||||
Some(_keyword_index) => {
|
||||
// wake-word activated, process further commands
|
||||
// capture current time
|
||||
start = SystemTime::now();
|
||||
|
|
@ -66,13 +67,18 @@ fn main_loop() -> Result<(), ()> {
|
|||
}
|
||||
recognized_voice = recognized_voice.trim().into();
|
||||
|
||||
// infer command
|
||||
if let Some((cmd_path, cmd_config)) = commands::fetch_command(
|
||||
&recognized_voice,
|
||||
&COMMANDS_LIST.get().unwrap(),
|
||||
) {
|
||||
// some debug info
|
||||
info!("Recognized voice (filtered): {}", recognized_voice);
|
||||
// infer command (try intent recognition first, fallback to levenshtein)
|
||||
let cmd_result = if let Some((intent_id, confidence)) =
|
||||
rt.block_on(intent::classify(&recognized_voice))
|
||||
{
|
||||
info!("Intent recognized: {} (confidence: {:.2})", intent_id, confidence);
|
||||
intent::get_command_by_intent(COMMANDS_LIST.get().unwrap(), &intent_id)
|
||||
} else {
|
||||
info!("Intent not recognized, trying levenshtein fallback...");
|
||||
commands::fetch_command(&recognized_voice, COMMANDS_LIST.get().unwrap())
|
||||
};
|
||||
|
||||
if let Some((cmd_path, cmd_config)) = cmd_result {
|
||||
info!("Command found: {:?}", cmd_path);
|
||||
info!("Executing!");
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use std::path::PathBuf;
|
||||
use parking_lot::RwLock;
|
||||
use std::sync::Arc;
|
||||
|
||||
// include core
|
||||
use jarvis_core::{
|
||||
audio, commands, config, db, listener, recorder, stt,
|
||||
audio, commands, config, db, listener, recorder, stt, intent,
|
||||
APP_CONFIG_DIR, APP_LOG_DIR, COMMANDS_LIST, DB,
|
||||
};
|
||||
|
||||
|
|
@ -32,7 +33,8 @@ fn main() -> Result<(), String> {
|
|||
info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display());
|
||||
|
||||
// initialize database (settings)
|
||||
let _ = DB.set(db::init_settings());
|
||||
DB.set(Arc::new(RwLock::new(db::init_settings())))
|
||||
.expect("DB already initialized");
|
||||
|
||||
// initialize tray
|
||||
// @TODO. macOS currently not supported for tray functionality,
|
||||
|
|
@ -58,7 +60,13 @@ fn main() -> Result<(), String> {
|
|||
|
||||
// init commands
|
||||
info!("Initializing commands.");
|
||||
let cmds = commands::parse_commands().unwrap();
|
||||
let cmds = match commands::parse_commands() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
warn!("Failed to parse commands: {}. Starting with empty command list.", e);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
info!("Commands initialized. Count: {}, List: {:?}", cmds.len(), commands::list(&cmds));
|
||||
COMMANDS_LIST.set(cmds).unwrap();
|
||||
|
||||
|
|
@ -73,6 +81,15 @@ fn main() -> Result<(), String> {
|
|||
app::close(1); // cannot continue without wake-word engine
|
||||
}
|
||||
|
||||
// init intent-recognition engine
|
||||
let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime");
|
||||
rt.block_on(async {
|
||||
if intent::init(COMMANDS_LIST.get().unwrap()).await.is_err() {
|
||||
error!("Failed to initialize intent classifier");
|
||||
app::close(1);
|
||||
}
|
||||
});
|
||||
|
||||
// start the app (in the background thread)
|
||||
std::thread::spawn(|| {
|
||||
let _ = app::start();
|
||||
|
|
|
|||
|
|
@ -21,12 +21,16 @@ kira.workspace = true
|
|||
pv_recorder.workspace = true
|
||||
rustpotter.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
toml.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
# pv_recorder = { workspace = true, optional = true }
|
||||
vosk = { version = "0.3.1", optional = true }
|
||||
intent-classifier = { version = "0.1.0", optional = true }
|
||||
# rustpotter = { workspace = true, optional = true }
|
||||
|
||||
tokio = { version = "1", features = ["sync"], optional = true }
|
||||
|
||||
[features]
|
||||
default = ["jarvis_app"]
|
||||
jarvis_app = ["vosk"]
|
||||
jarvis_app = ["vosk", "intent-classifier", "tokio"]
|
||||
|
|
@ -12,72 +12,96 @@ use std::process::{Child, Command};
|
|||
mod structs;
|
||||
pub use structs::*;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::{audio, config};
|
||||
|
||||
// @TODO. Allow commands both in yaml and json format.
|
||||
pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
|
||||
pub fn parse_commands() -> Result<Vec<JCommandsList>, String> {
|
||||
// collect commands
|
||||
let mut commands: Vec<AssistantCommand> = vec![];
|
||||
let mut commands: Vec<JCommandsList> = Vec::new();
|
||||
|
||||
// read commands directories first
|
||||
if let Ok(cpaths) = fs::read_dir(config::COMMANDS_PATH) {
|
||||
for cpath in cpaths {
|
||||
// validate this command, check if required files exists
|
||||
let _cpath = match cpath {
|
||||
Ok(entry) => entry.path(),
|
||||
Err(e) => {
|
||||
warn!("Failed to read command directory entry: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let cc_file = Path::new(&_cpath).join("command.yaml");
|
||||
let cmd_dirs = fs::read_dir(config::COMMANDS_PATH)
|
||||
.map_err(|e| format!("Error reading commands directory: {}", e))?;
|
||||
|
||||
if cc_file.exists() {
|
||||
// try parse config files
|
||||
let cc_reader = std::fs::File::open(&cc_file).unwrap();
|
||||
let cc_yaml: CommandsList;
|
||||
|
||||
// try parse command.yaml
|
||||
match serde_yaml::from_reader::<File, CommandsList>(cc_reader) {
|
||||
Ok(parse_result) => {
|
||||
cc_yaml = parse_result;
|
||||
}
|
||||
Err(msg) => {
|
||||
warn!(
|
||||
"Can't parse {}, skipping ...\nCommand parse error is: {:?}",
|
||||
&cc_file.display(),
|
||||
msg
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// everything seems to be Ok
|
||||
commands.push(AssistantCommand {
|
||||
path: _cpath,
|
||||
commands: cc_yaml,
|
||||
});
|
||||
for entry in cmd_dirs {
|
||||
let entry = match entry {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
warn!("Failed to read command directory entry: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let cmd_path = entry.path();
|
||||
let toml_file = cmd_path.join("command.toml");
|
||||
|
||||
if !toml_file.exists() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// read and parse TOML
|
||||
let content = match fs::read_to_string(&toml_file) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
warn!("Failed to read {}: {}", toml_file.display(), e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !commands.is_empty() {
|
||||
Ok(commands)
|
||||
} else {
|
||||
error!("No commands were found");
|
||||
Err("No commands were found".into())
|
||||
}
|
||||
let file: JCommandsList = match toml::from_str(&content) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
warn!("Failed to parse {}: {}", toml_file.display(), e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
commands.push(JCommandsList {
|
||||
path: cmd_path,
|
||||
commands: file.commands,
|
||||
});
|
||||
}
|
||||
|
||||
if commands.is_empty() {
|
||||
Err("No commands found".into())
|
||||
} else {
|
||||
error!("Error reading commands directory");
|
||||
return Err("Error reading commands directory".into());
|
||||
info!("Loaded {} commands", commands.len());
|
||||
Ok(commands)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Commands hash generation for cache invalidation (deterministi c)
|
||||
pub fn commands_hash(commands: &Vec<JCommandsList>) -> String {
|
||||
use sha2::{Sha256, Digest};
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
|
||||
// collect all command ids and phrases, sorted
|
||||
let mut all_ids: Vec<_> = commands.iter()
|
||||
.flat_map(|ac| ac.commands.iter().map(|c| (&c.id, &c.phrases)))
|
||||
.collect();
|
||||
all_ids.sort_by_key(|(id, _)| *id);
|
||||
|
||||
for (id, phrases) in all_ids {
|
||||
hasher.update(id.as_bytes());
|
||||
for phrase in phrases {
|
||||
hasher.update(phrase.as_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
|
||||
// @TODO. NLU or smthng else is required, in order to infer commands with highest accuracy possible.
|
||||
pub fn fetch_command<'a>(
|
||||
phrase: &str,
|
||||
commands: &'a Vec<AssistantCommand>,
|
||||
) -> Option<(&'a PathBuf, &'a Config)> {
|
||||
commands: &'a Vec<JCommandsList>,
|
||||
) -> Option<(&'a PathBuf, &'a JCommand)> {
|
||||
// result scmd
|
||||
let mut result_scmd: Option<(&PathBuf, &Config)> = None;
|
||||
let mut result_scmd: Option<(&PathBuf, &JCommand)> = None;
|
||||
let mut current_max_ratio = config::CMD_RATIO_THRESHOLD;
|
||||
|
||||
// convert fetch phrase to sequence
|
||||
|
|
@ -86,7 +110,7 @@ pub fn fetch_command<'a>(
|
|||
// list all the commands
|
||||
for cmd in commands {
|
||||
// list all subcommands
|
||||
for scmd in &cmd.commands.list {
|
||||
for scmd in &cmd.commands {
|
||||
// list all phrases in command
|
||||
for cmd_phrase in &scmd.phrases {
|
||||
// convert cmd phrase to sequence
|
||||
|
|
@ -135,18 +159,17 @@ pub fn execute_cli(cmd: &str, args: &Vec<String>) -> std::io::Result<Child> {
|
|||
|
||||
pub fn execute_command(
|
||||
cmd_path: &PathBuf,
|
||||
cmd_config: &Config,
|
||||
cmd_config: &JCommand,
|
||||
// app_handle: &tauri::AppHandle,
|
||||
) -> Result<bool, String> {
|
||||
let sounds_directory = audio::get_sound_directory().unwrap();
|
||||
|
||||
match cmd_config.command.action.as_str() {
|
||||
match cmd_config.action.as_str() {
|
||||
"voice" => {
|
||||
// VOICE command type
|
||||
let random_cmd_sound = format!(
|
||||
"{}.wav",
|
||||
cmd_config
|
||||
.voice
|
||||
.sounds
|
||||
.choose(&mut rand::thread_rng())
|
||||
.unwrap()
|
||||
|
|
@ -158,8 +181,8 @@ pub fn execute_command(
|
|||
}
|
||||
"ahk" => {
|
||||
// AutoHotkey command type
|
||||
let exe_path_absolute = Path::new(&cmd_config.command.exe_path);
|
||||
let exe_path_local = Path::new(&cmd_path).join(&cmd_config.command.exe_path);
|
||||
let exe_path_absolute = Path::new(&cmd_config.exe_path);
|
||||
let exe_path_local = Path::new(&cmd_path).join(&cmd_config.exe_path);
|
||||
|
||||
if let Ok(_) = execute_exe(
|
||||
if exe_path_absolute.exists() {
|
||||
|
|
@ -167,12 +190,11 @@ pub fn execute_command(
|
|||
} else {
|
||||
exe_path_local.to_str().unwrap()
|
||||
},
|
||||
&cmd_config.command.exe_args,
|
||||
&cmd_config.exe_args,
|
||||
) {
|
||||
let random_cmd_sound = format!(
|
||||
"{}.wav",
|
||||
cmd_config
|
||||
.voice
|
||||
.sounds
|
||||
.choose(&mut rand::thread_rng())
|
||||
.unwrap()
|
||||
|
|
@ -188,14 +210,13 @@ pub fn execute_command(
|
|||
}
|
||||
"cli" => {
|
||||
// CLI command type
|
||||
let cli_cmd = &cmd_config.command.cli_cmd;
|
||||
let cli_cmd = &cmd_config.cli_cmd;
|
||||
|
||||
match execute_cli(cli_cmd, &cmd_config.command.cli_args) {
|
||||
match execute_cli(cli_cmd, &cmd_config.cli_args) {
|
||||
Ok(_) => {
|
||||
let random_cmd_sound = format!(
|
||||
"{}.wav",
|
||||
cmd_config
|
||||
.voice
|
||||
.sounds
|
||||
.choose(&mut rand::thread_rng())
|
||||
.unwrap()
|
||||
|
|
@ -216,7 +237,6 @@ pub fn execute_command(
|
|||
let random_cmd_sound = format!(
|
||||
"{}.wav",
|
||||
cmd_config
|
||||
.voice
|
||||
.sounds
|
||||
.choose(&mut rand::thread_rng())
|
||||
.unwrap()
|
||||
|
|
@ -232,7 +252,6 @@ pub fn execute_command(
|
|||
let random_cmd_sound = format!(
|
||||
"{}.wav",
|
||||
cmd_config
|
||||
.voice
|
||||
.sounds
|
||||
.choose(&mut rand::thread_rng())
|
||||
.unwrap()
|
||||
|
|
@ -249,7 +268,7 @@ pub fn execute_command(
|
|||
}
|
||||
}
|
||||
|
||||
pub fn list(from: &[AssistantCommand]) -> Vec<String> {
|
||||
pub fn list(from: &Vec<JCommandsList>) -> Vec<String> {
|
||||
let mut out: Vec<String> = vec![];
|
||||
|
||||
for x in from.iter() {
|
||||
|
|
|
|||
|
|
@ -1,45 +1,38 @@
|
|||
use serde::Deserialize;
|
||||
use std::path::PathBuf;
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct AssistantCommand {
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct JCommandsList {
|
||||
#[serde(skip)]
|
||||
pub path: PathBuf,
|
||||
pub commands: CommandsList,
|
||||
|
||||
pub commands: Vec<JCommand>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct CommandsList {
|
||||
pub list: Vec<Config>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct Config {
|
||||
pub command: ConfigCommandSection,
|
||||
|
||||
pub voice: ConfigVoiceSection,
|
||||
|
||||
pub phrases: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct ConfigCommandSection {
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct JCommand {
|
||||
pub id: String,
|
||||
pub action: String,
|
||||
|
||||
|
||||
#[serde(default)]
|
||||
pub description: String,
|
||||
|
||||
#[serde(default)]
|
||||
pub exe_path: String,
|
||||
|
||||
|
||||
#[serde(default)]
|
||||
pub exe_args: Vec<String>,
|
||||
|
||||
|
||||
#[serde(default)]
|
||||
pub cli_cmd: String,
|
||||
|
||||
|
||||
#[serde(default)]
|
||||
pub cli_args: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct ConfigVoiceSection {
|
||||
|
||||
#[serde(default)]
|
||||
pub sounds: Vec<String>,
|
||||
|
||||
pub phrases: Vec<String>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ use rustpotter::{
|
|||
RustpotterConfig, ScoreMode,
|
||||
};
|
||||
|
||||
use crate::IntentRecognitionEngine;
|
||||
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
|
||||
|
||||
#[allow(dead_code)]
|
||||
|
|
@ -64,6 +65,7 @@ pub fn init_dirs() -> Result<(), String> {
|
|||
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
|
||||
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
|
||||
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter;
|
||||
pub const DEFAULT_INTENT_RECOGNITION_ENGINE: IntentRecognitionEngine = IntentRecognitionEngine::IntentClassifier;
|
||||
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
|
||||
|
||||
pub const DEFAULT_VOICE: &str = "jarvis-og";
|
||||
|
|
@ -130,6 +132,9 @@ pub const VOSK_FETCH_PHRASE: &str = "джарвис";
|
|||
pub const VOSK_MODEL_PATH: &str = "vosk/model_small";
|
||||
pub const VOSK_MIN_RATIO: f64 = 70.0;
|
||||
|
||||
// IRE (intents recognition)
|
||||
pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.5;
|
||||
|
||||
// ETC
|
||||
pub const CMD_RATIO_THRESHOLD: f64 = 65f64;
|
||||
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,12 @@ pub enum WakeWordEngine {
|
|||
Porcupine,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
|
||||
pub enum IntentRecognitionEngine {
|
||||
IntentClassifier,
|
||||
Rasa,
|
||||
}
|
||||
|
||||
impl fmt::Display for WakeWordEngine {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize};
|
|||
|
||||
use crate::config::structs::SpeechToTextEngine;
|
||||
use crate::config::structs::WakeWordEngine;
|
||||
use crate::config::structs::IntentRecognitionEngine;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct Settings {
|
||||
|
|
@ -10,6 +11,7 @@ pub struct Settings {
|
|||
pub voice: String,
|
||||
|
||||
pub wake_word_engine: WakeWordEngine,
|
||||
pub intent_recognition_engine: IntentRecognitionEngine,
|
||||
pub speech_to_text_engine: SpeechToTextEngine,
|
||||
|
||||
pub api_keys: ApiKeys,
|
||||
|
|
@ -22,6 +24,7 @@ impl Default for Settings {
|
|||
voice: String::from(""),
|
||||
|
||||
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
|
||||
intent_recognition_engine: config::DEFAULT_INTENT_RECOGNITION_ENGINE,
|
||||
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
|
||||
|
||||
api_keys: ApiKeys {
|
||||
|
|
|
|||
62
crates/jarvis-core/src/intent.rs
Normal file
62
crates/jarvis-core/src/intent.rs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
mod intentclassifier;
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::{JCommandsList, commands::JCommand, config};
|
||||
use once_cell::sync::OnceCell;
|
||||
use crate::config::structs::IntentRecognitionEngine;
|
||||
|
||||
static IRE_TYPE: OnceCell<IntentRecognitionEngine> = OnceCell::new();
|
||||
|
||||
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
|
||||
if IRE_TYPE.get().is_some() {
|
||||
return Ok(());
|
||||
} // already initialized
|
||||
|
||||
// set default ire type
|
||||
// @TODO. Make it configurable?
|
||||
IRE_TYPE.set(config::DEFAULT_INTENT_RECOGNITION_ENGINE).unwrap();
|
||||
|
||||
// load given recorder
|
||||
match IRE_TYPE.get().unwrap() {
|
||||
IntentRecognitionEngine::IntentClassifier => {
|
||||
info!("Initializing IRE backend.");
|
||||
intentclassifier::init(&commands).await?;
|
||||
info!("IRE backend initialized.");
|
||||
},
|
||||
IntentRecognitionEngine::Rasa => todo!(),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn classify(text: &str) -> Option<(String, f64)> {
|
||||
match IRE_TYPE.get()? {
|
||||
IntentRecognitionEngine::IntentClassifier => {
|
||||
match intentclassifier::classify(text).await {
|
||||
Ok(prediction) => {
|
||||
let confidence = prediction.confidence.value();
|
||||
if confidence >= config::INTENT_CLASSIFIER_MIN_CONFIDENCE {
|
||||
Some((prediction.intent.to_string(), confidence))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Intent classification error: {}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
IntentRecognitionEngine::Rasa => todo!(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_command_by_intent(commands: &'static Vec<JCommandsList>, intent_id: &str) -> Option<(&'static PathBuf, &'static JCommand)> {
|
||||
match IRE_TYPE.get()? {
|
||||
IntentRecognitionEngine::IntentClassifier => {
|
||||
intentclassifier::get_command(commands, intent_id)
|
||||
}
|
||||
IntentRecognitionEngine::Rasa => todo!(),
|
||||
}
|
||||
}
|
||||
107
crates/jarvis-core/src/intent/intentclassifier.rs
Normal file
107
crates/jarvis-core/src/intent/intentclassifier.rs
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
use intent_classifier::{
|
||||
IntentClassifier, IntentPrediction, IntentError,
|
||||
TrainingExample, TrainingSource, IntentId
|
||||
};
|
||||
|
||||
use tokio::sync::OnceCell;
|
||||
use std::path::PathBuf;
|
||||
use std::fs;
|
||||
|
||||
use crate::commands::{self, JCommand, JCommandsList};
|
||||
use crate::{APP_CONFIG_DIR};
|
||||
|
||||
static CLASSIFIER: OnceCell<IntentClassifier> = OnceCell::const_new();
|
||||
// static COMMANDS_MAP: OnceCell<Vec<JCommandsList>> = OnceCell::const_new();
|
||||
|
||||
const TRAINING_CACHE_FILE: &str = "intent_training.json";
|
||||
const COMMANDS_HASH_FILE: &str = "commands_hash.txt";
|
||||
|
||||
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
|
||||
// parse commands first
|
||||
// let commands = commands::parse_commands()?;
|
||||
let current_hash = commands::commands_hash(&commands); // regen hash for current commands set
|
||||
|
||||
// init classifier
|
||||
let classifier = IntentClassifier::new().await
|
||||
.map_err(|e| format!("Failed to init IntentClassifier: {}", e))?;
|
||||
|
||||
// check if we can use cached training data
|
||||
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
|
||||
let hash_path = config_dir.join(COMMANDS_HASH_FILE);
|
||||
let cache_path = config_dir.join(TRAINING_CACHE_FILE);
|
||||
|
||||
let should_retrain = if hash_path.exists() && cache_path.exists() {
|
||||
let stored_hash = fs::read_to_string(&hash_path).unwrap_or_default();
|
||||
stored_hash.trim() != current_hash
|
||||
} else {
|
||||
true
|
||||
};
|
||||
|
||||
if should_retrain {
|
||||
info!("Training intent classifier with {} commands...", commands.len());
|
||||
train_classifier(&classifier, &commands).await?;
|
||||
|
||||
// save training data and hash
|
||||
if let Ok(export) = classifier.export_training_data().await {
|
||||
let _ = fs::write(&cache_path, export);
|
||||
let _ = fs::write(&hash_path, ¤t_hash);
|
||||
info!("Training data cached.");
|
||||
}
|
||||
} else {
|
||||
info!("Loading cached training data...");
|
||||
if let Ok(data) = fs::read_to_string(&cache_path) {
|
||||
classifier.import_training_data(&data).await
|
||||
.map_err(|e| format!("Failed to import training data: {}", e))?;
|
||||
}
|
||||
}
|
||||
|
||||
// store data
|
||||
CLASSIFIER.set(classifier).map_err(|_| "Classifier already set")?;
|
||||
// COMMANDS_MAP.set(commands).map_err(|_| "Commands map already set")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn classify(text: &str) -> Result<IntentPrediction, IntentError> {
|
||||
let classifier = CLASSIFIER.get().expect("IntentClassifier not initialized");
|
||||
classifier.predict_intent(text).await
|
||||
}
|
||||
|
||||
// get command by intent ID
|
||||
pub fn get_command(commands: &'static Vec<JCommandsList>, intent_id: &str) -> Option<(&'static PathBuf, &'static JCommand)> {
|
||||
// let commands = COMMANDS_MAP.get()?;
|
||||
|
||||
for assistant_cmd in commands {
|
||||
for cmd in &assistant_cmd.commands {
|
||||
if cmd.id == intent_id {
|
||||
return Some((&assistant_cmd.path, cmd));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
// based on: https://github.com/ciresnave/intent-classifier/blob/main/examples/basic_usage.rs
|
||||
async fn train_classifier(
|
||||
classifier: &IntentClassifier,
|
||||
commands: &Vec<JCommandsList>
|
||||
) -> Result<(), String> {
|
||||
for assistant_cmd in commands {
|
||||
for cmd in &assistant_cmd.commands {
|
||||
for phrase in &cmd.phrases {
|
||||
let example = TrainingExample {
|
||||
text: phrase.clone(),
|
||||
intent: IntentId::from(cmd.id.as_str()),
|
||||
confidence: 1.0,
|
||||
source: TrainingSource::Programmatic,
|
||||
};
|
||||
|
||||
classifier.add_training_example(example).await
|
||||
.map_err(|e| format!("Failed to add training example: {}", e))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
use once_cell::sync::{Lazy, OnceCell};
|
||||
use parking_lot::RwLock;
|
||||
use std::sync::Arc;
|
||||
use std::{sync::Arc};
|
||||
use platform_dirs::AppDirs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
|
|
@ -20,6 +20,9 @@ pub mod recorder;
|
|||
#[cfg(feature = "jarvis_app")]
|
||||
pub mod stt;
|
||||
|
||||
#[cfg(feature = "jarvis_app")]
|
||||
pub mod intent;
|
||||
|
||||
// shared statics
|
||||
pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| std::env::current_dir().unwrap());
|
||||
pub static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| APP_DIR.clone().join("sound"));
|
||||
|
|
@ -27,9 +30,11 @@ pub static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
|
|||
pub static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
|
||||
pub static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
|
||||
pub static DB: OnceCell<Arc<RwLock<db::structs::Settings>>> = OnceCell::new();
|
||||
pub static COMMANDS_LIST: OnceCell<Vec<commands::AssistantCommand>> = OnceCell::new();
|
||||
pub static COMMANDS_LIST: OnceCell<Vec<JCommandsList>> = OnceCell::new();
|
||||
|
||||
// re-exports
|
||||
pub use commands::AssistantCommand;
|
||||
pub use commands::JCommandsList;
|
||||
pub use config::structs::*;
|
||||
pub use db::structs::Settings;
|
||||
pub use db::structs::Settings;
|
||||
|
||||
// use crate::commands::{JComandsList, JCommand};
|
||||
|
|
@ -25,7 +25,7 @@ pub fn init() -> Result<(), ()> {
|
|||
|
||||
// store current engine
|
||||
WAKE_WORD_ENGINE
|
||||
.set(DB.get().unwrap().wake_word_engine)
|
||||
.set(DB.get().unwrap().read().wake_word_engine)
|
||||
.unwrap();
|
||||
|
||||
// load given wake-word engine
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ pub fn db_read(state: tauri::State<'_, AppState>, key: &str) -> String {
|
|||
"selected_microphone" => settings.microphone.to_string(),
|
||||
"assistant_voice" => settings.voice.clone(),
|
||||
"selected_wake_word_engine" => format!("{:?}", settings.wake_word_engine),
|
||||
"selected_intent_recognition_engine" => format!("{:?}", settings.intent_recognition_engine),
|
||||
"speech_to_text_engine" => format!("{:?}", settings.speech_to_text_engine),
|
||||
"api_key__picovoice" => settings.api_keys.picovoice.clone(),
|
||||
"api_key__openai" => settings.api_keys.openai.clone(),
|
||||
|
|
@ -41,6 +42,13 @@ pub fn db_write(state: tauri::State<'_, AppState>, key: &str, val: &str) -> bool
|
|||
_ => return false,
|
||||
}
|
||||
}
|
||||
"selected_intent_recognition_engine" => {
|
||||
match val.to_lowercase().as_str() {
|
||||
"intentclassifier" => settings.intent_recognition_engine = jarvis_core::config::structs::IntentRecognitionEngine::IntentClassifier,
|
||||
"rasa" => settings.intent_recognition_engine = jarvis_core::config::structs::IntentRecognitionEngine::Rasa,
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
"api_key__picovoice" => {
|
||||
settings.api_keys.picovoice = val.to_string();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue