intent recognition added (first implementation)

This commit is contained in:
Priler 2026-01-05 01:22:45 +05:00
parent 0c28304840
commit 5f1f9ce321
20 changed files with 625 additions and 255 deletions

View file

@ -16,6 +16,9 @@ winit = "0.30"
image.workspace = true
platform-dirs.workspace = true
rand.workspace = true
parking_lot.workspace = true
tokio = { version = "1", features = ["rt-multi-thread"] }
[target.'cfg(windows)'.dependencies.winit]
version = "0.30"
@ -26,4 +29,4 @@ winapi = { version = "0.3", features = ["winuser"] }
[target.'cfg(target_os = "linux")'.dependencies]
gtk = "0.18"
glib = "0.18"
glib = "0.21.5"

View file

@ -1,6 +1,6 @@
use std::time::SystemTime;
use jarvis_core::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST};
use jarvis_core::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST, intent};
use rand::prelude::*;
pub fn start() -> Result<(), ()> {
@ -9,6 +9,7 @@ pub fn start() -> Result<(), ()> {
}
fn main_loop() -> Result<(), ()> {
let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime");
let mut start: SystemTime;
let sounds_directory = audio::get_sound_directory().unwrap();
let frame_length: usize = 512; // default for every wake-word engine
@ -34,7 +35,7 @@ fn main_loop() -> Result<(), ()> {
// recognize wake-word
match listener::data_callback(&frame_buffer) {
Some(keyword_index) => {
Some(_keyword_index) => {
// wake-word activated, process further commands
// capture current time
start = SystemTime::now();
@ -66,13 +67,18 @@ fn main_loop() -> Result<(), ()> {
}
recognized_voice = recognized_voice.trim().into();
// infer command
if let Some((cmd_path, cmd_config)) = commands::fetch_command(
&recognized_voice,
&COMMANDS_LIST.get().unwrap(),
) {
// some debug info
info!("Recognized voice (filtered): {}", recognized_voice);
// infer command (try intent recognition first, fallback to levenshtein)
let cmd_result = if let Some((intent_id, confidence)) =
rt.block_on(intent::classify(&recognized_voice))
{
info!("Intent recognized: {} (confidence: {:.2})", intent_id, confidence);
intent::get_command_by_intent(COMMANDS_LIST.get().unwrap(), &intent_id)
} else {
info!("Intent not recognized, trying levenshtein fallback...");
commands::fetch_command(&recognized_voice, COMMANDS_LIST.get().unwrap())
};
if let Some((cmd_path, cmd_config)) = cmd_result {
info!("Command found: {:?}", cmd_path);
info!("Executing!");

View file

@ -1,8 +1,9 @@
use std::path::PathBuf;
use parking_lot::RwLock;
use std::sync::Arc;
// include core
use jarvis_core::{
audio, commands, config, db, listener, recorder, stt,
audio, commands, config, db, listener, recorder, stt, intent,
APP_CONFIG_DIR, APP_LOG_DIR, COMMANDS_LIST, DB,
};
@ -32,7 +33,8 @@ fn main() -> Result<(), String> {
info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display());
// initialize database (settings)
let _ = DB.set(db::init_settings());
DB.set(Arc::new(RwLock::new(db::init_settings())))
.expect("DB already initialized");
// initialize tray
// @TODO. macOS currently not supported for tray functionality,
@ -58,7 +60,13 @@ fn main() -> Result<(), String> {
// init commands
info!("Initializing commands.");
let cmds = commands::parse_commands().unwrap();
let cmds = match commands::parse_commands() {
Ok(c) => c,
Err(e) => {
warn!("Failed to parse commands: {}. Starting with empty command list.", e);
Vec::new()
}
};
info!("Commands initialized. Count: {}, List: {:?}", cmds.len(), commands::list(&cmds));
COMMANDS_LIST.set(cmds).unwrap();
@ -73,6 +81,15 @@ fn main() -> Result<(), String> {
app::close(1); // cannot continue without wake-word engine
}
// init intent-recognition engine
let rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime");
rt.block_on(async {
if intent::init(COMMANDS_LIST.get().unwrap()).await.is_err() {
error!("Failed to initialize intent classifier");
app::close(1);
}
});
// start the app (in the background thread)
std::thread::spawn(|| {
let _ = app::start();

View file

@ -21,12 +21,16 @@ kira.workspace = true
pv_recorder.workspace = true
rustpotter.workspace = true
parking_lot.workspace = true
toml.workspace = true
sha2.workspace = true
# pv_recorder = { workspace = true, optional = true }
vosk = { version = "0.3.1", optional = true }
intent-classifier = { version = "0.1.0", optional = true }
# rustpotter = { workspace = true, optional = true }
tokio = { version = "1", features = ["sync"], optional = true }
[features]
default = ["jarvis_app"]
jarvis_app = ["vosk"]
jarvis_app = ["vosk", "intent-classifier", "tokio"]

View file

@ -12,72 +12,96 @@ use std::process::{Child, Command};
mod structs;
pub use structs::*;
use std::collections::HashMap;
use crate::{audio, config};
// @TODO. Allow commands both in yaml and json format.
pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
pub fn parse_commands() -> Result<Vec<JCommandsList>, String> {
// collect commands
let mut commands: Vec<AssistantCommand> = vec![];
let mut commands: Vec<JCommandsList> = Vec::new();
// read commands directories first
if let Ok(cpaths) = fs::read_dir(config::COMMANDS_PATH) {
for cpath in cpaths {
// validate this command, check if required files exists
let _cpath = match cpath {
Ok(entry) => entry.path(),
Err(e) => {
warn!("Failed to read command directory entry: {}", e);
continue;
}
};
let cc_file = Path::new(&_cpath).join("command.yaml");
let cmd_dirs = fs::read_dir(config::COMMANDS_PATH)
.map_err(|e| format!("Error reading commands directory: {}", e))?;
if cc_file.exists() {
// try parse config files
let cc_reader = std::fs::File::open(&cc_file).unwrap();
let cc_yaml: CommandsList;
// try parse command.yaml
match serde_yaml::from_reader::<File, CommandsList>(cc_reader) {
Ok(parse_result) => {
cc_yaml = parse_result;
}
Err(msg) => {
warn!(
"Can't parse {}, skipping ...\nCommand parse error is: {:?}",
&cc_file.display(),
msg
);
continue;
}
}
// everything seems to be Ok
commands.push(AssistantCommand {
path: _cpath,
commands: cc_yaml,
});
for entry in cmd_dirs {
let entry = match entry {
Ok(e) => e,
Err(e) => {
warn!("Failed to read command directory entry: {}", e);
continue;
}
};
let cmd_path = entry.path();
let toml_file = cmd_path.join("command.toml");
if !toml_file.exists() {
continue;
}
// read and parse TOML
let content = match fs::read_to_string(&toml_file) {
Ok(c) => c,
Err(e) => {
warn!("Failed to read {}: {}", toml_file.display(), e);
continue;
}
};
if !commands.is_empty() {
Ok(commands)
} else {
error!("No commands were found");
Err("No commands were found".into())
}
let file: JCommandsList = match toml::from_str(&content) {
Ok(f) => f,
Err(e) => {
warn!("Failed to parse {}: {}", toml_file.display(), e);
continue;
}
};
commands.push(JCommandsList {
path: cmd_path,
commands: file.commands,
});
}
if commands.is_empty() {
Err("No commands found".into())
} else {
error!("Error reading commands directory");
return Err("Error reading commands directory".into());
info!("Loaded {} commands", commands.len());
Ok(commands)
}
}
// Commands hash generation for cache invalidation (deterministi c)
pub fn commands_hash(commands: &Vec<JCommandsList>) -> String {
use sha2::{Sha256, Digest};
let mut hasher = Sha256::new();
// collect all command ids and phrases, sorted
let mut all_ids: Vec<_> = commands.iter()
.flat_map(|ac| ac.commands.iter().map(|c| (&c.id, &c.phrases)))
.collect();
all_ids.sort_by_key(|(id, _)| *id);
for (id, phrases) in all_ids {
hasher.update(id.as_bytes());
for phrase in phrases {
hasher.update(phrase.as_bytes());
}
}
format!("{:x}", hasher.finalize())
}
// @TODO. NLU or smthng else is required, in order to infer commands with highest accuracy possible.
pub fn fetch_command<'a>(
phrase: &str,
commands: &'a Vec<AssistantCommand>,
) -> Option<(&'a PathBuf, &'a Config)> {
commands: &'a Vec<JCommandsList>,
) -> Option<(&'a PathBuf, &'a JCommand)> {
// result scmd
let mut result_scmd: Option<(&PathBuf, &Config)> = None;
let mut result_scmd: Option<(&PathBuf, &JCommand)> = None;
let mut current_max_ratio = config::CMD_RATIO_THRESHOLD;
// convert fetch phrase to sequence
@ -86,7 +110,7 @@ pub fn fetch_command<'a>(
// list all the commands
for cmd in commands {
// list all subcommands
for scmd in &cmd.commands.list {
for scmd in &cmd.commands {
// list all phrases in command
for cmd_phrase in &scmd.phrases {
// convert cmd phrase to sequence
@ -135,18 +159,17 @@ pub fn execute_cli(cmd: &str, args: &Vec<String>) -> std::io::Result<Child> {
pub fn execute_command(
cmd_path: &PathBuf,
cmd_config: &Config,
cmd_config: &JCommand,
// app_handle: &tauri::AppHandle,
) -> Result<bool, String> {
let sounds_directory = audio::get_sound_directory().unwrap();
match cmd_config.command.action.as_str() {
match cmd_config.action.as_str() {
"voice" => {
// VOICE command type
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@ -158,8 +181,8 @@ pub fn execute_command(
}
"ahk" => {
// AutoHotkey command type
let exe_path_absolute = Path::new(&cmd_config.command.exe_path);
let exe_path_local = Path::new(&cmd_path).join(&cmd_config.command.exe_path);
let exe_path_absolute = Path::new(&cmd_config.exe_path);
let exe_path_local = Path::new(&cmd_path).join(&cmd_config.exe_path);
if let Ok(_) = execute_exe(
if exe_path_absolute.exists() {
@ -167,12 +190,11 @@ pub fn execute_command(
} else {
exe_path_local.to_str().unwrap()
},
&cmd_config.command.exe_args,
&cmd_config.exe_args,
) {
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@ -188,14 +210,13 @@ pub fn execute_command(
}
"cli" => {
// CLI command type
let cli_cmd = &cmd_config.command.cli_cmd;
let cli_cmd = &cmd_config.cli_cmd;
match execute_cli(cli_cmd, &cmd_config.command.cli_args) {
match execute_cli(cli_cmd, &cmd_config.cli_args) {
Ok(_) => {
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@ -216,7 +237,6 @@ pub fn execute_command(
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@ -232,7 +252,6 @@ pub fn execute_command(
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
@ -249,7 +268,7 @@ pub fn execute_command(
}
}
pub fn list(from: &[AssistantCommand]) -> Vec<String> {
pub fn list(from: &Vec<JCommandsList>) -> Vec<String> {
let mut out: Vec<String> = vec![];
for x in from.iter() {

View file

@ -1,45 +1,38 @@
use serde::Deserialize;
use std::path::PathBuf;
use serde::Deserialize;
#[derive(Debug)]
pub struct AssistantCommand {
#[derive(Deserialize, Debug)]
pub struct JCommandsList {
#[serde(skip)]
pub path: PathBuf,
pub commands: CommandsList,
pub commands: Vec<JCommand>,
}
#[derive(Deserialize, Debug)]
pub struct CommandsList {
pub list: Vec<Config>,
}
#[derive(Deserialize, Debug)]
pub struct Config {
pub command: ConfigCommandSection,
pub voice: ConfigVoiceSection,
pub phrases: Vec<String>,
}
#[derive(Deserialize, Debug)]
pub struct ConfigCommandSection {
#[derive(Deserialize, Debug, Clone)]
pub struct JCommand {
pub id: String,
pub action: String,
#[serde(default)]
pub description: String,
#[serde(default)]
pub exe_path: String,
#[serde(default)]
pub exe_args: Vec<String>,
#[serde(default)]
pub cli_cmd: String,
#[serde(default)]
pub cli_args: Vec<String>,
}
#[derive(Deserialize, Debug)]
pub struct ConfigVoiceSection {
#[serde(default)]
pub sounds: Vec<String>,
pub phrases: Vec<String>,
}

View file

@ -17,6 +17,7 @@ use rustpotter::{
RustpotterConfig, ScoreMode,
};
use crate::IntentRecognitionEngine;
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
#[allow(dead_code)]
@ -64,6 +65,7 @@ pub fn init_dirs() -> Result<(), String> {
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter;
pub const DEFAULT_INTENT_RECOGNITION_ENGINE: IntentRecognitionEngine = IntentRecognitionEngine::IntentClassifier;
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
pub const DEFAULT_VOICE: &str = "jarvis-og";
@ -130,6 +132,9 @@ pub const VOSK_FETCH_PHRASE: &str = "джарвис";
pub const VOSK_MODEL_PATH: &str = "vosk/model_small";
pub const VOSK_MIN_RATIO: f64 = 70.0;
// IRE (intents recognition)
pub const INTENT_CLASSIFIER_MIN_CONFIDENCE: f64 = 0.5;
// ETC
pub const CMD_RATIO_THRESHOLD: f64 = 65f64;
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);

View file

@ -8,6 +8,12 @@ pub enum WakeWordEngine {
Porcupine,
}
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
pub enum IntentRecognitionEngine {
IntentClassifier,
Rasa,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)

View file

@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize};
use crate::config::structs::SpeechToTextEngine;
use crate::config::structs::WakeWordEngine;
use crate::config::structs::IntentRecognitionEngine;
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Settings {
@ -10,6 +11,7 @@ pub struct Settings {
pub voice: String,
pub wake_word_engine: WakeWordEngine,
pub intent_recognition_engine: IntentRecognitionEngine,
pub speech_to_text_engine: SpeechToTextEngine,
pub api_keys: ApiKeys,
@ -22,6 +24,7 @@ impl Default for Settings {
voice: String::from(""),
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
intent_recognition_engine: config::DEFAULT_INTENT_RECOGNITION_ENGINE,
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
api_keys: ApiKeys {

View file

@ -0,0 +1,62 @@
mod intentclassifier;
use std::path::PathBuf;
use crate::{JCommandsList, commands::JCommand, config};
use once_cell::sync::OnceCell;
use crate::config::structs::IntentRecognitionEngine;
static IRE_TYPE: OnceCell<IntentRecognitionEngine> = OnceCell::new();
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
if IRE_TYPE.get().is_some() {
return Ok(());
} // already initialized
// set default ire type
// @TODO. Make it configurable?
IRE_TYPE.set(config::DEFAULT_INTENT_RECOGNITION_ENGINE).unwrap();
// load given recorder
match IRE_TYPE.get().unwrap() {
IntentRecognitionEngine::IntentClassifier => {
info!("Initializing IRE backend.");
intentclassifier::init(&commands).await?;
info!("IRE backend initialized.");
},
IntentRecognitionEngine::Rasa => todo!(),
}
Ok(())
}
pub async fn classify(text: &str) -> Option<(String, f64)> {
match IRE_TYPE.get()? {
IntentRecognitionEngine::IntentClassifier => {
match intentclassifier::classify(text).await {
Ok(prediction) => {
let confidence = prediction.confidence.value();
if confidence >= config::INTENT_CLASSIFIER_MIN_CONFIDENCE {
Some((prediction.intent.to_string(), confidence))
} else {
None
}
}
Err(e) => {
error!("Intent classification error: {}", e);
None
}
}
}
IntentRecognitionEngine::Rasa => todo!(),
}
}
pub fn get_command_by_intent(commands: &'static Vec<JCommandsList>, intent_id: &str) -> Option<(&'static PathBuf, &'static JCommand)> {
match IRE_TYPE.get()? {
IntentRecognitionEngine::IntentClassifier => {
intentclassifier::get_command(commands, intent_id)
}
IntentRecognitionEngine::Rasa => todo!(),
}
}

View file

@ -0,0 +1,107 @@
use intent_classifier::{
IntentClassifier, IntentPrediction, IntentError,
TrainingExample, TrainingSource, IntentId
};
use tokio::sync::OnceCell;
use std::path::PathBuf;
use std::fs;
use crate::commands::{self, JCommand, JCommandsList};
use crate::{APP_CONFIG_DIR};
static CLASSIFIER: OnceCell<IntentClassifier> = OnceCell::const_new();
// static COMMANDS_MAP: OnceCell<Vec<JCommandsList>> = OnceCell::const_new();
const TRAINING_CACHE_FILE: &str = "intent_training.json";
const COMMANDS_HASH_FILE: &str = "commands_hash.txt";
pub async fn init(commands: &Vec<JCommandsList>) -> Result<(), String> {
// parse commands first
// let commands = commands::parse_commands()?;
let current_hash = commands::commands_hash(&commands); // regen hash for current commands set
// init classifier
let classifier = IntentClassifier::new().await
.map_err(|e| format!("Failed to init IntentClassifier: {}", e))?;
// check if we can use cached training data
let config_dir = APP_CONFIG_DIR.get().ok_or("Config dir not set")?;
let hash_path = config_dir.join(COMMANDS_HASH_FILE);
let cache_path = config_dir.join(TRAINING_CACHE_FILE);
let should_retrain = if hash_path.exists() && cache_path.exists() {
let stored_hash = fs::read_to_string(&hash_path).unwrap_or_default();
stored_hash.trim() != current_hash
} else {
true
};
if should_retrain {
info!("Training intent classifier with {} commands...", commands.len());
train_classifier(&classifier, &commands).await?;
// save training data and hash
if let Ok(export) = classifier.export_training_data().await {
let _ = fs::write(&cache_path, export);
let _ = fs::write(&hash_path, &current_hash);
info!("Training data cached.");
}
} else {
info!("Loading cached training data...");
if let Ok(data) = fs::read_to_string(&cache_path) {
classifier.import_training_data(&data).await
.map_err(|e| format!("Failed to import training data: {}", e))?;
}
}
// store data
CLASSIFIER.set(classifier).map_err(|_| "Classifier already set")?;
// COMMANDS_MAP.set(commands).map_err(|_| "Commands map already set")?;
Ok(())
}
pub async fn classify(text: &str) -> Result<IntentPrediction, IntentError> {
let classifier = CLASSIFIER.get().expect("IntentClassifier not initialized");
classifier.predict_intent(text).await
}
// get command by intent ID
pub fn get_command(commands: &'static Vec<JCommandsList>, intent_id: &str) -> Option<(&'static PathBuf, &'static JCommand)> {
// let commands = COMMANDS_MAP.get()?;
for assistant_cmd in commands {
for cmd in &assistant_cmd.commands {
if cmd.id == intent_id {
return Some((&assistant_cmd.path, cmd));
}
}
}
None
}
// based on: https://github.com/ciresnave/intent-classifier/blob/main/examples/basic_usage.rs
async fn train_classifier(
classifier: &IntentClassifier,
commands: &Vec<JCommandsList>
) -> Result<(), String> {
for assistant_cmd in commands {
for cmd in &assistant_cmd.commands {
for phrase in &cmd.phrases {
let example = TrainingExample {
text: phrase.clone(),
intent: IntentId::from(cmd.id.as_str()),
confidence: 1.0,
source: TrainingSource::Programmatic,
};
classifier.add_training_example(example).await
.map_err(|e| format!("Failed to add training example: {}", e))?;
}
}
}
Ok(())
}

View file

@ -1,6 +1,6 @@
use once_cell::sync::{Lazy, OnceCell};
use parking_lot::RwLock;
use std::sync::Arc;
use std::{sync::Arc};
use platform_dirs::AppDirs;
use std::path::PathBuf;
@ -20,6 +20,9 @@ pub mod recorder;
#[cfg(feature = "jarvis_app")]
pub mod stt;
#[cfg(feature = "jarvis_app")]
pub mod intent;
// shared statics
pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| std::env::current_dir().unwrap());
pub static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| APP_DIR.clone().join("sound"));
@ -27,9 +30,11 @@ pub static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
pub static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
pub static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
pub static DB: OnceCell<Arc<RwLock<db::structs::Settings>>> = OnceCell::new();
pub static COMMANDS_LIST: OnceCell<Vec<commands::AssistantCommand>> = OnceCell::new();
pub static COMMANDS_LIST: OnceCell<Vec<JCommandsList>> = OnceCell::new();
// re-exports
pub use commands::AssistantCommand;
pub use commands::JCommandsList;
pub use config::structs::*;
pub use db::structs::Settings;
pub use db::structs::Settings;
// use crate::commands::{JComandsList, JCommand};

View file

@ -25,7 +25,7 @@ pub fn init() -> Result<(), ()> {
// store current engine
WAKE_WORD_ENGINE
.set(DB.get().unwrap().wake_word_engine)
.set(DB.get().unwrap().read().wake_word_engine)
.unwrap();
// load given wake-word engine

View file

@ -9,6 +9,7 @@ pub fn db_read(state: tauri::State<'_, AppState>, key: &str) -> String {
"selected_microphone" => settings.microphone.to_string(),
"assistant_voice" => settings.voice.clone(),
"selected_wake_word_engine" => format!("{:?}", settings.wake_word_engine),
"selected_intent_recognition_engine" => format!("{:?}", settings.intent_recognition_engine),
"speech_to_text_engine" => format!("{:?}", settings.speech_to_text_engine),
"api_key__picovoice" => settings.api_keys.picovoice.clone(),
"api_key__openai" => settings.api_keys.openai.clone(),
@ -41,6 +42,13 @@ pub fn db_write(state: tauri::State<'_, AppState>, key: &str, val: &str) -> bool
_ => return false,
}
}
"selected_intent_recognition_engine" => {
match val.to_lowercase().as_str() {
"intentclassifier" => settings.intent_recognition_engine = jarvis_core::config::structs::IntentRecognitionEngine::IntentClassifier,
"rasa" => settings.intent_recognition_engine = jarvis_core::config::structs::IntentRecognitionEngine::Rasa,
_ => return false,
}
}
"api_key__picovoice" => {
settings.api_keys.picovoice = val.to_string();
}