project restructure with Rust workspaces

crates/jarvis-gui/src/events.rs

@@ -0,0 +1,41 @@
+use tauri::Emitter;
+
+// the payload type must implement `Serialize` and `Clone`.
+#[derive(Clone, serde::Serialize)]
+pub struct Payload {
+    pub data: String,
+}
+
+#[allow(dead_code)]
+pub enum EventTypes {
+    AudioPlay,
+    AssistantWaiting,
+    AssistantGreet,
+    CommandStart,
+    CommandInProcess,
+    CommandEnd,
+}
+
+impl EventTypes {
+    pub fn get(&self) -> &str {
+        match self {
+            Self::AudioPlay => "audio-play",
+            Self::AssistantWaiting => "assistant-waiting",
+            Self::AssistantGreet => "assistant-greet",
+            Self::CommandStart => "command-start",
+            Self::CommandInProcess => "command-in-process",
+            Self::CommandEnd => "command-end",
+        }
+    }
+}
+
+pub fn play(phrase: &str, app_handle: &tauri::AppHandle) {
+    app_handle
+        .emit(
+            EventTypes::AudioPlay.get(),
+            Payload {
+                data: phrase.into(),
+            },
+        )
+        .unwrap();
+}

crates/jarvis-gui/src/main.rs

@@ -0,0 +1,30 @@
+// Prevents additional console window on Windows in release, DO NOT REMOVE!!
+#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
+
+use jarvis_core::{config, db, APP_CONFIG_DIR, APP_LOG_DIR, DB};
+
+#[macro_use]
+extern crate simple_log;
+
+mod events;
+
+// mod tauri_commands;
+
+fn main() {
+    config::init_dirs().expect("Failed to init dirs");
+    
+    // basic logging setup (simpler for GUI)
+    simple_log::quick!("info");
+
+    let _ = DB.set(db::init_settings());
+
+    tauri::Builder::default()
+        .plugin(tauri_plugin_shell::init())
+        .plugin(tauri_plugin_dialog::init())
+        .plugin(tauri_plugin_fs::init())
+        .invoke_handler(tauri::generate_handler![
+            // commands will be added here after tauri_commands module is fixed
+        ])
+        .run(tauri::generate_context!())
+        .expect("error while running tauri application");
+}

crates/jarvis-gui/src/tauri_commands.rs

@@ -0,0 +1,27 @@
+// import DB related commands
+mod db;
+pub use db::*;
+
+// import RECORDER commands
+mod audio;
+pub use audio::*;
+
+// import PORCUPINE commands
+mod listener;
+pub use listener::*;
+
+// import SYS commands
+mod sys;
+pub use sys::*;
+
+// import VOICE commands
+mod voice;
+pub use voice::*;
+
+// import FS commands
+mod fs;
+pub use fs::*;
+
+// import ETC commands
+mod etc;
+pub use etc::*;

crates/jarvis-gui/src/tauri_commands/audio.rs

@@ -0,0 +1,33 @@
+use pv_recorder::RecorderBuilder;
+
+#[tauri::command]
+pub fn pv_get_audio_devices() -> Vec<String> {
+    let audio_devices = RecorderBuilder::default().get_audio_devices();
+    match audio_devices {
+        Ok(audio_devices) => audio_devices,
+        Err(err) => panic!("Failed to get audio devices: {}", err),
+    }
+}
+
+#[tauri::command]
+pub fn pv_get_audio_device_name(idx: i32) -> String {
+    let audio_devices = RecorderBuilder::default().get_audio_devices();
+    let mut first_device: String = String::new();
+    match audio_devices {
+        Ok(audio_devices) => {
+            for (_idx, device) in audio_devices.iter().enumerate() {
+                if idx as usize == _idx {
+                    return device.to_string();
+                }
+
+                if _idx == 0 {
+                    first_device = device.to_string()
+                }
+            }
+        }
+        Err(err) => panic!("Failed to get audio devices: {}", err),
+    };
+
+    // return first device as default, if none were matched
+    first_device
+}

crates/jarvis-gui/src/tauri_commands/db.rs

@@ -0,0 +1,19 @@
+use crate::DB;
+
+#[tauri::command]
+pub fn db_read(key: &str) -> String {
+    if let Some(value) = DB.lock().unwrap().get::<String>(key) {
+        return value
+    }
+
+    String::from("")
+}
+
+#[tauri::command]
+pub fn db_write(key: &str, val: &str) -> bool {
+    if let Ok(_) = DB.lock().unwrap().set(key, &val) {
+        true
+    } else {
+        false
+    }
+}

crates/jarvis-gui/src/tauri_commands/etc.rs

@@ -0,0 +1,54 @@
+use crate::config;
+use crate::APP_LOG_DIR;
+
+// Learn more about Tauri commands at https://tauri.app/v1/guides/features/command
+
+#[tauri::command]
+pub fn get_app_version() -> String {
+    if let Some(res) = config::APP_VERSION {
+        res.to_string()
+    } else {
+        String::from("error")
+    }
+}
+
+#[tauri::command]
+pub fn get_author_name() -> String {
+    if let Some(res) = config::AUTHOR_NAME {
+        res.to_string()
+    } else {
+        String::from("error")
+    }
+}
+
+#[tauri::command]
+pub fn get_repository_link() -> String {
+    if let Some(res) = config::REPOSITORY_LINK {
+        res.to_string()
+    } else {
+        String::from("error")
+    }
+}
+
+#[tauri::command]
+pub fn get_tg_official_link() -> String {
+    if let Some(ver) = config::TG_OFFICIAL_LINK {
+        ver.to_string()
+    } else {
+        String::from("error")
+    }
+}
+
+#[tauri::command]
+pub fn get_feedback_link() -> String {
+    if let Some(res) = config::FEEDBACK_LINK {
+        res.to_string()
+    } else {
+        String::from("error")
+    }
+}
+
+#[tauri::command]
+pub fn get_log_file_path() -> String {
+    format!("{}", APP_LOG_DIR.lock().unwrap())
+}

crates/jarvis-gui/src/tauri_commands/fs.rs

@@ -0,0 +1,47 @@
+use std::process::Command;
+
+// taken from https://github.com/tauri-apps/tauri/issues/4062#issuecomment-1338048169
+#[tauri::command]
+pub fn show_in_folder(path: String) {
+  #[cfg(target_os = "windows")]
+  {
+    Command::new("explorer")
+        .args(["/select,", &path]) // The comma after select is not a typo
+        .spawn()
+        .unwrap();
+  }
+
+  #[cfg(target_os = "linux")]
+  {
+    if path.contains(",") {
+      // see https://gitlab.freedesktop.org/dbus/dbus/-/issues/76
+      let new_path = match metadata(&path).unwrap().is_dir() {
+        true => path,
+        false => {
+          let mut path2 = PathBuf::from(path);
+          path2.pop();
+          path2.into_os_string().into_string().unwrap()
+        }
+      };
+      Command::new("xdg-open")
+          .arg(&new_path)
+          .spawn()
+          .unwrap();
+    } else {
+      Command::new("dbus-send")
+          .args(["--session", "--dest=org.freedesktop.FileManager1", "--type=method_call",
+                "/org/freedesktop/FileManager1", "org.freedesktop.FileManager1.ShowItems",
+                format!("array:string:\"file://{path}\"").as_str(), "string:\"\""])
+          .spawn()
+          .unwrap();
+    }
+  }
+
+  #[cfg(target_os = "macos")]
+  {
+    Command::new("open")
+        .args(["-R", &path])
+        .spawn()
+        .unwrap();
+  }
+}

crates/jarvis-gui/src/tauri_commands/listener.rs

@@ -0,0 +1,416 @@
+use porcupine::{Porcupine, PorcupineBuilder};
+use std::ops::Sub;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::path::Path;
+use log::{info, warn, error};
+use rustpotter::{Rustpotter, RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig};
+// use dasp::{sample::ToSample, Sample};
+
+// use crate::events::Payload;
+use tauri::Manager;
+
+use rand::seq::SliceRandom;
+use std::time::SystemTime;
+use once_cell::sync::OnceCell;
+use std::sync::Mutex;
+
+use crate::assistant_commands;
+use crate::events;
+
+use crate::config;
+use crate::vosk;
+use crate::recorder::{self, FRAME_LENGTH};
+
+use crate::COMMANDS;
+use crate::DB;
+
+// track listening state
+static LISTENING: AtomicBool = AtomicBool::new(false);
+
+// stop listening with Atomic flag (to make it work between different threads)
+static STOP_LISTENING: AtomicBool = AtomicBool::new(false);
+
+// store tauri app_handle
+static TAURI_APP_HANDLE: OnceCell<tauri::AppHandle> = OnceCell::new();
+
+// store porcupine instance
+static PORCUPINE: OnceCell<Porcupine> = OnceCell::new();
+
+// store rustpotter instance
+static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
+
+#[tauri::command]
+pub fn is_listening() -> bool {
+    LISTENING.load(Ordering::SeqCst)
+}
+
+#[tauri::command]
+pub fn stop_listening() {
+    if is_listening() {
+        STOP_LISTENING.store(true, Ordering::SeqCst);
+        stop_recording();
+    }
+
+    // wait until listening stops
+    while is_listening() {}
+}
+
+fn get_wake_word_engine() -> config::WakeWordEngine {
+    let selected_wake_word_engine;
+    if let Some(wwengine) = DB.lock().unwrap().get::<String>("selected_wake_word_engine") {
+        // from db
+        match wwengine.trim().to_lowercase().as_str() {
+            "rustpotter" => selected_wake_word_engine = config::WakeWordEngine::Rustpotter,
+            "vosk" => selected_wake_word_engine = config::WakeWordEngine::Vosk,
+            "picovoice" => selected_wake_word_engine = config::WakeWordEngine::Porcupine,
+            _ => selected_wake_word_engine = config::DEFAULT_WAKE_WORD_ENGINE
+        }
+    } else {
+        // default
+        selected_wake_word_engine = config::DEFAULT_WAKE_WORD_ENGINE; // set default wake_word engine
+    }
+
+    selected_wake_word_engine
+}
+
+#[tauri::command(async)]
+pub fn start_listening(app_handle: tauri::AppHandle) -> Result<bool, String> {
+    // only one listener thread is allowed
+    if is_listening() {
+        return Err("Already listening.".into());
+    }
+
+    // keep app handle
+    if TAURI_APP_HANDLE.get().is_none() {
+        TAURI_APP_HANDLE.set(app_handle);
+    }
+
+    // call selected wake-word engine listener command
+    match get_wake_word_engine() {
+        config::WakeWordEngine::Rustpotter => {
+            info!("Starting RUSTPOTTER wake-word engine ...");
+            return rustpotter_init();
+        },
+        config::WakeWordEngine::Vosk => {
+            info!("Starting VOSK wake-word engine ...");
+            return vosk_init();
+        },
+        config::WakeWordEngine::Porcupine => {
+            info!("Starting PICOVOICE PORCUPINE wake-word engine ...");
+            return picovoice_init();
+        }
+    }
+}
+
+fn keyword_callback(_keyword_index: i32) {
+    // vars
+    let mut start: SystemTime = SystemTime::now();
+    let mut frame_buffer = vec![0; recorder::FRAME_LENGTH.load(Ordering::SeqCst) as usize];
+
+    // play greet phrase
+    events::play(
+        config::ASSISTANT_GREET_PHRASES
+            .choose(&mut rand::thread_rng())
+            .unwrap(),
+        TAURI_APP_HANDLE.get().unwrap(),
+    );
+
+    // emit assistant greet event
+    TAURI_APP_HANDLE.get().unwrap()
+        .emit_all(events::EventTypes::AssistantGreet.get(), ())
+        .unwrap();
+
+    // the loop
+    while !STOP_LISTENING.load(Ordering::SeqCst) {
+        recorder::read_microphone(&mut frame_buffer);
+
+        // vosk part (partials included)
+        if let Some(mut test) = vosk::recognize(&frame_buffer, false) {
+            if !test.is_empty() {
+                println!("Recognized: {}", test);
+
+                // some filtration
+                test = test.to_lowercase();
+                for tbr in config::ASSISTANT_PHRASES_TBR {
+                    test = test.replace(tbr, "");
+                }
+                test = test.trim().into();
+
+                // infer command
+                if let Some((cmd_path, cmd_config)) =
+                    assistant_commands::fetch_command(&test, &COMMANDS)
+                {
+                    println!("Recognized (filtered): {}", test);
+                    println!("Command found: {:?}", cmd_path);
+                    println!("Executing ...");
+
+                    let cmd_result = assistant_commands::execute_command(
+                        &cmd_path,
+                        &cmd_config,
+                        TAURI_APP_HANDLE.get().unwrap(),
+                    );
+
+                    match cmd_result {
+                        Ok(chain) => {
+                            println!("Command executed successfully!");
+
+                            if chain {
+                                // continue chaining commands
+                                start = SystemTime::now(); // listen for more commands
+                            } else {
+                                // skip forward if chaining is not required
+                                start = start.checked_sub(core::time::Duration::from_secs(1000)).unwrap();
+                            }
+
+                            continue;
+                        }
+                        Err(error_message) => {
+                            println!("Error executing command: {}", error_message);
+                        }
+                    }
+
+                    TAURI_APP_HANDLE.get().unwrap()
+                        .emit_all(events::EventTypes::AssistantWaiting.get(), ())
+                        .unwrap();
+                    break; // return to picovoice after command execution (no matter successfull or not)
+                }
+            }
+        }
+
+        match start.elapsed() {
+            Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
+                // return to picovoice after N seconds
+                TAURI_APP_HANDLE.get().unwrap()
+                    .emit_all(events::EventTypes::AssistantWaiting.get(), ())
+                    .unwrap();
+                break;
+            }
+            _ => (),
+        }
+    }
+}
+
+pub fn data_callback(frame_buffer: &[i16]) {
+    // println!("DATA CALLBACK {}", frame_buffer.len());
+    match get_wake_word_engine() {
+        config::WakeWordEngine::Rustpotter => {
+            let mut lock = RUSTPOTTER.get().unwrap().lock();
+            let rustpotter = lock.as_mut().unwrap();
+            let detection = rustpotter.process_i16(&frame_buffer);
+
+            if let Some(detection) = detection {
+                if detection.score > config::RUSPOTTER_MIN_SCORE {
+                    info!("Rustpotter detection info:\n{:?}", detection);
+                    keyword_callback(0);
+                } else {
+                    info!("Rustpotter detection info:\n{:?}", detection);
+                }
+            }
+        },
+        config::WakeWordEngine::Vosk => {
+            // recognize & convert to sequence
+            let recognized_phrase = vosk::recognize(&frame_buffer, true).unwrap_or("".into());
+
+            if !recognized_phrase.trim().is_empty() {
+                info!("Rec: {}", recognized_phrase);
+                let recognized_phrases = recognized_phrase.split_whitespace();
+                for phrase in recognized_phrases {
+                    let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
+            
+                    // compare
+                    let compare_ratio = seqdiff::ratio(&config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(), &recognized_phrase_chars);
+                    info!("OG phrase: {:?}", &config::VOSK_FETCH_PHRASE);
+                    info!("Recognized phrase: {:?}", &recognized_phrase_chars);
+                    info!("Compare ratio: {}", compare_ratio);
+
+                    if compare_ratio >= config::VOSK_MIN_RATIO {
+                        info!("Phrase activated.");
+                        keyword_callback(0);
+                        break;
+                    }
+                }
+            }
+        },
+        config::WakeWordEngine::Porcupine => {
+            if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) {
+                if keyword_index >= 0 {
+                    // println!("Yes, sir! {}", keyword_index);
+                    keyword_callback(keyword_index);
+                }
+            }
+        }
+    }
+}
+
+fn start_recording() -> Result<bool, String> {
+    // vars
+    let frame_length: usize;
+
+    // idenfity frame length
+    match get_wake_word_engine() {
+        config::WakeWordEngine::Rustpotter => {
+            // start recording for Rustpotter
+            // You need a buffer of size `rustpotter.get_samples_per_frame()` when using samples.
+            // You need a buffer of size `rustpotter.get_bytes_per_frame()` when using bytes.
+            frame_length = RUSTPOTTER.get().unwrap().lock().unwrap().get_samples_per_frame();
+            recorder::FRAME_LENGTH.store(frame_length as u32, Ordering::SeqCst);
+        },
+        config::WakeWordEngine::Vosk => {
+            // start recording for Vosk
+            frame_length = 128;
+            recorder::FRAME_LENGTH.store(frame_length as u32, Ordering::SeqCst);
+        },
+        config::WakeWordEngine::Porcupine => {
+            // start recording for Porcupine
+            frame_length = PORCUPINE.get().unwrap().frame_length() as usize;
+            recorder::FRAME_LENGTH.store(PORCUPINE.get().unwrap().frame_length(), Ordering::SeqCst);
+        }
+    }
+
+    // define frame buffer
+    let mut frame_buffer: Vec<i16> = vec![0; frame_length];
+
+    // init stuff
+    recorder::init(); // init
+    recorder::start_recording(); // start
+    LISTENING.store(true, Ordering::SeqCst);
+    info!("START listening ...");
+
+    // greet user
+    events::play("run", TAURI_APP_HANDLE.get().unwrap());
+
+    // record
+    match recorder::RECORDER_TYPE.load(Ordering::SeqCst) {
+        recorder::RecorderType::PvRecorder => {
+            while !STOP_LISTENING.load(Ordering::SeqCst) {
+                recorder::read_microphone(&mut frame_buffer);
+                data_callback(&frame_buffer);
+            }
+
+            // stop
+            stop_recording();
+
+            Ok(true)
+        },
+        recorder::RecorderType::PortAudio => {
+            while !STOP_LISTENING.load(Ordering::SeqCst) {
+                recorder::read_microphone(&mut frame_buffer);
+                data_callback(&frame_buffer);
+            }
+
+            // stop
+            stop_recording();
+
+            Ok(true)
+        }
+        recorder::RecorderType::Cpal => {
+            todo!()
+        }
+    }
+}
+
+fn stop_recording() {
+    // Stop listening
+    recorder::stop_recording();
+
+    LISTENING.store(false, Ordering::SeqCst);
+    STOP_LISTENING.store(false, Ordering::SeqCst);
+    info!("STOP listening ...");
+}
+
+fn rustpotter_init() -> Result<bool, String> {
+
+    // init rustpotter
+    let rustpotter_config = RustpotterConfig {
+        fmt: WavFmt::default(),
+        detector: DetectorConfig {
+            avg_threshold: 0.,
+            threshold: 0.5,
+            min_scores: 15,
+            score_mode: ScoreMode::Average,
+            comparator_band_size: 5,
+            comparator_ref: 0.22
+        },
+        filters: FiltersConfig {
+            gain_normalizer: GainNormalizationConfig {
+                enabled: true,
+                gain_ref: None,
+                min_gain: 0.7,
+                max_gain: 1.0,
+            },
+            band_pass: BandPassConfig {
+                enabled: true,
+                low_cutoff: 80.,
+                high_cutoff: 400.,
+            }
+        }
+    };
+    let mut rustpotter = Rustpotter::new(&rustpotter_config).unwrap();
+
+    // load a wakeword
+    let rustpotter_wake_word_files: [&str; 5] = [
+        "rustpotter/jarvis-default.rpw",
+        "rustpotter/jarvis-community-1.rpw",
+        "rustpotter/jarvis-community-2.rpw",
+        "rustpotter/jarvis-community-3.rpw",
+        "rustpotter/jarvis-community-4.rpw",
+        // "rustpotter/jarvis-community-5.rpw",
+    ];
+
+    for rpw in rustpotter_wake_word_files {
+        rustpotter.add_wakeword_from_file(rpw).unwrap();
+    }
+
+    // store rustpotter
+    if RUSTPOTTER.get().is_none() {
+        RUSTPOTTER.set(Mutex::new(rustpotter));
+    }
+
+    // start recording
+    start_recording()
+}
+
+fn vosk_init() -> Result<bool, String> {
+    start_recording()
+}
+
+fn picovoice_init() -> Result<bool, String> {
+    // VARS
+    let porcupine: Porcupine;
+    let picovoice_api_key: String;
+
+    // Retrieve API key from DB
+    if let Some(pkey) = DB.lock().unwrap().get::<String>("api_key__picovoice") {
+        picovoice_api_key = pkey;
+    } else {
+        warn!("Picovoice API key is not set!");
+        return Err("Picovoice API key is not set!".into());
+    }
+
+    // Create instance of Porcupine with the given API key
+    match PorcupineBuilder::new_with_keyword_paths(picovoice_api_key, &[Path::new(config::KEYWORDS_PATH).join("jarvis_windows.ppn")])
+        .sensitivities(&[1.0f32]) // max sensitivity possible
+        .init() {
+            Ok(pinstance) => {
+                // porcupine successfully initialized with the valid API key
+                info!("Porcupine successfully initialized with the valid API key ...");
+                porcupine = pinstance;
+            }
+            Err(e) => {
+                error!("Porcupine error: either API key is not valid or there is no internet connection");
+                error!("Error details: {}", e);
+                return Err(
+                    "Porcupine error: either API key is not valid or there is no internet connection"
+                        .into(),
+                );
+            }
+    }
+
+    // store
+    if PORCUPINE.get().is_none() {
+        PORCUPINE.set(porcupine);
+    }
+
+    // start recording
+    start_recording()
+}

crates/jarvis-gui/src/tauri_commands/sys.rs

@@ -0,0 +1,48 @@
+use peak_alloc::PeakAlloc;
+
+#[global_allocator]
+static PEAK_ALLOC: PeakAlloc = PeakAlloc;
+
+extern crate systemstat;
+use std::thread;
+use std::time::Duration;
+use systemstat::{Platform, System};
+
+lazy_static! {
+    static ref SYS: System = System::new();
+}
+
+#[tauri::command]
+pub fn get_current_ram_usage() -> String {
+    let result = String::from(format!("{}", PEAK_ALLOC.current_usage_as_mb()));
+
+    result
+}
+
+#[tauri::command]
+pub fn get_peak_ram_usage() -> String {
+    let result = String::from(format!("{}", PEAK_ALLOC.peak_usage_as_gb()));
+
+    result
+}
+
+#[tauri::command]
+pub fn get_cpu_temp() -> String {
+    if let Ok(cpu_temp) = SYS.cpu_temp() {
+        String::from(format!("{}", cpu_temp))
+    } else {
+        String::from("error")
+    }
+}
+
+// https://github.com/valpackett/systemstat/blob/trunk/examples/info.rs
+#[tauri::command(async)]
+pub async fn get_cpu_usage() -> String {
+    if let Ok(cpu) = SYS.cpu_load_aggregate() {
+        thread::sleep(Duration::from_secs(1));
+        let cpu = cpu.done().unwrap();
+        String::from(format!("{}", cpu.user * 100.0))
+    } else {
+        String::from("error")
+    }
+}

crates/jarvis-gui/src/tauri_commands/voice.rs

@@ -0,0 +1,29 @@
+use std::fs::File;
+use std::io::BufReader;
+use rodio::{Decoder, OutputStream, Sink};
+
+#[tauri::command(async)]
+pub fn play_sound(filename: &str, sleep: bool) {
+    // Get a output stream handle to the default physical sound device
+    let (_stream, stream_handle) = OutputStream::try_default().unwrap();
+    let sink = Sink::try_new(&stream_handle).unwrap();
+    
+    // Load a sound from a file, using a path relative to Cargo.toml
+    // let filepath = format!("{PUBLIC_PATH}/sound/{filename}.wav");
+    let filepath = filename;
+    let file = BufReader::new(File::open(&filepath).unwrap());
+    
+    // Decode that sound file into a source
+    let source = Decoder::new(file).unwrap();
+    
+    // Play the sound directly on the device
+    println!("Playing {} ...", filepath);
+    // stream_handle.play_raw(source.convert_samples());
+    sink.append(source);
+
+    if sleep {
+        // The sound plays in a separate thread. This call will block the current thread until the sink
+        // has finished playing all its queued sounds.
+        sink.sleep_until_end();
+    }
+}