bossiara13/J.A.R.V.I.S-py
1
1
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Activity Actions

Latest changes

Browse source
This commit is contained in:
Priler 2025-12-11 23:43:50 +05:00
parent 909b1ab0e4
commit 703e316257
26 changed files with 1917 additions and 709 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/.gitignore vendored
View file

@ -1,3 +1,5 @@
# Generated by Cargo # Generated by Cargo
# will have compiled files and executables # will have compiled files and executables
/target/ /target/
*.db
log.txt

1908
app/Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

34
app/Cargo.toml
View file

@ -10,26 +10,28 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
serde = { version = "1.0.164", features = ["derive"] } serde = { version = "1.0.197", features = ["derive"] }
serde_json = "1.0.96" serde_json = "1.0.114"
hound = "3.5.0" hound = "3.5.1"
pv_recorder = "1.1.2" pv_recorder = "1.2.2"
pv_porcupine = "2.2.1" pv_porcupine = "3.0.2"
seqdiff = "0.3.0" seqdiff = "0.3.0"
vosk = "0.2.0" vosk = "0.2.0"
rand = "0.8.5" rand = "0.9.0-alpha.1"
rodio = "0.17.1" rodio = "0.17.3"
rustpotter = "2.0.0" rustpotter = "3.0.2"
log = "0.4.18" log = "0.4.21"
once_cell = "1.18.0" once_cell = "1.19.0"
atomic_enum = "0.2.0" atomic_enum = "0.3.0"
portaudio = "0.7.0" portaudio = "0.7.0"
platform-dirs = "0.3.0" platform-dirs = "0.3.0"
simple-log = "1.6.0" simple-log = "1.6.0"
tray-icon = { version = "0.5.1" } tray-icon = { version = "0.12.0" }
winit = "0.28.6" winit = "0.29.15"
image = "0.24.6" image = "0.25.0"
serde_yaml = "0.9.21" serde_yaml = "0.9.33"
kira = "0.8.3" kira = "0.8.7"
[features] [features]
default = ["jarvis_app"]
jarvis_app = [] # feature flag saying this is an app

2
app/post_build.py
View file

@ -15,6 +15,8 @@ SOURCE = (
"vosk/", "vosk/",
"lib/", "lib/",
"keywords/", "keywords/",
"rustpotter/",
"sound/",
"libgcc_s_seh-1.dll", "libgcc_s_seh-1.dll",
"libstdc++-6.dll", "libstdc++-6.dll",
"libvosk.dll", "libvosk.dll",

36
app/src/app.rs
View file

@ -1,7 +1,7 @@
use std::time::SystemTime; use std::time::SystemTime;
use crate::{config, audio, recorder, listener, stt, commands, COMMANDS_LIST}; use crate::{audio, commands, config, listener, recorder, stt, COMMANDS_LIST};
use rand::seq::SliceRandom; use rand::seq::IndexedRandom;
pub fn start() -> Result<(), ()> { pub fn start() -> Result<(), ()> {
// start the loop // start the loop
@ -41,7 +41,12 @@ fn main_loop() -> Result<(), ()> {
// play some greet phrase // play some greet phrase
// @TODO. Make it via commands or upcoming events system. // @TODO. Make it via commands or upcoming events system.
audio::play_sound(&sounds_directory.join(format!("{}.wav", config::ASSISTANT_GREET_PHRASES.choose(&mut rand::thread_rng()).unwrap()))); audio::play_sound(&sounds_directory.join(format!(
"{}.wav",
config::ASSISTANT_GREET_PHRASES
.choose(&mut rand::thread_rng())
.unwrap()
)));
// wait for voice commands // wait for voice commands
'voice_recognition: loop { 'voice_recognition: loop {
@ -62,7 +67,10 @@ fn main_loop() -> Result<(), ()> {
recognized_voice = recognized_voice.trim().into(); recognized_voice = recognized_voice.trim().into();
// infer command // infer command
if let Some((cmd_path, cmd_config)) = commands::fetch_command(&recognized_voice, &COMMANDS_LIST.get().unwrap()) { if let Some((cmd_path, cmd_config)) = commands::fetch_command(
&recognized_voice,
&COMMANDS_LIST.get().unwrap(),
) {
// some debug info // some debug info
info!("Recognized voice (filtered): {}", recognized_voice); info!("Recognized voice (filtered): {}", recognized_voice);
info!("Command found: {:?}", cmd_path); info!("Command found: {:?}", cmd_path);
@ -79,11 +87,13 @@ fn main_loop() -> Result<(), ()> {
start = SystemTime::now(); start = SystemTime::now();
} else { } else {
// skip, if chaining is not required // skip, if chaining is not required
start = start.checked_sub(core::time::Duration::from_secs(1000)).unwrap(); start = start
.checked_sub(core::time::Duration::from_secs(1000))
.unwrap();
} }
continue 'voice_recognition; // continue voice recognition continue 'voice_recognition; // continue voice recognition
}, }
Err(msg) => { Err(msg) => {
// fail // fail
error!("Error executing command: {}", msg); error!("Error executing command: {}", msg);
@ -100,23 +110,21 @@ fn main_loop() -> Result<(), ()> {
Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => { Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
// return to wake-word listening after N seconds // return to wake-word listening after N seconds
break 'voice_recognition; break 'voice_recognition;
}, }
_ => () _ => (),
} }
} }
}, }
None => () None => (),
} }
} }
Ok(()) Ok(())
} }
fn keyword_callback(keyword_index: i32) { fn keyword_callback(keyword_index: i32) {}
}
pub fn close(code: i32) { pub fn close(code: i32) {
info!("Closing application."); info!("Closing application.");
std::process::exit(code); std::process::exit(code);
} }

32
app/src/audio.rs
View file

@ -1,18 +1,19 @@
mod rodio;
mod kira; mod kira;
mod rodio;
use once_cell::sync::OnceCell;
use std::cmp::Ordering; use std::cmp::Ordering;
use std::path::PathBuf; use std::path::PathBuf;
use once_cell::sync::OnceCell;
use crate::{config, DB, SOUND_DIR};
use crate::config::structs::AudioType; use crate::config::structs::AudioType;
use crate::{config, DB, SOUND_DIR};
static AUDIO_TYPE: OnceCell<AudioType> = OnceCell::new(); static AUDIO_TYPE: OnceCell<AudioType> = OnceCell::new();
pub fn init() -> Result<(), ()> { pub fn init() -> Result<(), ()> {
if !AUDIO_TYPE.get().is_none() {return Ok(());} // already initialized if !AUDIO_TYPE.get().is_none() {
return Ok(());
} // already initialized
// set default audio type // set default audio type
// @TODO. Make it configurable? // @TODO. Make it configurable?
@ -27,14 +28,14 @@ pub fn init() -> Result<(), ()> {
match rodio::init() { match rodio::init() {
Ok(_) => { Ok(_) => {
info!("Successfully initialized Rodio audio backend."); info!("Successfully initialized Rodio audio backend.");
}, }
Err(msg) => { Err(msg) => {
error!("Failed to initialize Rodio audio backend."); error!("Failed to initialize Rodio audio backend.");
return Err(()) return Err(());
} }
} }
}, }
AudioType::Kira => { AudioType::Kira => {
// Init Kira // Init Kira
info!("Initializing Kira audio backend."); info!("Initializing Kira audio backend.");
@ -42,11 +43,11 @@ pub fn init() -> Result<(), ()> {
match kira::init() { match kira::init() {
Ok(_) => { Ok(_) => {
info!("Successfully initialized Kira audio backend."); info!("Successfully initialized Kira audio backend.");
}, }
Err(msg) => { Err(msg) => {
error!("Failed to initialize Kira audio backend."); error!("Failed to initialize Kira audio backend.");
return Err(()) return Err(());
} }
} }
} }
@ -61,10 +62,8 @@ pub fn play_sound(filename: &PathBuf) {
match AUDIO_TYPE.get().unwrap() { match AUDIO_TYPE.get().unwrap() {
AudioType::Rodio => { AudioType::Rodio => {
rodio::play_sound(filename, true); rodio::play_sound(filename, true);
},
AudioType::Kira => {
kira::play_sound(filename)
} }
AudioType::Kira => kira::play_sound(filename),
} }
} }
@ -79,8 +78,11 @@ pub fn get_sound_directory() -> Option<PathBuf> {
match default_voice_path.exists() { match default_voice_path.exists() {
true => Some(default_voice_path), true => Some(default_voice_path),
_ => None _ => {
error!("No sounds found. Search path - {:?}", voice_path);
None
}
} }
} }
} }
} }

19
app/src/audio/kira.rs
View file

@ -1,20 +1,19 @@
use once_cell::sync::OnceCell;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Mutex; use std::sync::Mutex;
use once_cell::sync::OnceCell;
use kira::{ use kira::{
manager::{ manager::{backend::DefaultBackend, AudioManager, AudioManagerSettings},
AudioManager, AudioManagerSettings, sound::static_sound::{StaticSoundData, StaticSoundSettings},
backend::DefaultBackend,
},
sound::static_sound::{StaticSoundData, StaticSoundSettings},
}; };
thread_local!(static MANAGER: OnceCell<Mutex<AudioManager>> = OnceCell::new()); thread_local!(static MANAGER: OnceCell<Mutex<AudioManager>> = OnceCell::new());
pub fn init() -> Result<(), ()> { pub fn init() -> Result<(), ()> {
MANAGER.with(|m| { MANAGER.with(|m| {
if !m.get().is_none() {return Ok(());} // already initialized if !m.get().is_none() {
return Ok(());
} // already initialized
// Create an audio manager. This plays sounds and manages resources. // Create an audio manager. This plays sounds and manages resources.
match AudioManager::<DefaultBackend>::new(AudioManagerSettings::default()) { match AudioManager::<DefaultBackend>::new(AudioManagerSettings::default()) {
@ -24,7 +23,7 @@ pub fn init() -> Result<(), ()> {
// success // success
Ok(()) Ok(())
}, }
Err(msg) => { Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg); error!("Failed to initialize audio stream.\nError details: {}", msg);
@ -47,9 +46,9 @@ pub fn play_sound(filename: &PathBuf) {
let audio_manager = &mut m.get().unwrap().lock().unwrap(); let audio_manager = &mut m.get().unwrap().lock().unwrap();
audio_manager.play(sound_data.clone()).unwrap(); audio_manager.play(sound_data.clone()).unwrap();
}); });
}, }
Err(msg) => { Err(msg) => {
warn!("Cannot find sound file: {}", filename.display()); warn!("Cannot find sound file: {}", filename.display());
} }
} }
} }

18
app/src/audio/rodio.rs
View file

@ -4,10 +4,10 @@
Possible fixes are running rodio in a separate thread or smthng. Possible fixes are running rodio in a separate thread or smthng.
*/ */
use std::fs::File;
use std::path::PathBuf;
use std::io::BufReader;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink}; use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink};
@ -16,7 +16,9 @@ static STREAM_HANDLE: OnceCell<OutputStreamHandle> = OnceCell::new();
static SINK: OnceCell<Sink> = OnceCell::new(); static SINK: OnceCell<Sink> = OnceCell::new();
pub fn init() -> Result<(), ()> { pub fn init() -> Result<(), ()> {
if !STREAM_HANDLE.get().is_none() {return Ok(());} // already initialized if !STREAM_HANDLE.get().is_none() {
return Ok(());
} // already initialized
// get output stream handle to the default physical sound device // get output stream handle to the default physical sound device
match OutputStream::try_default() { match OutputStream::try_default() {
@ -30,12 +32,12 @@ pub fn init() -> Result<(), ()> {
Ok(s) => { Ok(s) => {
info!("Sink initialized."); info!("Sink initialized.");
sink = s; sink = s;
}, }
Err(msg) => { Err(msg) => {
error!("Cannot create sink.\nError details: {}", msg); error!("Cannot create sink.\nError details: {}", msg);
// failed // failed
return Err(()) return Err(());
} }
} }
@ -46,7 +48,7 @@ pub fn init() -> Result<(), ()> {
// success // success
Ok(()) Ok(())
}, }
Err(msg) => { Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg); error!("Failed to initialize audio stream.\nError details: {}", msg);
@ -73,4 +75,4 @@ pub fn play_sound(filename: &PathBuf, sleep: bool) {
// has finished playing all its queued sounds. // has finished playing all its queued sounds.
SINK.get().unwrap().sleep_until_end(); SINK.get().unwrap().sleep_until_end();
} }
} }

92
app/src/commands.rs
View file

@ -1,4 +1,4 @@
use rand::seq::SliceRandom; use rand::seq::IndexedRandom;
use seqdiff::ratio; use seqdiff::ratio;
use serde_yaml; use serde_yaml;
use std::path::Path; use std::path::Path;
@ -6,13 +6,13 @@ use std::{fs, fs::File};
use core::time::Duration; use core::time::Duration;
use std::path::PathBuf; use std::path::PathBuf;
use std::process::{Command, Child}; use std::process::{Child, Command};
// use tauri::Manager; // use tauri::Manager;
mod structs; mod structs;
pub use structs::*; pub use structs::*;
use crate::{config, audio}; use crate::{audio, config};
// @TODO. Allow commands both in yaml and json format. // @TODO. Allow commands both in yaml and json format.
pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> { pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
@ -35,9 +35,13 @@ pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
match serde_yaml::from_reader::<File, CommandsList>(cc_reader) { match serde_yaml::from_reader::<File, CommandsList>(cc_reader) {
Ok(parse_result) => { Ok(parse_result) => {
cc_yaml = parse_result; cc_yaml = parse_result;
}, }
Err(msg) => { Err(msg) => {
warn!("Can't parse {}, skipping ...\nCommand parse error is: {:?}", &cc_file.display(), msg); warn!(
"Can't parse {}, skipping ...\nCommand parse error is: {:?}",
&cc_file.display(),
msg
);
continue; continue;
} }
} }
@ -98,7 +102,10 @@ pub fn fetch_command<'a>(
if let Some((cmd_path, scmd)) = result_scmd { if let Some((cmd_path, scmd)) = result_scmd {
println!("Ratio is: {}", current_max_ratio); println!("Ratio is: {}", current_max_ratio);
info!("CMD is: {cmd_path:?}, SCMD is: {scmd:?}, Ratio is: {}", current_max_ratio); info!(
"CMD is: {cmd_path:?}, SCMD is: {scmd:?}, Ratio is: {}",
current_max_ratio
);
Some((&cmd_path, &scmd)) Some((&cmd_path, &scmd))
} else { } else {
None None
@ -111,21 +118,12 @@ pub fn execute_exe(exe: &str, args: &Vec<String>) -> std::io::Result<Child> {
} }
pub fn execute_cli(cmd: &str, args: &Vec<String>) -> std::io::Result<Child> { pub fn execute_cli(cmd: &str, args: &Vec<String>) -> std::io::Result<Child> {
println!("Spawning cmd as: cmd /C {} {:?}", cmd, args); println!("Spawning cmd as: cmd /C {} {:?}", cmd, args);
if cfg!(target_os = "windows") { if cfg!(target_os = "windows") {
Command::new("cmd") Command::new("cmd").arg("/C").arg(cmd).args(args).spawn()
.arg("/C")
.arg(cmd)
.args(args)
.spawn()
} else { } else {
Command::new("sh") Command::new("sh").arg("-c").arg(cmd).args(args).spawn()
.arg("-c")
.arg(cmd)
.args(args)
.spawn()
} }
} }
@ -139,7 +137,14 @@ pub fn execute_command(
match cmd_config.command.action.as_str() { match cmd_config.command.action.as_str() {
"voice" => { "voice" => {
// VOICE command type // VOICE command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap()); let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle); // events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound)); audio::play_sound(&sounds_directory.join(random_cmd_sound));
@ -158,7 +163,14 @@ pub fn execute_command(
}, },
&cmd_config.command.exe_args, &cmd_config.command.exe_args,
) { ) {
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap()); let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle); // events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound)); audio::play_sound(&sounds_directory.join(random_cmd_sound));
@ -172,17 +184,21 @@ pub fn execute_command(
// CLI command type // CLI command type
let cli_cmd = &cmd_config.command.cli_cmd; let cli_cmd = &cmd_config.command.cli_cmd;
match execute_cli( match execute_cli(cli_cmd, &cmd_config.command.cli_args) {
cli_cmd, Ok(_) => {
&cmd_config.command.cli_args, let random_cmd_sound = format!(
) { "{}.wav",
Ok(_) => { cmd_config
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap()); .voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle); // events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound)); audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(true) Ok(true)
}, }
Err(msg) => { Err(msg) => {
error!("CLI command error ({})", msg); error!("CLI command error ({})", msg);
Err(format!("Shell command error ({})", msg).into()) Err(format!("Shell command error ({})", msg).into())
@ -191,7 +207,14 @@ pub fn execute_command(
} }
"terminate" => { "terminate" => {
// TERMINATE command type // TERMINATE command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap()); let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle); // events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound)); audio::play_sound(&sounds_directory.join(random_cmd_sound));
@ -200,7 +223,14 @@ pub fn execute_command(
} }
"stop_chaining" => { "stop_chaining" => {
// STOP_CHAINING command type // STOP_CHAINING command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap()); let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle); // events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound)); audio::play_sound(&sounds_directory.join(random_cmd_sound));
@ -209,7 +239,7 @@ pub fn execute_command(
_ => { _ => {
error!("Command type unknown"); error!("Command type unknown");
Err("Command type unknown".into()) Err("Command type unknown".into())
}, }
} }
} }
@ -222,4 +252,4 @@ pub fn list(from: &[AssistantCommand]) -> Vec<String> {
} }
out out
} }

3
app/src/commands/structs.rs
View file

@ -35,12 +35,11 @@ pub struct ConfigCommandSection {
pub cli_cmd: String, pub cli_cmd: String,
#[serde(default)] #[serde(default)]
pub cli_args: Vec<String> pub cli_args: Vec<String>,
} }
#[derive(Deserialize, Debug)] #[derive(Deserialize, Debug)]
pub struct ConfigVoiceSection { pub struct ConfigVoiceSection {
#[serde(default)] #[serde(default)]
pub sounds: Vec<String>, pub sounds: Vec<String>,
} }

58
app/src/config.rs
View file

@ -1,21 +1,25 @@
pub mod structs; pub mod structs;
use structs::WakeWordEngine;
use structs::SpeechToTextEngine;
use structs::RecorderType;
use structs::AudioType; use structs::AudioType;
use structs::RecorderType;
use structs::SpeechToTextEngine;
use structs::WakeWordEngine;
use std::fs;
use std::env;
use std::path::PathBuf;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use std::env;
use std::fs;
use std::path::PathBuf;
use platform_dirs::{AppDirs}; use platform_dirs::AppDirs;
use rustpotter::{RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig};
use crate::{config, APP_DIRS, APP_CONFIG_DIR, APP_LOG_DIR}; #[cfg(feature="jarvis_app")]
use rustpotter::{
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig,
RustpotterConfig, ScoreMode,
};
use crate::{config, APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
#[allow(dead_code)] #[allow(dead_code)]
pub fn init_dirs() -> Result<(), String> { pub fn init_dirs() -> Result<(), String> {
// infer app dirs // infer app dirs
if APP_DIRS.get().is_some() { if APP_DIRS.get().is_some() {
@ -23,7 +27,9 @@ pub fn init_dirs() -> Result<(), String> {
} }
// cache_dir, config_dir, data_dir, state_dir // cache_dir, config_dir, data_dir, state_dir
APP_DIRS.set(AppDirs::new(Some(config::BUNDLE_IDENTIFIER), false).unwrap()).unwrap(); APP_DIRS
.set(AppDirs::new(Some(config::BUNDLE_IDENTIFIER), false).unwrap())
.unwrap();
// setup directories // setup directories
let mut config_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir); let mut config_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
@ -33,7 +39,8 @@ pub fn init_dirs() -> Result<(), String> {
if !config_dir.exists() { if !config_dir.exists() {
if fs::create_dir_all(&config_dir).is_err() { if fs::create_dir_all(&config_dir).is_err() {
config_dir = env::current_dir().expect("Cannot infer the config directory"); config_dir = env::current_dir().expect("Cannot infer the config directory");
fs::create_dir_all(&config_dir).expect("Cannot create config directory, access denied?"); fs::create_dir_all(&config_dir)
.expect("Cannot create config directory, access denied?");
} }
} }
@ -51,10 +58,9 @@ pub fn init_dirs() -> Result<(), String> {
Ok(()) Ok(())
} }
/* /*
Defaults. Defaults.
*/ */
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira; pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder; pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter; pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter;
@ -72,23 +78,29 @@ pub const TG_OFFICIAL_LINK: Option<&str> = Some("https://t.me/howdyho_official")
pub const FEEDBACK_LINK: Option<&str> = Some("https://t.me/jarvis_feedback_bot"); pub const FEEDBACK_LINK: Option<&str> = Some("https://t.me/jarvis_feedback_bot");
/* /*
Tray. Tray.
*/ */
pub const TRAY_ICON: &str = "32x32.png"; pub const TRAY_ICON: &str = "32x32.png";
pub const TRAY_TOOLTIP: &str = "Jarvis Voice Assistant"; pub const TRAY_TOOLTIP: &str = "Jarvis Voice Assistant";
// RUSPOTTER // RUSPOTTER
#[cfg(feature="jarvis_app")]
pub const RUSPOTTER_MIN_SCORE: f32 = 0.62; pub const RUSPOTTER_MIN_SCORE: f32 = 0.62;
#[cfg(feature="jarvis_app")]
pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| { pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
RustpotterConfig { RustpotterConfig {
fmt: WavFmt::default(), fmt: AudioFmt::default(),
detector: DetectorConfig { detector: DetectorConfig {
avg_threshold: 0., avg_threshold: 0.,
threshold: 0.5, threshold: 0.5,
min_scores: 15, min_scores: 15,
score_mode: ScoreMode::Average, score_ref: 0.22,
comparator_band_size: 5, band_size: 5,
comparator_ref: 0.22 vad_mode: None,
score_mode: ScoreMode::Max,
eager: false,
// comparator_band_size: 5,
// comparator_ref: 0.22
}, },
filters: FiltersConfig { filters: FiltersConfig {
gain_normalizer: GainNormalizationConfig { gain_normalizer: GainNormalizationConfig {
@ -101,8 +113,8 @@ pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
enabled: true, enabled: true,
low_cutoff: 80., low_cutoff: 80.,
high_cutoff: 400., high_cutoff: 400.,
} },
} },
} }
}); });

23
app/src/config/structs.rs
View file

@ -1,30 +1,43 @@
use std::fmt;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Serialize, Deserialize, Debug)] #[derive(Clone, Copy, Serialize, Deserialize, Debug)]
pub enum WakeWordEngine { pub enum WakeWordEngine {
Rustpotter, Rustpotter,
Vosk, Vosk,
Porcupine Porcupine,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
pub enum SpeechToTextEngine { pub enum SpeechToTextEngine {
Vosk Vosk,
}
impl fmt::Display for SpeechToTextEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
} }
#[derive(PartialEq, Debug)] #[derive(PartialEq, Debug)]
pub enum RecorderType { pub enum RecorderType {
Cpal, Cpal,
PvRecorder, PvRecorder,
PortAudio PortAudio,
} }
#[derive(PartialEq, Debug)] #[derive(PartialEq, Debug)]
pub enum AudioType { pub enum AudioType {
Rodio, Rodio,
Kira Kira,
} }
// pub enum TextToSpeechEngine {} // pub enum TextToSpeechEngine {}
// pub enum IntentRecognitionEngine {} // pub enum IntentRecognitionEngine {}

19
app/src/db.rs
View file

@ -1,22 +1,29 @@
pub mod structs; pub mod structs;
use crate::{config, APP_CONFIG_DIR}; use crate::{config, APP_CONFIG_DIR};
use std::path::PathBuf; use log::info;
use std::fs::File; use std::fs::File;
use std::io::{BufReader, Read}; use std::io::{BufReader, Read};
use log::info; use std::path::PathBuf;
use serde_json; use serde_json;
fn get_db_file_path() -> PathBuf { fn get_db_file_path() -> PathBuf {
PathBuf::from(format!("{}/{}", APP_CONFIG_DIR.get().unwrap().display(), config::DB_FILE_NAME)) PathBuf::from(format!(
"{}/{}",
APP_CONFIG_DIR.get().unwrap().display(),
config::DB_FILE_NAME
))
} }
pub fn init_settings() -> structs::Settings { pub fn init_settings() -> structs::Settings {
let mut db = None; let mut db = None;
let db_file_path = get_db_file_path(); let db_file_path = get_db_file_path();
info!("Loading settings db file located at: {}", db_file_path.display()); info!(
"Loading settings db file located at: {}",
db_file_path.display()
);
if db_file_path.exists() { if db_file_path.exists() {
// try load existing settings // try load existing settings
@ -43,9 +50,9 @@ pub fn save_settings(settings: &structs::Settings) -> Result<(), std::io::Error>
std::fs::write( std::fs::write(
db_file_path, db_file_path,
serde_json::to_string_pretty(&settings).unwrap() serde_json::to_string_pretty(&settings).unwrap(),
)?; )?;
info!("Settings saved."); info!("Settings saved.");
Ok(()) Ok(())
} }

14
app/src/db/structs.rs
View file

@ -1,8 +1,8 @@
use serde::{Deserialize, Serialize};
use crate::config; use crate::config;
use serde::{Deserialize, Serialize};
use crate::config::structs::WakeWordEngine;
use crate::config::structs::SpeechToTextEngine; use crate::config::structs::SpeechToTextEngine;
use crate::config::structs::WakeWordEngine;
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
pub struct Settings { pub struct Settings {
@ -12,7 +12,7 @@ pub struct Settings {
pub wake_word_engine: WakeWordEngine, pub wake_word_engine: WakeWordEngine,
pub speech_to_text_engine: SpeechToTextEngine, pub speech_to_text_engine: SpeechToTextEngine,
pub api_keys: ApiKeys pub api_keys: ApiKeys,
} }
impl Default for Settings { impl Default for Settings {
@ -26,8 +26,8 @@ impl Default for Settings {
api_keys: ApiKeys { api_keys: ApiKeys {
picovoice: String::from(""), picovoice: String::from(""),
openai: String::from("") openai: String::from(""),
} },
} }
} }
} }
@ -35,5 +35,5 @@ impl Default for Settings {
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
pub struct ApiKeys { pub struct ApiKeys {
pub picovoice: String, pub picovoice: String,
pub openai: String pub openai: String,
} }

30
app/src/listener.rs
View file

@ -7,8 +7,8 @@ mod vosk;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use crate::{config, stt};
use crate::config::structs::WakeWordEngine; use crate::config::structs::WakeWordEngine;
use crate::{config, stt};
use crate::DB; use crate::DB;
@ -19,10 +19,14 @@ static WAKE_WORD_ENGINE: OnceCell<WakeWordEngine> = OnceCell::new();
static LISTENING: AtomicBool = AtomicBool::new(false); static LISTENING: AtomicBool = AtomicBool::new(false);
pub fn init() -> Result<(), ()> { pub fn init() -> Result<(), ()> {
if !WAKE_WORD_ENGINE.get().is_none() {return Ok(());} // already initialized if !WAKE_WORD_ENGINE.get().is_none() {
return Ok(());
} // already initialized
// store current engine // store current engine
WAKE_WORD_ENGINE.set(DB.get().unwrap().wake_word_engine).unwrap(); WAKE_WORD_ENGINE
.set(DB.get().unwrap().wake_word_engine)
.unwrap();
// load given wake-word engine // load given wake-word engine
match WAKE_WORD_ENGINE.get().unwrap() { match WAKE_WORD_ENGINE.get().unwrap() {
@ -31,33 +35,27 @@ pub fn init() -> Result<(), ()> {
info!("Initializing Porcupine wake-word engine."); info!("Initializing Porcupine wake-word engine.");
return porcupine::init(); return porcupine::init();
}, }
WakeWordEngine::Rustpotter => { WakeWordEngine::Rustpotter => {
// Init Rustpotter wake-word engine // Init Rustpotter wake-word engine
info!("Initializing Rustpotter wake-word engine."); info!("Initializing Rustpotter wake-word engine.");
return rustpotter::init(); return rustpotter::init();
}, }
WakeWordEngine::Vosk => { WakeWordEngine::Vosk => {
// Init Vosk as wake-word engine (very slow, though) // Init Vosk as wake-word engine (very slow, though)
info!("Initializing Vosk as wake-word engine."); info!("Initializing Vosk as wake-word engine.");
warn!("Using Vosk as wake-word engine is highly not recommended, because it's very slow for this task."); warn!("Using Vosk as wake-word engine is highly not recommended, because it's very slow for this task.");
return vosk::init(); return vosk::init();
}, }
} }
} }
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> { pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
match WAKE_WORD_ENGINE.get().unwrap() { match WAKE_WORD_ENGINE.get().unwrap() {
WakeWordEngine::Porcupine => { WakeWordEngine::Porcupine => porcupine::data_callback(frame_buffer),
porcupine::data_callback(frame_buffer) WakeWordEngine::Rustpotter => rustpotter::data_callback(frame_buffer),
}, WakeWordEngine::Vosk => vosk::data_callback(frame_buffer),
WakeWordEngine::Rustpotter => {
rustpotter::data_callback(frame_buffer)
},
WakeWordEngine::Vosk => {
vosk::data_callback(frame_buffer)
}
} }
} }

40
app/src/listener/porcupine.rs
View file

@ -3,8 +3,8 @@ use std::path::Path;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use porcupine::{Porcupine, PorcupineBuilder}; use porcupine::{Porcupine, PorcupineBuilder};
use crate::DB;
use crate::config; use crate::config;
use crate::DB;
// store porcupine instance // store porcupine instance
static PORCUPINE: OnceCell<Porcupine> = OnceCell::new(); static PORCUPINE: OnceCell<Porcupine> = OnceCell::new();
@ -16,26 +16,30 @@ pub fn init() -> Result<(), ()> {
picovoice_api_key = DB.get().unwrap().api_keys.picovoice.clone(); picovoice_api_key = DB.get().unwrap().api_keys.picovoice.clone();
if picovoice_api_key.trim().is_empty() { if picovoice_api_key.trim().is_empty() {
warn!("Picovoice API key is not set."); warn!("Picovoice API key is not set.");
return Err(()) return Err(());
} }
// create porcupine instance with the given API key // create porcupine instance with the given API key
match PorcupineBuilder::new_with_keyword_paths(picovoice_api_key, &[Path::new(config::KEYWORDS_PATH).join(config::DEFAULT_KEYWORD)]) match PorcupineBuilder::new_with_keyword_paths(
.sensitivities(&[config::DEFAULT_SENSITIVITY]) // set sensitivity picovoice_api_key,
.init() { &[Path::new(config::KEYWORDS_PATH).join(config::DEFAULT_KEYWORD)],
Ok(pinstance) => { )
// success .sensitivities(&[config::DEFAULT_SENSITIVITY]) // set sensitivity
info!("Porcupine successfully initialized with the given API key."); .init()
{
Ok(pinstance) => {
// success
info!("Porcupine successfully initialized with the given API key.");
// store // store
PORCUPINE.set(pinstance); PORCUPINE.set(pinstance);
}, }
Err(msg) => { Err(msg) => {
error!("Porcupine failed to initialize, either API key is not valid or there is no internet connection."); error!("Porcupine failed to initialize, either API key is not valid or there is no internet connection.");
error!("Error details: {}", msg); error!("Error details: {}", msg);
return Err(()); return Err(());
} }
} }
Ok(()) Ok(())
@ -44,9 +48,9 @@ pub fn init() -> Result<(), ()> {
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> { pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) { if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) {
if keyword_index >= 0 { if keyword_index >= 0 {
return Some(keyword_index) return Some(keyword_index);
} }
} }
None None
} }

27
app/src/listener/rustpotter.rs
View file

@ -2,10 +2,13 @@ use std::path::Path;
use std::sync::Mutex; use std::sync::Mutex;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use rustpotter::{Rustpotter, RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig}; use rustpotter::{
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig, Rustpotter,
RustpotterConfig, ScoreMode,
};
use crate::DB;
use crate::config; use crate::config;
use crate::DB;
// store rustpotter instance // store rustpotter instance
static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new(); static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
@ -19,23 +22,23 @@ pub fn init() -> Result<(), ()> {
// success // success
// wake word files list // wake word files list
// @TODO. Make it configurable via GUI for custom user voice. // @TODO. Make it configurable via GUI for custom user voice.
let rustpotter_wake_word_files: [&str; 5] = [ let rustpotter_wake_word_files: [&str; 1] = [
"rustpotter/jarvis-default.rpw", "rustpotter/jarvis-default.rpw",
"rustpotter/jarvis-community-1.rpw", // "rustpotter/jarvis-community-1.rpw",
"rustpotter/jarvis-community-2.rpw", // "rustpotter/jarvis-community-2.rpw",
"rustpotter/jarvis-community-3.rpw", // "rustpotter/jarvis-community-3.rpw",
"rustpotter/jarvis-community-4.rpw", // "rustpotter/jarvis-community-4.rpw",
// "rustpotter/jarvis-community-5.rpw", // "rustpotter/jarvis-community-5.rpw",
]; ];
// load wake word files // load wake word files
for rpw in rustpotter_wake_word_files { for rpw in rustpotter_wake_word_files {
rinstance.add_wakeword_from_file(rpw).unwrap(); rinstance.add_wakeword_from_file(rpw, rpw).unwrap(); // @TODO: Change wakeword key to something else?
} }
// store // store
RUSTPOTTER.set(Mutex::new(rinstance)); RUSTPOTTER.set(Mutex::new(rinstance));
}, }
Err(msg) => { Err(msg) => {
error!("Rustpotter failed to initialize.\nError details: {}", msg); error!("Rustpotter failed to initialize.\nError details: {}", msg);
@ -49,17 +52,17 @@ pub fn init() -> Result<(), ()> {
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> { pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
let mut lock = RUSTPOTTER.get().unwrap().lock(); let mut lock = RUSTPOTTER.get().unwrap().lock();
let rustpotter = lock.as_mut().unwrap(); let rustpotter = lock.as_mut().unwrap();
let detection = rustpotter.process_i16(&frame_buffer); let detection = rustpotter.process_samples(frame_buffer.to_vec()); // @TODO. Temp crutch. Fix optimization issue, frame_buffer should not be copied to a new vector!
if let Some(detection) = detection { if let Some(detection) = detection {
if detection.score > config::RUSPOTTER_MIN_SCORE { if detection.score > config::RUSPOTTER_MIN_SCORE {
info!("Rustpotter detection info:\n{:?}", detection); info!("Rustpotter detection info:\n{:?}", detection);
return Some(0) return Some(0);
} else { } else {
info!("Rustpotter detection info:\n{:?}", detection) info!("Rustpotter detection info:\n{:?}", detection)
} }
} }
None None
} }

9
app/src/listener/vosk.rs
View file

@ -17,17 +17,20 @@ pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>(); let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
// compare // compare
let compare_ratio = seqdiff::ratio(&config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(), &recognized_phrase_chars); let compare_ratio = seqdiff::ratio(
&config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(),
&recognized_phrase_chars,
);
info!("og phrase: {:?}", &config::VOSK_FETCH_PHRASE); info!("og phrase: {:?}", &config::VOSK_FETCH_PHRASE);
info!("recognized phrase: {:?}", &recognized_phrase_chars); info!("recognized phrase: {:?}", &recognized_phrase_chars);
info!("compare ratio: {}", compare_ratio); info!("compare ratio: {}", compare_ratio);
if compare_ratio >= config::VOSK_MIN_RATIO { if compare_ratio >= config::VOSK_MIN_RATIO {
info!("Phrase activated."); info!("Phrase activated.");
return Some(0) return Some(0);
} }
} }
} }
None None
} }

8
app/src/log.rs
View file

@ -6,7 +6,11 @@ use crate::APP_LOG_DIR;
pub fn init_logging() -> Result<(), String> { pub fn init_logging() -> Result<(), String> {
// configure logging // configure logging
let config = LogConfigBuilder::builder() let config = LogConfigBuilder::builder()
.path(format!("{}/{}", APP_LOG_DIR.get().unwrap().display(), config::LOG_FILE_NAME)) .path(format!(
"{}/{}",
APP_LOG_DIR.get().unwrap().display(),
config::LOG_FILE_NAME
))
.size(1 * 100) .size(1 * 100)
.roll_count(10) .roll_count(10)
.time_format("%Y-%m-%d %H:%M:%S.%f") //E.g:%H:%M:%S.%f .time_format("%Y-%m-%d %H:%M:%S.%f") //E.g:%H:%M:%S.%f
@ -18,4 +22,4 @@ pub fn init_logging() -> Result<(), String> {
simple_log::new(config)?; simple_log::new(config)?;
Ok(()) Ok(())
} }

21
app/src/main.rs
View file

@ -3,7 +3,7 @@ use std::error::Error;
use std::path::PathBuf; use std::path::PathBuf;
use once_cell::sync::{Lazy, OnceCell}; use once_cell::sync::{Lazy, OnceCell};
use platform_dirs::{AppDirs}; use platform_dirs::AppDirs;
// expose the config // expose the config
mod config; mod config;
@ -35,8 +35,8 @@ mod stt;
// include commands // include commands
mod commands; mod commands;
use commands::AssistantCommand;
use crate::commands::list; use crate::commands::list;
use commands::AssistantCommand;
// include audio // include audio
mod audio; mod audio;
@ -45,8 +45,8 @@ mod audio;
mod listener; mod listener;
// some global data // some global data
static APP_DIR: Lazy<PathBuf> = Lazy::new(|| {env::current_dir().unwrap()}); static APP_DIR: Lazy<PathBuf> = Lazy::new(|| env::current_dir().unwrap());
static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| {APP_DIR.clone().join("sound")}); static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| APP_DIR.clone().join("sound"));
static APP_DIRS: OnceCell<AppDirs> = OnceCell::new(); static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new(); static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new(); static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
@ -62,7 +62,10 @@ fn main() -> Result<(), String> {
// log some base info // log some base info
info!("Starting Jarvis v{} ...", config::APP_VERSION.unwrap()); info!("Starting Jarvis v{} ...", config::APP_VERSION.unwrap());
info!("Config directory is: {}", APP_CONFIG_DIR.get().unwrap().display()); info!(
"Config directory is: {}",
APP_CONFIG_DIR.get().unwrap().display()
);
info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display()); info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display());
// initialize database (settings) // initialize database (settings)
@ -93,7 +96,11 @@ fn main() -> Result<(), String> {
// init commands // init commands
info!("Initializing commands."); info!("Initializing commands.");
let commands = commands::parse_commands().unwrap(); let commands = commands::parse_commands().unwrap();
info!("Commands initialized.\nOverall commands parsed: {}\nParsed commands: {:?}", commands.len(), commands::list(&commands)); info!(
"Commands initialized.\nOverall commands parsed: {}\nParsed commands: {:?}",
commands.len(),
commands::list(&commands)
);
COMMANDS_LIST.set(commands).unwrap(); COMMANDS_LIST.set(commands).unwrap();
// init audio // init audio
@ -111,4 +118,4 @@ fn main() -> Result<(), String> {
app::start(); app::start();
Ok(()) Ok(())
} }

71
app/src/recorder.rs
View file

@ -4,7 +4,7 @@ mod pvrecorder;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use crate::{DB, config, config::structs::RecorderType}; use crate::{config, config::structs::RecorderType, DB};
static RECORDER_TYPE: OnceCell<RecorderType> = OnceCell::new(); static RECORDER_TYPE: OnceCell<RecorderType> = OnceCell::new();
static FRAME_LENGTH: OnceCell<u32> = OnceCell::new(); static FRAME_LENGTH: OnceCell<u32> = OnceCell::new();
@ -14,23 +14,35 @@ pub fn init() -> Result<(), ()> {
// @TODO. Make it configurable? // @TODO. Make it configurable?
RECORDER_TYPE.set(config::DEFAULT_RECORDER_TYPE).unwrap(); RECORDER_TYPE.set(config::DEFAULT_RECORDER_TYPE).unwrap();
// some info
info!("Loading recorder ...");
info!("Available audio_devices are:\n{:?}", get_audio_devices());
// load given recorder // load given recorder
match RECORDER_TYPE.get().unwrap() { match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => { RecorderType::PvRecorder => {
// Init Pv Recorder // Init Pv Recorder
info!("Initializing PvRecorder recording backend."); info!("Initializing PvRecorder recording backend.");
FRAME_LENGTH.set(512u32).unwrap(); // pvrecorder requires frame buffer of 512 FRAME_LENGTH.set(512u32).unwrap(); // pvrecorder requires frame buffer of 512
match pvrecorder::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.get().unwrap().to_owned()) { let selected_microphone = get_selected_microphone_index();
match pvrecorder::init_microphone(
selected_microphone,
FRAME_LENGTH.get().unwrap().to_owned(),
) {
false => { false => {
error!("Recorder initialization failed."); error!("Recorder initialization failed.");
return Err(()) return Err(());
}, }
_ => { _ => {
info!("Recorder initialization success."); info!(
"Recorder initialization success. Listening to microphone ({}): {}",
selected_microphone,
get_audio_device_name(selected_microphone)
);
} }
} }
}, }
RecorderType::PortAudio => { RecorderType::PortAudio => {
// Init PortAudio // Init PortAudio
info!("Initializing PortAudio recording backend"); info!("Initializing PortAudio recording backend");
@ -42,7 +54,7 @@ pub fn init() -> Result<(), ()> {
// }, // },
// _ => () // _ => ()
// } // }
}, }
RecorderType::Cpal => { RecorderType::Cpal => {
// Init CPAL // Init CPAL
info!("Initializing CPAL recording backend"); info!("Initializing CPAL recording backend");
@ -64,11 +76,11 @@ pub fn read_microphone(frame_buffer: &mut [i16]) {
match RECORDER_TYPE.get().unwrap() { match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => { RecorderType::PvRecorder => {
pvrecorder::read_microphone(frame_buffer); pvrecorder::read_microphone(frame_buffer);
}, }
RecorderType::PortAudio => { RecorderType::PortAudio => {
todo!(); todo!();
// portaudio::read_microphone(frame_buffer); // portaudio::read_microphone(frame_buffer);
}, }
RecorderType::Cpal => { RecorderType::Cpal => {
// cpal::read_microphone(frame_buffer); // cpal::read_microphone(frame_buffer);
panic!("Cpal should be used via callback assignment"); panic!("Cpal should be used via callback assignment");
@ -79,12 +91,15 @@ pub fn read_microphone(frame_buffer: &mut [i16]) {
pub fn start_recording() -> Result<(), ()> { pub fn start_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() { match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => { RecorderType::PvRecorder => {
return pvrecorder::start_recording(get_selected_microphone_index(), FRAME_LENGTH.get().unwrap().to_owned()); return pvrecorder::start_recording(
}, get_selected_microphone_index(),
FRAME_LENGTH.get().unwrap().to_owned(),
);
}
RecorderType::PortAudio => { RecorderType::PortAudio => {
todo!(); todo!();
// portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)); // portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
}, }
RecorderType::Cpal => { RecorderType::Cpal => {
todo!(); todo!();
// cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)); // cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
@ -94,13 +109,11 @@ pub fn start_recording() -> Result<(), ()> {
pub fn stop_recording() -> Result<(), ()> { pub fn stop_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() { match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => { RecorderType::PvRecorder => pvrecorder::stop_recording(),
pvrecorder::stop_recording()
},
RecorderType::PortAudio => { RecorderType::PortAudio => {
todo!(); todo!();
// portaudio::stop_recording(); // portaudio::stop_recording();
}, }
RecorderType::Cpal => { RecorderType::Cpal => {
todo!(); todo!();
// cpal::stop_recording(); // cpal::stop_recording();
@ -110,4 +123,28 @@ pub fn stop_recording() -> Result<(), ()> {
pub fn get_selected_microphone_index() -> i32 { pub fn get_selected_microphone_index() -> i32 {
DB.get().unwrap().microphone DB.get().unwrap().microphone
} }
pub fn get_audio_devices() -> Vec<String> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => pvrecorder::list_audio_devices(),
RecorderType::PortAudio => {
todo!();
}
RecorderType::Cpal => {
todo!();
}
}
}
pub fn get_audio_device_name(idx: i32) -> String {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => pvrecorder::get_audio_device_name(idx),
RecorderType::PortAudio => {
todo!();
}
RecorderType::Cpal => {
todo!();
}
}
}

61
app/src/recorder/pvrecorder.rs
View file

@ -1,16 +1,16 @@
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use pv_recorder::{PvRecorder, PvRecorderBuilder};
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use pv_recorder::{Recorder, RecorderBuilder};
static RECORDER: OnceCell<Recorder> = OnceCell::new(); static RECORDER: OnceCell<PvRecorder> = OnceCell::new();
static IS_RECORDING: AtomicBool = AtomicBool::new(false); static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool { pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
match RECORDER.get().is_none() { match RECORDER.get().is_none() {
true => { true => {
let pv_recorder = RecorderBuilder::new() let pv_recorder = PvRecorderBuilder::new(frame_length as i32)
.device_index(device_index) .device_index(device_index)
.frame_length(frame_length as i32) // .frame_length(frame_length as i32)
.init(); .init();
match pv_recorder { match pv_recorder {
@ -20,7 +20,7 @@ pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
// success // success
true true
}, }
Err(msg) => { Err(msg) => {
error!("Failed to initialize pvrecorder.\nError details: {:?}", msg); error!("Failed to initialize pvrecorder.\nError details: {:?}", msg);
@ -28,8 +28,8 @@ pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
false false
} }
} }
}, }
_ => true // already initialized _ => true, // already initialized
} }
} }
@ -37,12 +37,17 @@ pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized // ensure microphone is initialized
if !RECORDER.get().is_none() { if !RECORDER.get().is_none() {
// read to frame buffer // read to frame buffer
match RECORDER.get().unwrap().read(frame_buffer) {
let frame = RECORDER.get().unwrap().read();
match frame {
Ok(f) => {
frame_buffer.copy_from_slice(f.as_slice());
}
Err(msg) => { Err(msg) => {
// @TODO: Fix? PvRecorder always wait for PCM buffer size of 512. // @TODO: Fix? PvRecorder always wait for PCM buffer size of 512.
error!("Failed to read audio frame. {:?}", msg); error!("Failed to read audio frame. {:?}", msg);
}, }
_ => ()
} }
} }
} }
@ -61,7 +66,7 @@ pub fn start_recording(device_index: i32, frame_length: u32) -> Result<(), ()> {
// success // success
Ok(()) Ok(())
}, }
Err(msg) => { Err(msg) => {
error!("Failed to start audio recording!"); error!("Failed to start audio recording!");
@ -83,16 +88,42 @@ pub fn stop_recording() -> Result<(), ()> {
IS_RECORDING.store(false, Ordering::SeqCst); IS_RECORDING.store(false, Ordering::SeqCst);
// success // success
return Ok(()) return Ok(());
}, }
Err(msg) => { Err(msg) => {
error!("Failed to stop audio recording!"); error!("Failed to stop audio recording!");
// fail // fail
return Err(()) return Err(());
} }
} }
} }
Ok(()) // if already stopped or not yet initialized Ok(()) // if already stopped or not yet initialized
} }
pub fn list_audio_devices() -> Vec<String> {
let audio_devices = PvRecorderBuilder::default().get_available_devices();
match audio_devices {
Ok(audio_devices) => audio_devices,
Err(err) => panic!("Failed to get audio devices: {}", err),
}
}
pub fn get_audio_device_name(idx: i32) -> String {
let audio_devices = list_audio_devices();
let mut first_device: String = String::new();
for (_idx, device) in audio_devices.iter().enumerate() {
if idx as usize == _idx {
return device.to_string();
}
if _idx == 0 {
first_device = device.to_string()
}
}
// return first device as default, if none were matched
first_device
}

12
app/src/stt.rs
View file

@ -1,14 +1,16 @@
mod vosk; mod vosk;
use once_cell::sync::OnceCell;
use crate::config; use crate::config;
use once_cell::sync::OnceCell;
use crate::config::structs::SpeechToTextEngine; use crate::config::structs::SpeechToTextEngine;
static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new(); static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new();
pub fn init() -> Result<(), ()> { pub fn init() -> Result<(), ()> {
if !STT_TYPE.get().is_none() {return Ok(());} // already initialized if !STT_TYPE.get().is_none() {
return Ok(());
} // already initialized
// set default stt type // set default stt type
// @TODO. Make it configurable? // @TODO. Make it configurable?
@ -29,8 +31,6 @@ pub fn init() -> Result<(), ()> {
pub fn recognize(data: &[i16], partial: bool) -> Option<String> { pub fn recognize(data: &[i16], partial: bool) -> Option<String> {
match STT_TYPE.get().unwrap() { match STT_TYPE.get().unwrap() {
SpeechToTextEngine::Vosk => { SpeechToTextEngine::Vosk => vosk::recognize(data, partial),
vosk::recognize(data, partial)
}
} }
} }

28
app/src/stt/vosk.rs
View file

@ -9,7 +9,9 @@ static MODEL: OnceCell<Model> = OnceCell::new();
static RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new(); static RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
pub fn init_vosk() { pub fn init_vosk() {
if !RECOGNIZER.get().is_none() {return;} // already initialized if !RECOGNIZER.get().is_none() {
return;
} // already initialized
let model = Model::new(VOSK_MODEL_PATH).unwrap(); let model = Model::new(VOSK_MODEL_PATH).unwrap();
let mut recognizer = Recognizer::new(&model, 16000.0).unwrap(); let mut recognizer = Recognizer::new(&model, 16000.0).unwrap();
@ -23,12 +25,26 @@ pub fn init_vosk() {
} }
pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> { pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
let state = RECOGNIZER.get().unwrap().lock().unwrap().accept_waveform(data); let state = RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.accept_waveform(data);
match state { match state {
DecodingState::Running => { DecodingState::Running => {
if include_partial { if include_partial {
Some(RECOGNIZER.get().unwrap().lock().unwrap().partial_result().partial.into()) Some(
RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.partial_result()
.partial
.into(),
)
} else { } else {
None None
} }
@ -36,7 +52,11 @@ pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
DecodingState::Finalized => { DecodingState::Finalized => {
// Result will always be multiple because we called set_max_alternatives // Result will always be multiple because we called set_max_alternatives
Some( Some(
RECOGNIZER.get().unwrap().lock().unwrap() RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.result() .result()
.multiple() .multiple()
.unwrap() .unwrap()

51
app/src/tray.rs
View file

@ -1,11 +1,11 @@
mod menu; mod menu;
use image;
use tray_icon::{ use tray_icon::{
menu::{AboutMetadata, Menu, MenuEvent, MenuItem, PredefinedMenuItem}, menu::{AboutMetadata, Menu, MenuEvent, MenuItem, PredefinedMenuItem},
TrayEvent, TrayIconBuilder, TrayIconBuilder, TrayIconEvent,
}; };
use winit::event_loop::{ControlFlow, EventLoopBuilder}; use winit::event_loop::{ControlFlow, EventLoopBuilder};
use image;
use winit::platform::windows::EventLoopBuilderExtWindows; use winit::platform::windows::EventLoopBuilderExtWindows;
use crate::config; use crate::config;
@ -15,7 +15,6 @@ pub fn init() {
// New thread will prevent tray icon to work on MacOS // New thread will prevent tray icon to work on MacOS
// @TODO: MacOS support. // @TODO: MacOS support.
std::thread::spawn(|| { std::thread::spawn(|| {
// load tray icon // load tray icon
let icon_path = format!("{}/icons/{}", env!("CARGO_MANIFEST_DIR"), config::TRAY_ICON); let icon_path = format!("{}/icons/{}", env!("CARGO_MANIFEST_DIR"), config::TRAY_ICON);
let icon = load_icon(std::path::Path::new(&icon_path)); let icon = load_icon(std::path::Path::new(&icon_path));
@ -24,8 +23,9 @@ pub fn init() {
let tray_menu = Menu::with_items(&[ let tray_menu = Menu::with_items(&[
&MenuItem::new("Перезапуск", true, None), &MenuItem::new("Перезапуск", true, None),
&MenuItem::new("Настройки", true, None), &MenuItem::new("Настройки", true, None),
&MenuItem::new("Выход", true, None) &MenuItem::new("Выход", true, None),
]); ])
.unwrap();
#[cfg(not(target_os = "linux"))] #[cfg(not(target_os = "linux"))]
let mut tray_icon = Some( let mut tray_icon = Some(
@ -34,7 +34,7 @@ pub fn init() {
.with_tooltip(config::TRAY_TOOLTIP) .with_tooltip(config::TRAY_TOOLTIP)
.with_icon(icon) .with_icon(icon)
.build() .build()
.unwrap() .unwrap(),
); );
// Since winit doesn't use gtk on Linux, and we need gtk for // Since winit doesn't use gtk on Linux, and we need gtk for
@ -55,33 +55,37 @@ pub fn init() {
} }
// run the event loop // run the event loop
let event_loop = EventLoopBuilder::new().with_any_thread(true).build(); let event_loop = EventLoopBuilder::new()
.with_any_thread(true)
.build()
.unwrap();
let menu_channel = MenuEvent::receiver(); let menu_channel = MenuEvent::receiver();
let tray_channel = TrayEvent::receiver(); let tray_channel = TrayIconEvent::receiver();
event_loop.run(move |_event, _, control_flow| { event_loop
*control_flow = ControlFlow::Poll; .run(move |_event, event_loop| {
event_loop.set_control_flow(ControlFlow::Poll);
//if let Ok(event) = tray_channel.try_recv() { //if let Ok(event) = tray_channel.try_recv() {
// println!("tray event: {event:?}"); // println!("tray event: {event:?}");
//} //}
if let Ok(event) = menu_channel.try_recv() { if let Ok(event) = menu_channel.try_recv() {
println!("menu event: {:?}", event); println!("menu event: {:?}", event);
if event.id == 1002 { if event.id == "1002" {
std::process::exit(0); std::process::exit(0);
}
} }
} })
}); .expect("Tray event loop run error");
}); });
info!("Tray initialized."); info!("Tray initialized.");
} }
fn load_icon(path: &std::path::Path) -> tray_icon::icon::Icon { fn load_icon(path: &std::path::Path) -> tray_icon::Icon {
let (icon_rgba, icon_width, icon_height) = { let (icon_rgba, icon_width, icon_height) = {
let image = image::open(path) let image = image::open(path)
.expect("Failed to open icon path") .expect("Failed to open icon path")
@ -90,6 +94,5 @@ fn load_icon(path: &std::path::Path) -> tray_icon::icon::Icon {
let rgba = image.into_raw(); let rgba = image.into_raw();
(rgba, width, height) (rgba, width, height)
}; };
tray_icon::icon::Icon::from_rgba(icon_rgba, icon_width, icon_height) tray_icon::Icon::from_rgba(icon_rgba, icon_width, icon_height).expect("Failed to open icon")
.expect("Failed to open icon") }
}

6
app/src/tray/menu.rs
View file

@ -1,7 +1,7 @@
pub enum TrayMenuItem { pub enum TrayMenuItem {
Restart, Restart,
Settings, Settings,
Exit Exit,
} }
impl TrayMenuItem { impl TrayMenuItem {
@ -9,7 +9,7 @@ impl TrayMenuItem {
match *self { match *self {
TrayMenuItem::Restart => "Перезапустить", TrayMenuItem::Restart => "Перезапустить",
TrayMenuItem::Settings => "Настройки", TrayMenuItem::Settings => "Настройки",
TrayMenuItem::Exit => "Выход" TrayMenuItem::Exit => "Выход",
} }
} }
} }