bossiara13/J.A.R.V.I.S-py
1
1
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Activity Actions

App architecture modifications.

Browse source 
Now GUI and the app itself is divided into two different binaries.
The app also provides system tray icon.
Whereas the GUI can be used to configure the app.
This commit is contained in:
Abraham 2023-06-11 19:17:50 +05:00
parent ea489d5719
commit 1323505b0a
232 changed files with 5059 additions and 8081 deletions
Download patch file Download diff file Expand all files Collapse all files

92
app/src/_main.rs Normal file
View file

@ -0,0 +1,92 @@
#[macro_use]
extern crate lazy_static; // better switch to once_cell ?
use pickledb::{PickleDb, PickleDbDumpPolicy, SerializationMethod};
use log::{info};
use log::LevelFilter;
use std::sync::Mutex;
// include assistant commands
mod assistant_commands;
use assistant_commands::AssistantCommand;
// include vosk
mod vosk;
// include events
mod events;
// include recorder
mod recorder;
// init PickleDb connection
lazy_static! {
static ref DB: Mutex<PickleDb> = Mutex::new(
PickleDb::load(
format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME),
PickleDbDumpPolicy::AutoDump,
SerializationMethod::Json
)
.unwrap_or_else(|_x: _| {
info!("Creating new db file at {} ...", format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME));
PickleDb::new(
format!("{}/{}", APP_CONFIG_DIR.lock().unwrap(), DB_FILE_NAME),
PickleDbDumpPolicy::AutoDump,
SerializationMethod::Json,
)
})
);
}
fn main() {
// init vosk
vosk::init_vosk();
// run the app
tauri::Builder::default()
.setup(|app| {
std::fs::create_dir_all(app.path_resolver().app_config_dir().unwrap())?;
APP_CONFIG_DIR.lock().unwrap().push_str(app.path_resolver().app_config_dir().unwrap().to_str().unwrap());
std::fs::create_dir_all(app.path_resolver().app_log_dir().unwrap())?;
APP_LOG_DIR.lock().unwrap().push_str(app.path_resolver().app_log_dir().unwrap().to_str().unwrap());
// log to file
let log_file_path = format!("{}/{}", APP_LOG_DIR.lock().unwrap(), config::LOG_FILE_NAME);
println!("!!!===============!!!\nLOGGING TO {}\n!!!===============!!!\n", &log_file_path);
simple_logging::log_to_file(&log_file_path, LevelFilter::max()).expect("Failed to start logger ... is directory writable?");
Ok(())
})
.invoke_handler(tauri::generate_handler![
// db commands
tauri_commands::db_read,
tauri_commands::db_write,
// recorder commands
tauri_commands::pv_get_audio_devices,
tauri_commands::pv_get_audio_device_name,
// listener commands
tauri_commands::start_listening,
tauri_commands::stop_listening,
tauri_commands::is_listening,
// sys commands
tauri_commands::get_current_ram_usage,
tauri_commands::get_peak_ram_usage,
tauri_commands::get_cpu_temp,
tauri_commands::get_cpu_usage,
// sound commands
tauri_commands::play_sound,
// fs commands
tauri_commands::show_in_folder,
// etc commands
tauri_commands::get_app_version,
tauri_commands::get_author_name,
tauri_commands::get_repository_link,
tauri_commands::get_tg_official_link,
tauri_commands::get_feedback_link,
tauri_commands::get_log_file_path
])
.run(tauri::generate_context!())
.expect("error while running tauri application");
}

122
app/src/app.rs Normal file
View file

@ -0,0 +1,122 @@
use std::time::SystemTime;
use crate::{config, audio, recorder, listener, stt, commands, COMMANDS_LIST};
use rand::seq::SliceRandom;
pub fn start() -> Result<(), ()> {
// start the loop
main_loop()
}
fn main_loop() -> Result<(), ()> {
let mut start: SystemTime;
let sounds_directory = audio::get_sound_directory().unwrap();
let frame_length: usize = 512; // default for every wake-word engine
let mut frame_buffer: Vec<i16> = vec![0; frame_length];
// play some run phrase
// @TODO. Different sounds? Or better make it via commands or upcoming events system.
audio::play_sound(&sounds_directory.join("run.wav"));
// start recording
match recorder::start_recording() {
Ok(_) => info!("Recording started."),
Err(_) => {
error!("Cannot start recording.");
return Err(()); // quit
}
}
// the loop
'wake_word: loop {
// read from microphone
recorder::read_microphone(&mut frame_buffer);
// recognize wake-word
match listener::data_callback(&frame_buffer) {
Some(keyword_index) => {
// wake-word activated, process further commands
// capture current time
start = SystemTime::now();
// play some greet phrase
// @TODO. Make it via commands or upcoming events system.
audio::play_sound(&sounds_directory.join(format!("{}.wav", config::ASSISTANT_GREET_PHRASES.choose(&mut rand::thread_rng()).unwrap())));
// wait for voice commands
'voice_recognition: loop {
// read from microphone
recorder::read_microphone(&mut frame_buffer);
// stt part (without partials)
if let Some(mut recognized_voice) = stt::recognize(&frame_buffer, false) {
// something was recognized
info!("Recognized voice: {}", recognized_voice);
// filter recognized voice
// @TODO. Better recognized voice filtration.
recognized_voice = recognized_voice.to_lowercase();
for tbr in config::ASSISTANT_PHRASES_TBR {
recognized_voice = recognized_voice.replace(tbr, "");
}
recognized_voice = recognized_voice.trim().into();
// infer command
if let Some((cmd_path, cmd_config)) = commands::fetch_command(&recognized_voice, &COMMANDS_LIST.get().unwrap()) {
// some debug info
info!("Recognized voice (filtered): {}", recognized_voice);
info!("Command found: {:?}", cmd_path);
info!("Executing!");
// execute the command
match commands::execute_command(&cmd_path, &cmd_config) {
Ok(chain) => {
// success
info!("Command executed successfully.");
if chain {
// chain commands
start = SystemTime::now();
} else {
// skip, if chaining is not required
start = start.checked_sub(core::time::Duration::from_secs(1000)).unwrap();
}
continue 'voice_recognition; // continue voice recognition
},
Err(msg) => {
// fail
error!("Error executing command: {}", msg);
}
}
}
// return to wake-word listening after command execution (no matter successful or not)
break 'voice_recognition;
}
// only recognize voice for a certain period of time
match start.elapsed() {
Ok(elapsed) if elapsed > config::CMS_WAIT_DELAY => {
// return to wake-word listening after N seconds
break 'voice_recognition;
},
_ => ()
}
}
},
None => ()
}
}
Ok(())
}
fn keyword_callback(keyword_index: i32) {
}
pub fn close(code: i32) {
info!("Closing application.");
std::process::exit(code);
}

86
app/src/audio.rs Normal file
View file

@ -0,0 +1,86 @@
mod rodio;
mod kira;
use std::cmp::Ordering;
use std::path::PathBuf;
use once_cell::sync::OnceCell;
use crate::{config, DB, SOUND_DIR};
use crate::config::structs::AudioType;
static AUDIO_TYPE: OnceCell<AudioType> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !AUDIO_TYPE.get().is_none() {return Ok(());} // already initialized
// set default audio type
// @TODO. Make it configurable?
AUDIO_TYPE.set(config::DEFAULT_AUDIO_TYPE).unwrap();
// load given audio backend
match AUDIO_TYPE.get().unwrap() {
AudioType::Rodio => {
// Init Rodio
info!("Initializing Rodio audio backend.");
match rodio::init() {
Ok(_) => {
info!("Successfully initialized Rodio audio backend.");
},
Err(msg) => {
error!("Failed to initialize Rodio audio backend.");
return Err(())
}
}
},
AudioType::Kira => {
// Init Kira
info!("Initializing Kira audio backend.");
match kira::init() {
Ok(_) => {
info!("Successfully initialized Kira audio backend.");
},
Err(msg) => {
error!("Failed to initialize Kira audio backend.");
return Err(())
}
}
}
}
Ok(())
}
pub fn play_sound(filename: &PathBuf) {
info!("Playing {}", filename.display());
match AUDIO_TYPE.get().unwrap() {
AudioType::Rodio => {
rodio::play_sound(filename, true);
},
AudioType::Kira => {
kira::play_sound(filename)
}
}
}
pub fn get_sound_directory() -> Option<PathBuf> {
let voice = DB.get().unwrap().voice.as_str();
let voice_path = SOUND_DIR.join(voice);
match voice_path.exists() && voice_path.cmp(&SOUND_DIR) != Ordering::Equal {
true => Some(voice_path),
_ => {
let default_voice_path = SOUND_DIR.join(config::DEFAULT_VOICE);
match default_voice_path.exists() {
true => Some(default_voice_path),
_ => None
}
}
}
}

55
app/src/audio/kira.rs Normal file
View file

@ -0,0 +1,55 @@
use std::path::PathBuf;
use std::sync::Mutex;
use once_cell::sync::OnceCell;
use kira::{
manager::{
AudioManager, AudioManagerSettings,
backend::DefaultBackend,
},
sound::static_sound::{StaticSoundData, StaticSoundSettings},
};
thread_local!(static MANAGER: OnceCell<Mutex<AudioManager>> = OnceCell::new());
pub fn init() -> Result<(), ()> {
MANAGER.with(|m| {
if !m.get().is_none() {return Ok(());} // already initialized
// Create an audio manager. This plays sounds and manages resources.
match AudioManager::<DefaultBackend>::new(AudioManagerSettings::default()) {
Ok(x) => {
// store
m.set(Mutex::new(x));
// success
Ok(())
},
Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg);
// failed
Err(())
}
}
})
}
// @TODO. Cache sounds in memory? With a pool of a certain size, for instance.
pub fn play_sound(filename: &PathBuf) {
// load the file
match StaticSoundData::from_file(filename, StaticSoundSettings::default()) {
Ok(sound_data) => {
// sound_data.duration() can be used in order to sleep, if (for some reason) blocking behaviour is required
// play it (non-blocking)
MANAGER.with(|m| {
let audio_manager = &mut m.get().unwrap().lock().unwrap();
audio_manager.play(sound_data.clone()).unwrap();
});
},
Err(msg) => {
warn!("Cannot find sound file: {}", filename.display());
}
}
}

76
app/src/audio/rodio.rs Normal file
View file

@ -0,0 +1,76 @@
/*
Abandoned temporary.
Problems with blocking behaviour.
Possible fixes are running rodio in a separate thread or smthng.
*/
use std::fs::File;
use std::path::PathBuf;
use std::io::BufReader;
use once_cell::sync::OnceCell;
use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink};
// static STREAM: OnceCell<OutputStream> = OnceCell::new();
static STREAM_HANDLE: OnceCell<OutputStreamHandle> = OnceCell::new();
static SINK: OnceCell<Sink> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !STREAM_HANDLE.get().is_none() {return Ok(());} // already initialized
// get output stream handle to the default physical sound device
match OutputStream::try_default() {
Ok(out) => {
// divide
let (_stream, stream_handle) = out;
// create sink
let sink;
match Sink::try_new(&stream_handle) {
Ok(s) => {
info!("Sink initialized.");
sink = s;
},
Err(msg) => {
error!("Cannot create sink.\nError details: {}", msg);
// failed
return Err(())
}
}
// store
// STREAM.set(_stream).unwrap();
STREAM_HANDLE.set(stream_handle);
SINK.set(sink);
// success
Ok(())
},
Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg);
// failed
Err(())
}
}
}
pub fn play_sound(filename: &PathBuf, sleep: bool) {
// Load a sound from a file, using a path relative to Cargo.toml
// let filepath = format!("{PUBLIC_PATH}/sound/{filename}.wav");
let file = BufReader::new(File::open(&filename).unwrap());
// Decode that sound file into a source
let source = Decoder::new(file).unwrap();
// Play the sound directly on the device
// STREAM_HANDLE.get().unwrap().play_raw(source.convert_samples());
SINK.get().unwrap().append(source);
if sleep {
// The sound plays in a separate thread. This call will block the current thread until the sink
// has finished playing all its queued sounds.
SINK.get().unwrap().sleep_until_end();
}
}

225
app/src/commands.rs Normal file
View file

@ -0,0 +1,225 @@
use rand::seq::SliceRandom;
use seqdiff::ratio;
use serde_yaml;
use std::path::Path;
use std::{fs, fs::File};
use core::time::Duration;
use std::path::PathBuf;
use std::process::{Command, Child};
// use tauri::Manager;
mod structs;
pub use structs::*;
use crate::{config, audio};
// @TODO. Allow commands both in yaml and json format.
pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
// collect commands
let mut commands: Vec<AssistantCommand> = vec![];
// read commands directories first
if let Ok(cpaths) = fs::read_dir(config::COMMANDS_PATH) {
for cpath in cpaths {
// validate this command, check if required files exists
let _cpath = cpath.unwrap().path();
let cc_file = Path::new(&_cpath).join("command.yaml");
if cc_file.exists() {
// try parse config files
let cc_reader = std::fs::File::open(&cc_file).unwrap();
let cc_yaml: CommandsList;
// try parse command.yaml
match serde_yaml::from_reader::<File, CommandsList>(cc_reader) {
Ok(parse_result) => {
cc_yaml = parse_result;
},
Err(msg) => {
warn!("Can't parse {}, skipping ...\nCommand parse error is: {:?}", &cc_file.display(), msg);
continue;
}
}
// everything seems to be Ok
commands.push(AssistantCommand {
path: _cpath,
commands: cc_yaml,
});
}
}
if commands.len() > 0 {
Ok(commands)
} else {
error!("No commands were found");
Err("No commands were found".into())
}
} else {
error!("Error reading commands directory");
return Err("Error reading commands directory".into());
}
}
// @TODO. NLU or smthng else is required, in order to infer commands with highest accuracy possible.
pub fn fetch_command<'a>(
phrase: &str,
commands: &'a Vec<AssistantCommand>,
) -> Option<(&'a PathBuf, &'a Config)> {
// result scmd
let mut result_scmd: Option<(&PathBuf, &Config)> = None;
let mut current_max_ratio = config::CMD_RATIO_THRESHOLD;
// convert fetch phrase to sequence
let fetch_phrase_chars = phrase.chars().collect::<Vec<_>>();
// list all the commands
for cmd in commands {
// list all subcommands
for scmd in &cmd.commands.list {
// list all phrases in command
for cmd_phrase in &scmd.phrases {
// convert cmd phrase to sequence
let cmd_phrase_chars = cmd_phrase.chars().collect::<Vec<_>>();
// compare fetch phrase with cmd phrase
let ratio = ratio(&fetch_phrase_chars, &cmd_phrase_chars);
// return, if it fits the given threshold
if ratio >= current_max_ratio {
result_scmd = Some((&cmd.path, &scmd));
current_max_ratio = ratio;
// println!("Ratio is: {}", ratio);
// return Some((&cmd.path, &scmd))
}
}
}
}
if let Some((cmd_path, scmd)) = result_scmd {
println!("Ratio is: {}", current_max_ratio);
info!("CMD is: {cmd_path:?}, SCMD is: {scmd:?}, Ratio is: {}", current_max_ratio);
Some((&cmd_path, &scmd))
} else {
None
}
}
// @TODO. Rewrite executors by executor type struct. (with match arms)
pub fn execute_exe(exe: &str, args: &Vec<String>) -> std::io::Result<Child> {
Command::new(exe).args(args).spawn()
}
pub fn execute_cli(cmd: &str, args: &Vec<String>) -> std::io::Result<Child> {
println!("Spawning cmd as: cmd /C {} {:?}", cmd, args);
if cfg!(target_os = "windows") {
Command::new("cmd")
.arg("/C")
.arg(cmd)
.args(args)
.spawn()
} else {
Command::new("sh")
.arg("-c")
.arg(cmd)
.args(args)
.spawn()
}
}
pub fn execute_command(
cmd_path: &PathBuf,
cmd_config: &Config,
// app_handle: &tauri::AppHandle,
) -> Result<bool, String> {
let sounds_directory = audio::get_sound_directory().unwrap();
match cmd_config.command.action.as_str() {
"voice" => {
// VOICE command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(true)
}
"ahk" => {
// AutoHotkey command type
let exe_path_absolute = Path::new(&cmd_config.command.exe_path);
let exe_path_local = Path::new(&cmd_path).join(&cmd_config.command.exe_path);
if let Ok(_) = execute_exe(
if exe_path_absolute.exists() {
exe_path_absolute.to_str().unwrap()
} else {
exe_path_local.to_str().unwrap()
},
&cmd_config.command.exe_args,
) {
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(true)
} else {
error!("AHK process spawn error (does exe path is valid?)");
Err("AHK process spawn error (does exe path is valid?)".into())
}
}
"cli" => {
// CLI command type
let cli_cmd = &cmd_config.command.cli_cmd;
match execute_cli(
cli_cmd,
&cmd_config.command.cli_args,
) {
Ok(_) => {
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(true)
},
Err(msg) => {
error!("CLI command error ({})", msg);
Err(format!("Shell command error ({})", msg).into())
}
}
}
"terminate" => {
// TERMINATE command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
std::thread::sleep(Duration::from_secs(2));
std::process::exit(0);
}
"stop_chaining" => {
// STOP_CHAINING command type
let random_cmd_sound = format!("{}.wav", cmd_config.voice.sounds.choose(&mut rand::thread_rng()).unwrap());
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(false)
}
_ => {
error!("Command type unknown");
Err("Command type unknown".into())
},
}
}
pub fn list(from: &[AssistantCommand]) -> Vec<String> {
let mut out: Vec<String> = vec![];
for x in from.iter() {
out.push(String::from(x.path.to_str().unwrap()));
// out.append()
}
out
}

46
app/src/commands/structs.rs Normal file
View file

@ -0,0 +1,46 @@
use serde::Deserialize;
use std::path::PathBuf;
#[derive(Debug)]
pub struct AssistantCommand {
pub path: PathBuf,
pub commands: CommandsList,
}
#[derive(Deserialize, Debug)]
pub struct CommandsList {
pub list: Vec<Config>,
}
#[derive(Deserialize, Debug)]
pub struct Config {
pub command: ConfigCommandSection,
pub voice: ConfigVoiceSection,
pub phrases: Vec<String>,
}
#[derive(Deserialize, Debug)]
pub struct ConfigCommandSection {
pub action: String,
#[serde(default)]
pub exe_path: String,
#[serde(default)]
pub exe_args: Vec<String>,
#[serde(default)]
pub cli_cmd: String,
#[serde(default)]
pub cli_args: Vec<String>
}
#[derive(Deserialize, Debug)]
pub struct ConfigVoiceSection {
#[serde(default)]
pub sounds: Vec<String>,
}

144
app/src/config.rs Normal file
View file

@ -0,0 +1,144 @@
pub mod structs;
use structs::WakeWordEngine;
use structs::SpeechToTextEngine;
use structs::RecorderType;
use structs::AudioType;
use std::fs;
use std::env;
use std::path::PathBuf;
use once_cell::sync::Lazy;
use platform_dirs::{AppDirs};
use rustpotter::{RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig};
use crate::{config, APP_DIRS, APP_CONFIG_DIR, APP_LOG_DIR};
#[allow(dead_code)]
pub fn init_dirs() -> Result<(), String> {
// infer app dirs
if APP_DIRS.get().is_some() {
return Ok(());
}
// cache_dir, config_dir, data_dir, state_dir
APP_DIRS.set(AppDirs::new(Some(config::BUNDLE_IDENTIFIER), false).unwrap()).unwrap();
// setup directories
let mut config_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
let mut log_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
// create dirs, if required
if !config_dir.exists() {
if fs::create_dir_all(&config_dir).is_err() {
config_dir = env::current_dir().expect("Cannot infer the config directory");
fs::create_dir_all(&config_dir).expect("Cannot create config directory, access denied?");
}
}
if !log_dir.exists() {
if fs::create_dir_all(&log_dir).is_err() {
log_dir = env::current_dir().expect("Cannot infer the log directory");
fs::create_dir_all(&log_dir).expect("Cannot create log directory, access denied?");
}
}
// store inferred paths
APP_CONFIG_DIR.set(config_dir).unwrap();
APP_LOG_DIR.set(log_dir).unwrap();
Ok(())
}
/*
Defaults.
*/
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter;
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
pub const DEFAULT_VOICE: &str = "jarvis-og";
pub const BUNDLE_IDENTIFIER: &str = "com.priler.jarvis";
pub const DB_FILE_NAME: &str = "app.db";
pub const LOG_FILE_NAME: &str = "log.txt";
pub const APP_VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION");
pub const AUTHOR_NAME: Option<&str> = option_env!("CARGO_PKG_AUTHORS");
pub const REPOSITORY_LINK: Option<&str> = option_env!("CARGO_PKG_REPOSITORY");
pub const TG_OFFICIAL_LINK: Option<&str> = Some("https://t.me/howdyho_official");
pub const FEEDBACK_LINK: Option<&str> = Some("https://t.me/jarvis_feedback_bot");
/*
Tray.
*/
pub const TRAY_ICON: &str = "32x32.png";
pub const TRAY_TOOLTIP: &str = "Jarvis Voice Assistant";
// RUSPOTTER
pub const RUSPOTTER_MIN_SCORE: f32 = 0.62;
pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
RustpotterConfig {
fmt: WavFmt::default(),
detector: DetectorConfig {
avg_threshold: 0.,
threshold: 0.5,
min_scores: 15,
score_mode: ScoreMode::Average,
comparator_band_size: 5,
comparator_ref: 0.22
},
filters: FiltersConfig {
gain_normalizer: GainNormalizationConfig {
enabled: true,
gain_ref: None,
min_gain: 0.7,
max_gain: 1.0,
},
band_pass: BandPassConfig {
enabled: true,
low_cutoff: 80.,
high_cutoff: 400.,
}
}
}
});
// PICOVOICE
pub const COMMANDS_PATH: &str = "commands/";
pub const KEYWORDS_PATH: &str = "picovoice/keywords/";
pub const DEFAULT_KEYWORD: &str = "jarvis_windows.ppn";
pub const DEFAULT_SENSITIVITY: f32 = 1.0;
// VOSK
// pub const VOSK_MODEL_PATH: &str = const_concat!(PUBLIC_PATH, "/vosk/model_small");
pub const VOSK_FETCH_PHRASE: &str = "джарвис";
pub const VOSK_MODEL_PATH: &str = "vosk/model_small";
pub const VOSK_MIN_RATIO: f64 = 70.0;
// ETC
pub const CMD_RATIO_THRESHOLD: f64 = 65f64;
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);
pub const ASSISTANT_GREET_PHRASES: [&str; 3] = ["greet1", "greet2", "greet3"];
pub const ASSISTANT_PHRASES_TBR: [&str; 17] = [
"джарвис",
"сэр",
"слушаю сэр",
"всегда к услугам",
"произнеси",
"ответь",
"покажи",
"скажи",
"давай",
"да сэр",
"к вашим услугам сэр",
"всегда к вашим услугам сэр",
"запрос выполнен сэр",
"выполнен сэр",
"есть",
"загружаю сэр",
"очень тонкое замечание сэр",
];

30
app/src/config/structs.rs Normal file
View file

@ -0,0 +1,30 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
pub enum WakeWordEngine {
Rustpotter,
Vosk,
Porcupine
}
#[derive(Serialize, Deserialize, Debug)]
pub enum SpeechToTextEngine {
Vosk
}
#[derive(PartialEq, Debug)]
pub enum RecorderType {
Cpal,
PvRecorder,
PortAudio
}
#[derive(PartialEq, Debug)]
pub enum AudioType {
Rodio,
Kira
}
// pub enum TextToSpeechEngine {}
// pub enum IntentRecognitionEngine {}

51
app/src/db.rs Normal file
View file

@ -0,0 +1,51 @@
pub mod structs;
use crate::{config, APP_CONFIG_DIR};
use std::path::PathBuf;
use std::fs::File;
use std::io::{BufReader, Read};
use log::info;
use serde_json;
fn get_db_file_path() -> PathBuf {
PathBuf::from(format!("{}/{}", APP_CONFIG_DIR.get().unwrap().display(), config::DB_FILE_NAME))
}
pub fn init_settings() -> structs::Settings {
let mut db = None;
let db_file_path = get_db_file_path();
info!("Loading settings db file located at: {}", db_file_path.display());
if db_file_path.exists() {
// try load existing settings
if let Ok(mut db_file) = File::open(db_file_path) {
let reader = BufReader::new(db_file);
if let Ok(parsed_json) = serde_json::from_reader(reader) {
info!("Settings loaded.");
db = Some(parsed_json);
}
}
}
if db.is_none() {
// create default settings db file
warn!("No settings file found or there was an error parsing it. Creating default struct.");
db = Some(structs::Settings::default());
}
db.unwrap()
}
pub fn save_settings(settings: &structs::Settings) -> Result<(), std::io::Error> {
let db_file_path = get_db_file_path();
std::fs::write(
db_file_path,
serde_json::to_string_pretty(&settings).unwrap()
)?;
info!("Settings saved.");
Ok(())
}

39
app/src/db/structs.rs Normal file
View file

@ -0,0 +1,39 @@
use serde::{Deserialize, Serialize};
use crate::config;
use crate::config::structs::WakeWordEngine;
use crate::config::structs::SpeechToTextEngine;
#[derive(Serialize, Deserialize, Debug)]
pub struct Settings {
pub microphone: i32,
pub voice: String,
pub wake_word_engine: WakeWordEngine,
pub speech_to_text_engine: SpeechToTextEngine,
pub api_keys: ApiKeys
}
impl Default for Settings {
fn default() -> Settings {
Settings {
microphone: -1,
voice: String::from(""),
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
api_keys: ApiKeys {
picovoice: String::from(""),
openai: String::from("")
}
}
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct ApiKeys {
pub picovoice: String,
pub openai: String
}

63
app/src/listener.rs Normal file
View file

@ -0,0 +1,63 @@
mod porcupine;
mod rustpotter;
mod vosk;
use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::{config, stt};
use crate::config::structs::WakeWordEngine;
use crate::DB;
// store wake-word engine being used
static WAKE_WORD_ENGINE: OnceCell<WakeWordEngine> = OnceCell::new();
// track listening state
static LISTENING: AtomicBool = AtomicBool::new(false);
pub fn init() -> Result<(), ()> {
if !WAKE_WORD_ENGINE.get().is_none() {return Ok(());} // already initialized
// store current engine
WAKE_WORD_ENGINE.set(DB.get().unwrap().wake_word_engine).unwrap();
// load given wake-word engine
match WAKE_WORD_ENGINE.get().unwrap() {
WakeWordEngine::Porcupine => {
// Init Porcupine wake-word engine
info!("Initializing Porcupine wake-word engine.");
return porcupine::init();
},
WakeWordEngine::Rustpotter => {
// Init Rustpotter wake-word engine
info!("Initializing Rustpotter wake-word engine.");
return rustpotter::init();
},
WakeWordEngine::Vosk => {
// Init Vosk as wake-word engine (very slow, though)
info!("Initializing Vosk as wake-word engine.");
warn!("Using Vosk as wake-word engine is highly not recommended, because it's very slow for this task.");
return vosk::init();
},
}
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
match WAKE_WORD_ENGINE.get().unwrap() {
WakeWordEngine::Porcupine => {
porcupine::data_callback(frame_buffer)
},
WakeWordEngine::Rustpotter => {
rustpotter::data_callback(frame_buffer)
},
WakeWordEngine::Vosk => {
vosk::data_callback(frame_buffer)
}
}
}

52
app/src/listener/porcupine.rs Normal file
View file

@ -0,0 +1,52 @@
use std::path::Path;
use once_cell::sync::OnceCell;
use porcupine::{Porcupine, PorcupineBuilder};
use crate::DB;
use crate::config;
// store porcupine instance
static PORCUPINE: OnceCell<Porcupine> = OnceCell::new();
pub fn init() -> Result<(), ()> {
let picovoice_api_key: String;
// retrieve picovoice api key
picovoice_api_key = DB.get().unwrap().api_keys.picovoice.clone();
if picovoice_api_key.trim().is_empty() {
warn!("Picovoice API key is not set.");
return Err(())
}
// create porcupine instance with the given API key
match PorcupineBuilder::new_with_keyword_paths(picovoice_api_key, &[Path::new(config::KEYWORDS_PATH).join(config::DEFAULT_KEYWORD)])
.sensitivities(&[config::DEFAULT_SENSITIVITY]) // set sensitivity
.init() {
Ok(pinstance) => {
// success
info!("Porcupine successfully initialized with the given API key.");
// store
PORCUPINE.set(pinstance);
},
Err(msg) => {
error!("Porcupine failed to initialize, either API key is not valid or there is no internet connection.");
error!("Error details: {}", msg);
return Err(());
}
}
Ok(())
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) {
if keyword_index >= 0 {
return Some(keyword_index)
}
}
None
}

65
app/src/listener/rustpotter.rs Normal file
View file

@ -0,0 +1,65 @@
use std::path::Path;
use std::sync::Mutex;
use once_cell::sync::OnceCell;
use rustpotter::{Rustpotter, RustpotterConfig, WavFmt, DetectorConfig, FiltersConfig, ScoreMode, GainNormalizationConfig, BandPassConfig};
use crate::DB;
use crate::config;
// store rustpotter instance
static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
pub fn init() -> Result<(), ()> {
let rustpotter_config = config::RUSTPOTTER_DEFAULT_CONFIG;
// create rustpotter instance
match Rustpotter::new(&rustpotter_config) {
Ok(mut rinstance) => {
// success
// wake word files list
// @TODO. Make it configurable via GUI for custom user voice.
let rustpotter_wake_word_files: [&str; 5] = [
"rustpotter/jarvis-default.rpw",
"rustpotter/jarvis-community-1.rpw",
"rustpotter/jarvis-community-2.rpw",
"rustpotter/jarvis-community-3.rpw",
"rustpotter/jarvis-community-4.rpw",
// "rustpotter/jarvis-community-5.rpw",
];
// load wake word files
for rpw in rustpotter_wake_word_files {
rinstance.add_wakeword_from_file(rpw).unwrap();
}
// store
RUSTPOTTER.set(Mutex::new(rinstance));
},
Err(msg) => {
error!("Rustpotter failed to initialize.\nError details: {}", msg);
return Err(());
}
}
Ok(())
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
let mut lock = RUSTPOTTER.get().unwrap().lock();
let rustpotter = lock.as_mut().unwrap();
let detection = rustpotter.process_i16(&frame_buffer);
if let Some(detection) = detection {
if detection.score > config::RUSPOTTER_MIN_SCORE {
info!("Rustpotter detection info:\n{:?}", detection);
return Some(0)
} else {
info!("Rustpotter detection info:\n{:?}", detection)
}
}
None
}

33
app/src/listener/vosk.rs Normal file
View file

@ -0,0 +1,33 @@
use crate::{config, stt};
pub fn init() -> Result<(), ()> {
Ok(()) // nothing to init for Vosk
}
// @TODO. Make it better somehow (more accurate or with higher sensitivity).
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
// recognize & convert to sequence
let recognized_phrase = stt::recognize(&frame_buffer, true).unwrap_or("".into());
if !recognized_phrase.trim().is_empty() {
info!("Vosk wake-word debug info:");
info!("rec: {}", recognized_phrase);
let recognized_phrases = recognized_phrase.split_whitespace();
for phrase in recognized_phrases {
let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
// compare
let compare_ratio = seqdiff::ratio(&config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(), &recognized_phrase_chars);
info!("og phrase: {:?}", &config::VOSK_FETCH_PHRASE);
info!("recognized phrase: {:?}", &recognized_phrase_chars);
info!("compare ratio: {}", compare_ratio);
if compare_ratio >= config::VOSK_MIN_RATIO {
info!("Phrase activated.");
return Some(0)
}
}
}
None
}

21
app/src/log.rs Normal file
View file

@ -0,0 +1,21 @@
use simple_log::LogConfigBuilder;
use crate::config;
use crate::APP_LOG_DIR;
pub fn init_logging() -> Result<(), String> {
// configure logging
let config = LogConfigBuilder::builder()
.path(format!("{}/{}", APP_LOG_DIR.get().unwrap().display(), config::LOG_FILE_NAME))
.size(1 * 100)
.roll_count(10)
.time_format("%Y-%m-%d %H:%M:%S.%f") //E.g:%H:%M:%S.%f
.level("debug")
.output_file()
.output_console()
.build();
simple_log::new(config)?;
Ok(())
}

114
app/src/main.rs Normal file
View file

@ -0,0 +1,114 @@
use std::env;
use std::error::Error;
use std::path::PathBuf;
use once_cell::sync::{Lazy, OnceCell};
use platform_dirs::{AppDirs};
// expose the config
mod config;
// include log
#[macro_use]
extern crate simple_log;
mod log;
// include app
mod app;
// include db
mod db;
// include tray
// @TODO. macOS currently not supported for tray functionality.
#[cfg(not(target_os = "macos"))]
mod tray;
// include recorder
mod recorder;
// include speech-to-text
mod stt;
// include text-to-speech
// empty
// include commands
mod commands;
use commands::AssistantCommand;
use crate::commands::list;
// include audio
mod audio;
// include listener
mod listener;
// some global data
static APP_DIR: Lazy<PathBuf> = Lazy::new(|| {env::current_dir().unwrap()});
static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| {APP_DIR.clone().join("sound")});
static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
static DB: OnceCell<db::structs::Settings> = OnceCell::new();
static COMMANDS_LIST: OnceCell<Vec<AssistantCommand>> = OnceCell::new();
fn main() -> Result<(), String> {
// initialize directories
config::init_dirs()?;
// initialize logging
log::init_logging()?;
// log some base info
info!("Starting Jarvis v{} ...", config::APP_VERSION.unwrap());
info!("Config directory is: {}", APP_CONFIG_DIR.get().unwrap().display());
info!("Log directory is: {}", APP_LOG_DIR.get().unwrap().display());
// initialize database (settings)
DB.set(db::init_settings());
// initialize tray
// @TODO. macOS currently not supported for tray functionality,
// due to the separate thread in which tray processing works,
// but macOS requires it to be processed in the main thread only
// The solution may be to include wake-word detection etc. in the winit event loop. (only for MacOS, though?)
#[cfg(not(target_os = "macos"))]
tray::init();
// init recorder
if recorder::init().is_err() {
app::close(1); // cannot continue without recorder
}
// init stt engine
if stt::init().is_err() {
// @TODO. Allow continuing even without STT, if commands is using keywords or smthng?
app::close(1); // cannot continue without stt
}
// init tts engine
// none for now (Silero-rs coming)
// init commands
info!("Initializing commands.");
let commands = commands::parse_commands().unwrap();
info!("Commands initialized.\nOverall commands parsed: {}\nParsed commands: {:?}", commands.len(), commands::list(&commands));
COMMANDS_LIST.set(commands).unwrap();
// init audio
if audio::init().is_err() {
// @TODO. Allow continuing even without audio?
app::close(1); // cannot continue without audio
}
// init wake-word engine
if listener::init().is_err() {
app::close(1); // cannot continue without wake-word engine
}
// start the app
app::start();
Ok(())
}

113
app/src/recorder.rs Normal file
View file

@ -0,0 +1,113 @@
mod pvrecorder;
// mod cpal;
// mod portaudio;
use once_cell::sync::OnceCell;
use crate::{DB, config, config::structs::RecorderType};
static RECORDER_TYPE: OnceCell<RecorderType> = OnceCell::new();
static FRAME_LENGTH: OnceCell<u32> = OnceCell::new();
pub fn init() -> Result<(), ()> {
// set default recorder type
// @TODO. Make it configurable?
RECORDER_TYPE.set(config::DEFAULT_RECORDER_TYPE).unwrap();
// load given recorder
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
// Init Pv Recorder
info!("Initializing PvRecorder recording backend.");
FRAME_LENGTH.set(512u32).unwrap(); // pvrecorder requires frame buffer of 512
match pvrecorder::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.get().unwrap().to_owned()) {
false => {
error!("Recorder initialization failed.");
return Err(())
},
_ => {
info!("Recorder initialization success.");
}
}
},
RecorderType::PortAudio => {
// Init PortAudio
info!("Initializing PortAudio recording backend");
todo!();
// match portaudio::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
// false => {
// // Switch to PortAudio recorder
// error!("PortAudio audio backend failed.");
// },
// _ => ()
// }
},
RecorderType::Cpal => {
// Init CPAL
info!("Initializing CPAL recording backend");
todo!();
// match cpal::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
// false => {
// // Switch to CPAL recorder
// error!("CPAL audio backend failed.");
// },
// _ => ()
// }
}
}
Ok(())
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
pvrecorder::read_microphone(frame_buffer);
},
RecorderType::PortAudio => {
todo!();
// portaudio::read_microphone(frame_buffer);
},
RecorderType::Cpal => {
// cpal::read_microphone(frame_buffer);
panic!("Cpal should be used via callback assignment");
}
}
}
pub fn start_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
return pvrecorder::start_recording(get_selected_microphone_index(), FRAME_LENGTH.get().unwrap().to_owned());
},
RecorderType::PortAudio => {
todo!();
// portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
},
RecorderType::Cpal => {
todo!();
// cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
}
}
}
pub fn stop_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
pvrecorder::stop_recording()
},
RecorderType::PortAudio => {
todo!();
// portaudio::stop_recording();
},
RecorderType::Cpal => {
todo!();
// cpal::stop_recording();
}
}
}
pub fn get_selected_microphone_index() -> i32 {
DB.get().unwrap().microphone
}

191
app/src/recorder/cpal.rs Normal file
View file

@ -0,0 +1,191 @@
/*
Abandoned temporary.
Problems with frame size.
*/
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{BufferSize, StreamConfig, SampleRate, Host, Device, Stream, SampleFormat};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::Arc;
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
use crate::tauri_commands::cpal_data_callback;
static HOST: OnceCell<Host> = OnceCell::new();
thread_local!(static RECORDER: OnceCell<ArcSwap<Stream>> = OnceCell::new());
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
// init host & frame buffer for the callback
if HOST.get().is_none() {
HOST.set(cpal::default_host());
// FRAME_BUFFER.set(Mutex::new(vec![0; FRAME_LENGTH.load(Ordering::SeqCst) as usize]));
}
// init microphone
RECORDER.with(|recorder| {
match recorder.get().is_none() {
true => {
if let Some(device) = get_device(device_index as usize) {
// store
recorder.set(ArcSwap::from_pointee(create_stream(device, frame_length)));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
} else {
false
}
},
false => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
stop_recording();
}
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
if let Some(device) = get_device(device_index as usize) {
// store
recorder.get().unwrap().store(Arc::new(create_stream(device, frame_length)));
// success
return true
} else {
return false
}
}
// success
true
}
}
})
}
fn create_stream(device: Device, frame_length: u32) -> Stream {
// get default input stream config
// let default_config = device.default_input_config().unwrap();
// create config for the stream
// let config: StreamConfig = StreamConfig {
// channels: default_config.channels(),
// sample_rate: SampleRate(16000),
// buffer_size: BufferSize::Fixed(frame_length)
// };
let config = device
.default_input_config()
.expect("Failed to load default input config");
let channels = config.channels();
let err_fn = move |err| {
eprintln!("an error occurred on stream: {}", err);
};
match config.sample_format() {
SampleFormat::F32 => device.build_input_stream(
&config.into(),
move |data: &[f32], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
SampleFormat::U16 => device.build_input_stream(
&config.into(),
move |data: &[u16], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
SampleFormat::I16 => device.build_input_stream(
&config.into(),
move |data: &[i16], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
_ => todo!()
}.unwrap()
}
pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
let mut result = Vec::with_capacity(input_data.len() / 2);
result.extend(
input_data
.chunks_exact(2)
.map(|chunk| chunk[0] / 2 + chunk[1] / 2),
);
result
}
fn get_device(device_index: usize) -> Option<Device> {
if let Some(device) = HOST.get().unwrap().input_devices().expect("Get devices error ...").nth(device_index) {
Some(device)
} else {
if let Some(default) = HOST.get().unwrap().default_input_device() {
Some(default)
} else {
error!("No default input device ...");
None
}
}
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.with(|recorder| {
match recorder.get().unwrap().load().play() {
Err(msg) => {
error!("[CPAL] Audio stream PLAY error ... {:?}", msg);
},
_ => ()
};
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
});
}
pub fn stop_recording() {
// ensure microphone is initialized
RECORDER.with(|recorder| {
if !recorder.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// pause instead of stop
match recorder.get().unwrap().load().pause() {
Err(msg) => {
error!("[CPAL] Audio stream PAUSE error ... {:?}", msg);
},
_ => ()
};
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
});
}

205
app/src/recorder/portaudio.rs Normal file
View file

@ -0,0 +1,205 @@
/*
Abandoned temporary.
*/
use portaudio as pa;
use pa::{DeviceIndex, Stream};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::{Arc, Mutex};
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
thread_local!(static RECORDER: OnceCell<ArcSwap<Mutex<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>>>> = OnceCell::new());
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
const CHANNELS: i32 = 1;
const SAMPLE_RATE: f64 = 16_000.0;
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
RECORDER.with(|r| {
match r.get().is_none() {
true => {
match create_stream(device_index, frame_length) {
Ok(stream) => {
// store
r.set(ArcSwap::from_pointee(Mutex::new(stream)));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
},
Err(msg) => {
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
// fail
false
}
}
},
_ => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
// RECORDER.get().unwrap().load().stop().expect("Failed to start audio recording!");
stop_recording();
}
// store
match create_stream(device_index, frame_length) {
Ok(stream) => {
// store new stream
r.get().unwrap().store(Arc::new(Mutex::new(stream)));
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
return true
},
Err(msg) => {
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
// fail
return false
}
}
}
// success
true
}
}
})
}
fn create_stream(device_index: i32, frame_length: u32) -> Result<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>, pa::Error> {
let pa_recorder: Result<pa::PortAudio, pa::Error> = pa::PortAudio::new();
match pa_recorder {
Ok(pa) => {
let input_settings = match get_input_settings(DeviceIndex(device_index as u32), &pa, SAMPLE_RATE, frame_length, CHANNELS) {
Ok(settings) => settings,
Err(error) => panic!("{}", String::from(error))
};
// Construct a stream with input and output sample types of i16
match pa.open_blocking_stream(input_settings) {
Ok(strm) => Ok(strm),
Err(error) => panic!("{}", error.to_string()),
}
},
Err(msg) => Err(msg)
}
}
fn get_input_latency(audio_port: &pa::PortAudio, input_index: pa::DeviceIndex) -> Result<f64, String>
{
let input_device_information = audio_port.device_info(input_index).or_else(|error| Err(String::from(format!("{}", error))));
Ok(input_device_information.unwrap().default_low_input_latency)
}
fn get_input_stream_parameters(input_index: pa::DeviceIndex, latency: f64, channels: i32) -> Result<pa::StreamParameters<i16>, String>
{
const INTERLEAVED: bool = true;
Ok(pa::StreamParameters::<i16>::new(input_index, channels, INTERLEAVED, latency))
}
fn get_input_settings(input_index: pa::DeviceIndex, audio_port: &pa::PortAudio, sample_rate: f64, frames: u32, channels: i32) -> Result<pa::InputStreamSettings<i16>, String>
{
Ok(
pa::InputStreamSettings::new(
(get_input_stream_parameters(
input_index,
(get_input_latency(
&audio_port,
input_index,
))?,
channels
))?,
sample_rate,
frames,
)
)
}
// We'll use this function to wait for read/write availability.
fn wait_for_stream<F>(f: F, name: &str) -> u32
where
F: Fn() -> Result<pa::StreamAvailable, pa::error::Error>,
{
loop {
match f() {
Ok(available) => match available {
pa::StreamAvailable::Frames(frames) => return frames as u32,
pa::StreamAvailable::InputOverflowed => println!("Input stream has overflowed"),
pa::StreamAvailable::OutputUnderflowed => {
println!("Output stream has underflowed")
}
},
Err(err) => panic!(
"An error occurred while waiting for the {} stream: {}",
name, err
),
}
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
RECORDER.with(|r| {
if !r.get().is_none() {
let cell = r.get().unwrap().load();
let mut lock = cell.lock();
let stream = lock.as_mut().unwrap();
// read to frame buffer
let in_frames = wait_for_stream(|| stream.read_available(), "Read");
if in_frames > 0 {
// let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
// println!("Read {:?} frames from the input stream.", in_frames);
let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
println!("Read: {} (required {})", input_samples.len(), frame_buffer.len());
frame_buffer.copy_from_slice(input_samples.chunks(frame_buffer.len()).last().unwrap());
}
// r.get().unwrap().load().read(frame_buffer).expect("Failed to read audio frame");
}
});
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.with(|r| {
r.get().unwrap().load().lock().unwrap().start().expect("Failed to start audio recording!");
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
});
}
pub fn stop_recording() {
RECORDER.with(|r| {
if !r.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// stop recording
let pa = r.get().unwrap().load();
r.get().unwrap().load().lock().unwrap().stop().expect("Failed to stop audio recording!");
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
});
}

98
app/src/recorder/pvrecorder.rs Normal file
View file

@ -0,0 +1,98 @@
use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicBool, Ordering};
use pv_recorder::{Recorder, RecorderBuilder};
static RECORDER: OnceCell<Recorder> = OnceCell::new();
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
match RECORDER.get().is_none() {
true => {
let pv_recorder = RecorderBuilder::new()
.device_index(device_index)
.frame_length(frame_length as i32)
.init();
match pv_recorder {
Ok(pv) => {
// store
RECORDER.set(pv);
// success
true
},
Err(msg) => {
error!("Failed to initialize pvrecorder.\nError details: {:?}", msg);
// fail
false
}
}
},
_ => true // already initialized
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
if !RECORDER.get().is_none() {
// read to frame buffer
match RECORDER.get().unwrap().read(frame_buffer) {
Err(msg) => {
// @TODO: Fix? PvRecorder always wait for PCM buffer size of 512.
error!("Failed to read audio frame. {:?}", msg);
},
_ => ()
}
}
}
pub fn start_recording(device_index: i32, frame_length: u32) -> Result<(), ()> {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
match RECORDER.get().unwrap().start() {
Ok(_) => {
info!("START recording from microphone ...");
// change recording state
IS_RECORDING.store(true, Ordering::SeqCst);
// success
Ok(())
},
Err(msg) => {
error!("Failed to start audio recording!");
// fail
Err(())
}
}
}
pub fn stop_recording() -> Result<(), ()> {
// ensure microphone is initialized & recording is in process
if !RECORDER.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// stop recording
match RECORDER.get().unwrap().stop() {
Ok(_) => {
info!("STOP recording from microphone ...");
// change recording state
IS_RECORDING.store(false, Ordering::SeqCst);
// success
return Ok(())
},
Err(msg) => {
error!("Failed to stop audio recording!");
// fail
return Err(())
}
}
}
Ok(()) // if already stopped or not yet initialized
}

36
app/src/stt.rs Normal file
View file

@ -0,0 +1,36 @@
mod vosk;
use once_cell::sync::OnceCell;
use crate::config;
use crate::config::structs::SpeechToTextEngine;
static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !STT_TYPE.get().is_none() {return Ok(());} // already initialized
// set default stt type
// @TODO. Make it configurable?
STT_TYPE.set(config::DEFAULT_SPEECH_TO_TEXT_ENGINE).unwrap();
// load given recorder
match STT_TYPE.get().unwrap() {
SpeechToTextEngine::Vosk => {
// Init Vosk
info!("Initializing Vosk STT backend.");
vosk::init_vosk();
info!("STT backend initialized.");
}
}
Ok(())
}
pub fn recognize(data: &[i16], partial: bool) -> Option<String> {
match STT_TYPE.get().unwrap() {
SpeechToTextEngine::Vosk => {
vosk::recognize(data, partial)
}
}
}

63
app/src/stt/vosk.rs Normal file
View file

@ -0,0 +1,63 @@
use once_cell::sync::OnceCell;
use vosk::{DecodingState, Model, Recognizer};
use std::sync::Mutex;
use crate::config::VOSK_MODEL_PATH;
static MODEL: OnceCell<Model> = OnceCell::new();
static RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
pub fn init_vosk() {
if !RECOGNIZER.get().is_none() {return;} // already initialized
let model = Model::new(VOSK_MODEL_PATH).unwrap();
let mut recognizer = Recognizer::new(&model, 16000.0).unwrap();
recognizer.set_max_alternatives(10);
recognizer.set_words(true);
recognizer.set_partial_words(true);
MODEL.set(model);
RECOGNIZER.set(Mutex::new(recognizer));
}
pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
let state = RECOGNIZER.get().unwrap().lock().unwrap().accept_waveform(data);
match state {
DecodingState::Running => {
if include_partial {
Some(RECOGNIZER.get().unwrap().lock().unwrap().partial_result().partial.into())
} else {
None
}
}
DecodingState::Finalized => {
// Result will always be multiple because we called set_max_alternatives
Some(
RECOGNIZER.get().unwrap().lock().unwrap()
.result()
.multiple()
.unwrap()
.alternatives
.first()
.unwrap()
.text
.into(),
)
}
DecodingState::Failed => None,
}
}
// pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
// let mut result = Vec::with_capacity(input_data.len() / 2);
// result.extend(
// input_data
// .chunks_exact(2)
// .map(|chunk| chunk[0] / 2 + chunk[1] / 2),
// );
// result
// }

95
app/src/tray.rs Normal file
View file

@ -0,0 +1,95 @@
mod menu;
use tray_icon::{
menu::{AboutMetadata, Menu, MenuEvent, MenuItem, PredefinedMenuItem},
TrayEvent, TrayIconBuilder,
};
use winit::event_loop::{ControlFlow, EventLoopBuilder};
use image;
use winit::platform::windows::EventLoopBuilderExtWindows;
use crate::config;
pub fn init() {
// spawn tray icon
// New thread will prevent tray icon to work on MacOS
// @TODO: MacOS support.
std::thread::spawn(|| {
// load tray icon
let icon_path = format!("{}/icons/{}", env!("CARGO_MANIFEST_DIR"), config::TRAY_ICON);
let icon = load_icon(std::path::Path::new(&icon_path));
// form tray menu
let tray_menu = Menu::with_items(&[
&MenuItem::new("Перезапуск", true, None),
&MenuItem::new("Настройки", true, None),
&MenuItem::new("Выход", true, None)
]);
#[cfg(not(target_os = "linux"))]
let mut tray_icon = Some(
TrayIconBuilder::new()
.with_menu(Box::new(tray_menu))
.with_tooltip(config::TRAY_TOOLTIP)
.with_icon(icon)
.build()
.unwrap()
);
// Since winit doesn't use gtk on Linux, and we need gtk for
// the tray icon to show up, we need to initialize gtk and create the tray_icon
#[cfg(target_os = "linux")]
{
use tray_icon::menu::Menu;
gtk::init().unwrap();
let _tray_icon = TrayIconBuilder::new()
.with_menu(Box::new(tray_menu))
.with_tooltip(config::TRAY_TOOLTIP)
.with_icon(icon)
.build()
.unwrap();
gtk::main();
}
// run the event loop
let event_loop = EventLoopBuilder::new().with_any_thread(true).build();
let menu_channel = MenuEvent::receiver();
let tray_channel = TrayEvent::receiver();
event_loop.run(move |_event, _, control_flow| {
*control_flow = ControlFlow::Poll;
//if let Ok(event) = tray_channel.try_recv() {
// println!("tray event: {event:?}");
//}
if let Ok(event) = menu_channel.try_recv() {
println!("menu event: {:?}", event);
if event.id == 1002 {
std::process::exit(0);
}
}
});
});
info!("Tray initialized.");
}
fn load_icon(path: &std::path::Path) -> tray_icon::icon::Icon {
let (icon_rgba, icon_width, icon_height) = {
let image = image::open(path)
.expect("Failed to open icon path")
.into_rgba8();
let (width, height) = image.dimensions();
let rgba = image.into_raw();
(rgba, width, height)
};
tray_icon::icon::Icon::from_rgba(icon_rgba, icon_width, icon_height)
.expect("Failed to open icon")
}

15
app/src/tray/menu.rs Normal file
View file

@ -0,0 +1,15 @@
pub enum TrayMenuItem {
Restart,
Settings,
Exit
}
impl TrayMenuItem {
pub fn label(&self) -> &str {
match *self {
TrayMenuItem::Restart => "Перезапустить",
TrayMenuItem::Settings => "Настройки",
TrayMenuItem::Exit => "Выход"
}
}
}