bossiara13/J.A.R.V.I.S-py
1
1
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Activity Actions

project restructure with Rust workspaces

Browse source
This commit is contained in:
Priler 2026-01-04 05:19:47 +05:00
parent 1f03a44f33
commit 6e585dd37d
428 changed files with 19372 additions and 6061 deletions
Download patch file Download diff file Expand all files Collapse all files

27
crates/jarvis-core/Cargo.toml Normal file
View file

@ -0,0 +1,27 @@
[package]
name = "jarvis-core"
version.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
edition.workspace = true
[dependencies]
serde.workspace = true
serde_json.workspace = true
serde_yaml.workspace = true
once_cell.workspace = true
log.workspace = true
rand.workspace = true
seqdiff.workspace = true
hound.workspace = true
platform-dirs.workspace = true
rodio.workspace = true
kira.workspace = true
pv_recorder.workspace = true
vosk.workspace = true
rustpotter.workspace = true
[features]
default = ["jarvis_app"]
jarvis_app = []

88
crates/jarvis-core/src/audio.rs Normal file
View file

@ -0,0 +1,88 @@
mod kira;
mod rodio;
use once_cell::sync::OnceCell;
use std::cmp::Ordering;
use std::path::PathBuf;
use crate::config::structs::AudioType;
use crate::{config, DB, SOUND_DIR};
static AUDIO_TYPE: OnceCell<AudioType> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !AUDIO_TYPE.get().is_none() {
return Ok(());
} // already initialized
// set default audio type
// @TODO. Make it configurable?
AUDIO_TYPE.set(config::DEFAULT_AUDIO_TYPE).unwrap();
// load given audio backend
match AUDIO_TYPE.get().unwrap() {
AudioType::Rodio => {
// Init Rodio
info!("Initializing Rodio audio backend.");
match rodio::init() {
Ok(_) => {
info!("Successfully initialized Rodio audio backend.");
}
Err(msg) => {
error!("Failed to initialize Rodio audio backend.");
return Err(());
}
}
}
AudioType::Kira => {
// Init Kira
info!("Initializing Kira audio backend.");
match kira::init() {
Ok(_) => {
info!("Successfully initialized Kira audio backend.");
}
Err(msg) => {
error!("Failed to initialize Kira audio backend.");
return Err(());
}
}
}
}
Ok(())
}
pub fn play_sound(filename: &PathBuf) {
info!("Playing {}", filename.display());
match AUDIO_TYPE.get().unwrap() {
AudioType::Rodio => {
rodio::play_sound(filename, true);
}
AudioType::Kira => kira::play_sound(filename),
}
}
pub fn get_sound_directory() -> Option<PathBuf> {
let voice = DB.get().unwrap().voice.as_str();
let voice_path = SOUND_DIR.join(voice);
match voice_path.exists() && voice_path.cmp(&SOUND_DIR) != Ordering::Equal {
true => Some(voice_path),
_ => {
let default_voice_path = SOUND_DIR.join(config::DEFAULT_VOICE);
match default_voice_path.exists() {
true => Some(default_voice_path),
_ => {
error!("No sounds found. Search path - {:?}", voice_path);
None
}
}
}
}
}

62
crates/jarvis-core/src/audio/kira.rs Normal file
View file

@ -0,0 +1,62 @@
use once_cell::sync::OnceCell;
use std::path::PathBuf;
use std::sync::Mutex;
// use kira::{
// manager::{backend::DefaultBackend, AudioManager, AudioManagerSettings},
// sound::static_sound::{StaticSoundData, StaticSoundSettings},
// };
use kira::{
AudioManager, AudioManagerSettings, DefaultBackend,
sound::static_sound::StaticSoundData,
};
static MANAGER: OnceCell<Mutex<AudioManager>> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if MANAGER.get().is_some() {
return Ok(());
} // already initialized
// Create an audio manager. This plays sounds and manages resources.
match AudioManager::<DefaultBackend>::new(AudioManagerSettings::default()) {
Ok(manager) => {
// store
MANAGER.set(Mutex::new(manager)).ok();
// success
Ok(())
}
Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg);
// failed
Err(())
}
}
}
// @TODO. Cache sounds in memory? With a pool of a certain size, for instance.
pub fn play_sound(filename: &PathBuf) {
// load the file
match StaticSoundData::from_file(filename) {
Ok(sound_data) => {
// sound_data.duration() can be used in order to sleep, if (for some reason) blocking behaviour is required
// play it (non-blocking)
if let Some(manager) = MANAGER.get() {
if let Ok(mut audio_manager) = manager.lock() {
if let Err(e) = audio_manager.play(sound_data) {
warn!("Failed to play sound: {}", e);
}
}
} else {
warn!("Audio manager not initialized");
}
}
Err(err) => {
warn!("Cannot find sound file: {} (err: {})", filename.display(), err);
}
}
}

65
crates/jarvis-core/src/audio/rodio.rs Normal file
View file

@ -0,0 +1,65 @@
/*
Abandoned temporary.
Problems with blocking behaviour.
Possible fixes are running rodio in a separate thread or smthng.
*/
use once_cell::sync::OnceCell;
use std::fs::File;
use std::io::BufReader;
use std::path::PathBuf;
// use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink};
use rodio::{Decoder, OutputStream, Sink};
// static STREAM: OnceCell<OutputStream> = OnceCell::new();
static STREAM_HANDLE: OnceCell<OutputStream> = OnceCell::new();
static SINK: OnceCell<Sink> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !STREAM_HANDLE.get().is_none() {
return Ok(());
} // already initialized
// get output stream handle to the default physical sound device
match rodio::OutputStreamBuilder::open_default_stream() {
Ok(stream_handle) => {
// create sink
let sink = Sink::connect_new(&stream_handle.mixer());
info!("Sink initialized.");
// store
// STREAM.set(_stream).unwrap();
STREAM_HANDLE.set(stream_handle);
SINK.set(sink);
// success
Ok(())
}
Err(msg) => {
error!("Failed to initialize audio stream.\nError details: {}", msg);
// failed
Err(())
}
}
}
pub fn play_sound(filename: &PathBuf, sleep: bool) {
// Load a sound from a file, using a path relative to Cargo.toml
// let filepath = format!("{PUBLIC_PATH}/sound/{filename}.wav");
let file = BufReader::new(File::open(&filename).unwrap());
// Decode that sound file into a source
let source = Decoder::new(file).unwrap();
// Play the sound directly on the device
// STREAM_HANDLE.get().unwrap().play_raw(source.convert_samples());
SINK.get().unwrap().append(source);
if sleep {
// The sound plays in a separate thread. This call will block the current thread until the sink
// has finished playing all its queued sounds.
SINK.get().unwrap().sleep_until_end();
}
}

255
crates/jarvis-core/src/commands.rs Normal file
View file

@ -0,0 +1,255 @@
use rand::prelude::*;
use seqdiff::ratio;
use serde_yaml;
use std::path::Path;
use std::{fs, fs::File};
use core::time::Duration;
use std::path::PathBuf;
use std::process::{Child, Command};
// use tauri::Manager;
mod structs;
pub use structs::*;
use crate::{audio, config};
// @TODO. Allow commands both in yaml and json format.
pub fn parse_commands() -> Result<Vec<AssistantCommand>, String> {
// collect commands
let mut commands: Vec<AssistantCommand> = vec![];
// read commands directories first
if let Ok(cpaths) = fs::read_dir(config::COMMANDS_PATH) {
for cpath in cpaths {
// validate this command, check if required files exists
let _cpath = cpath.unwrap().path();
let cc_file = Path::new(&_cpath).join("command.yaml");
if cc_file.exists() {
// try parse config files
let cc_reader = std::fs::File::open(&cc_file).unwrap();
let cc_yaml: CommandsList;
// try parse command.yaml
match serde_yaml::from_reader::<File, CommandsList>(cc_reader) {
Ok(parse_result) => {
cc_yaml = parse_result;
}
Err(msg) => {
warn!(
"Can't parse {}, skipping ...\nCommand parse error is: {:?}",
&cc_file.display(),
msg
);
continue;
}
}
// everything seems to be Ok
commands.push(AssistantCommand {
path: _cpath,
commands: cc_yaml,
});
}
}
if commands.len() > 0 {
Ok(commands)
} else {
error!("No commands were found");
Err("No commands were found".into())
}
} else {
error!("Error reading commands directory");
return Err("Error reading commands directory".into());
}
}
// @TODO. NLU or smthng else is required, in order to infer commands with highest accuracy possible.
pub fn fetch_command<'a>(
phrase: &str,
commands: &'a Vec<AssistantCommand>,
) -> Option<(&'a PathBuf, &'a Config)> {
// result scmd
let mut result_scmd: Option<(&PathBuf, &Config)> = None;
let mut current_max_ratio = config::CMD_RATIO_THRESHOLD;
// convert fetch phrase to sequence
let fetch_phrase_chars = phrase.chars().collect::<Vec<_>>();
// list all the commands
for cmd in commands {
// list all subcommands
for scmd in &cmd.commands.list {
// list all phrases in command
for cmd_phrase in &scmd.phrases {
// convert cmd phrase to sequence
let cmd_phrase_chars = cmd_phrase.chars().collect::<Vec<_>>();
// compare fetch phrase with cmd phrase
let ratio = ratio(&fetch_phrase_chars, &cmd_phrase_chars);
// return, if it fits the given threshold
if ratio >= current_max_ratio {
result_scmd = Some((&cmd.path, &scmd));
current_max_ratio = ratio;
// println!("Ratio is: {}", ratio);
// return Some((&cmd.path, &scmd))
}
}
}
}
if let Some((cmd_path, scmd)) = result_scmd {
println!("Ratio is: {}", current_max_ratio);
info!(
"CMD is: {cmd_path:?}, SCMD is: {scmd:?}, Ratio is: {}",
current_max_ratio
);
Some((&cmd_path, &scmd))
} else {
None
}
}
// @TODO. Rewrite executors by executor type struct. (with match arms)
pub fn execute_exe(exe: &str, args: &Vec<String>) -> std::io::Result<Child> {
Command::new(exe).args(args).spawn()
}
pub fn execute_cli(cmd: &str, args: &Vec<String>) -> std::io::Result<Child> {
println!("Spawning cmd as: cmd /C {} {:?}", cmd, args);
if cfg!(target_os = "windows") {
Command::new("cmd").arg("/C").arg(cmd).args(args).spawn()
} else {
Command::new("sh").arg("-c").arg(cmd).args(args).spawn()
}
}
pub fn execute_command(
cmd_path: &PathBuf,
cmd_config: &Config,
// app_handle: &tauri::AppHandle,
) -> Result<bool, String> {
let sounds_directory = audio::get_sound_directory().unwrap();
match cmd_config.command.action.as_str() {
"voice" => {
// VOICE command type
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(true)
}
"ahk" => {
// AutoHotkey command type
let exe_path_absolute = Path::new(&cmd_config.command.exe_path);
let exe_path_local = Path::new(&cmd_path).join(&cmd_config.command.exe_path);
if let Ok(_) = execute_exe(
if exe_path_absolute.exists() {
exe_path_absolute.to_str().unwrap()
} else {
exe_path_local.to_str().unwrap()
},
&cmd_config.command.exe_args,
) {
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(true)
} else {
error!("AHK process spawn error (does exe path is valid?)");
Err("AHK process spawn error (does exe path is valid?)".into())
}
}
"cli" => {
// CLI command type
let cli_cmd = &cmd_config.command.cli_cmd;
match execute_cli(cli_cmd, &cmd_config.command.cli_args) {
Ok(_) => {
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(true)
}
Err(msg) => {
error!("CLI command error ({})", msg);
Err(format!("Shell command error ({})", msg).into())
}
}
}
"terminate" => {
// TERMINATE command type
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
std::thread::sleep(Duration::from_secs(2));
std::process::exit(0);
}
"stop_chaining" => {
// STOP_CHAINING command type
let random_cmd_sound = format!(
"{}.wav",
cmd_config
.voice
.sounds
.choose(&mut rand::thread_rng())
.unwrap()
);
// events::play(random_cmd_sound, app_handle);
audio::play_sound(&sounds_directory.join(random_cmd_sound));
Ok(false)
}
_ => {
error!("Command type unknown");
Err("Command type unknown".into())
}
}
}
pub fn list(from: &[AssistantCommand]) -> Vec<String> {
let mut out: Vec<String> = vec![];
for x in from.iter() {
out.push(String::from(x.path.to_str().unwrap()));
// out.append()
}
out
}

45
crates/jarvis-core/src/commands/structs.rs Normal file
View file

@ -0,0 +1,45 @@
use serde::Deserialize;
use std::path::PathBuf;
#[derive(Debug)]
pub struct AssistantCommand {
pub path: PathBuf,
pub commands: CommandsList,
}
#[derive(Deserialize, Debug)]
pub struct CommandsList {
pub list: Vec<Config>,
}
#[derive(Deserialize, Debug)]
pub struct Config {
pub command: ConfigCommandSection,
pub voice: ConfigVoiceSection,
pub phrases: Vec<String>,
}
#[derive(Deserialize, Debug)]
pub struct ConfigCommandSection {
pub action: String,
#[serde(default)]
pub exe_path: String,
#[serde(default)]
pub exe_args: Vec<String>,
#[serde(default)]
pub cli_cmd: String,
#[serde(default)]
pub cli_args: Vec<String>,
}
#[derive(Deserialize, Debug)]
pub struct ConfigVoiceSection {
#[serde(default)]
pub sounds: Vec<String>,
}

156
crates/jarvis-core/src/config.rs Normal file
View file

@ -0,0 +1,156 @@
pub mod structs;
use structs::AudioType;
use structs::RecorderType;
use structs::SpeechToTextEngine;
use structs::WakeWordEngine;
use once_cell::sync::Lazy;
use std::env;
use std::fs;
use std::path::PathBuf;
use platform_dirs::AppDirs;
#[cfg(feature="jarvis_app")]
use rustpotter::{
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig,
RustpotterConfig, ScoreMode,
};
use crate::{APP_CONFIG_DIR, APP_DIRS, APP_LOG_DIR};
#[allow(dead_code)]
pub fn init_dirs() -> Result<(), String> {
// infer app dirs
if APP_DIRS.get().is_some() {
return Ok(());
}
// cache_dir, config_dir, data_dir, state_dir
APP_DIRS
.set(AppDirs::new(Some(BUNDLE_IDENTIFIER), false).unwrap())
.unwrap();
// setup directories
let mut config_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
let mut log_dir = PathBuf::from(&APP_DIRS.get().unwrap().config_dir);
// create dirs, if required
if !config_dir.exists() {
if fs::create_dir_all(&config_dir).is_err() {
config_dir = env::current_dir().expect("Cannot infer the config directory");
fs::create_dir_all(&config_dir)
.expect("Cannot create config directory, access denied?");
}
}
if !log_dir.exists() {
if fs::create_dir_all(&log_dir).is_err() {
log_dir = env::current_dir().expect("Cannot infer the log directory");
fs::create_dir_all(&log_dir).expect("Cannot create log directory, access denied?");
}
}
// store inferred paths
APP_CONFIG_DIR.set(config_dir).unwrap();
APP_LOG_DIR.set(log_dir).unwrap();
Ok(())
}
/*
Defaults.
*/
pub const DEFAULT_AUDIO_TYPE: AudioType = AudioType::Kira;
pub const DEFAULT_RECORDER_TYPE: RecorderType = RecorderType::PvRecorder;
pub const DEFAULT_WAKE_WORD_ENGINE: WakeWordEngine = WakeWordEngine::Rustpotter;
pub const DEFAULT_SPEECH_TO_TEXT_ENGINE: SpeechToTextEngine = SpeechToTextEngine::Vosk;
pub const DEFAULT_VOICE: &str = "jarvis-og";
pub const BUNDLE_IDENTIFIER: &str = "com.priler.jarvis";
pub const DB_FILE_NAME: &str = "app.db";
pub const LOG_FILE_NAME: &str = "log.txt";
pub const APP_VERSION: Option<&str> = option_env!("CARGO_PKG_VERSION");
pub const AUTHOR_NAME: Option<&str> = option_env!("CARGO_PKG_AUTHORS");
pub const REPOSITORY_LINK: Option<&str> = option_env!("CARGO_PKG_REPOSITORY");
pub const TG_OFFICIAL_LINK: Option<&str> = Some("https://t.me/howdyho_official");
pub const FEEDBACK_LINK: Option<&str> = Some("https://t.me/jarvis_feedback_bot");
/*
Tray.
*/
pub const TRAY_ICON: &str = "32x32.png";
pub const TRAY_TOOLTIP: &str = "Jarvis Voice Assistant";
// RUSPOTTER
pub const RUSPOTTER_MIN_SCORE: f32 = 0.62;
#[cfg(feature="jarvis_app")]
pub const RUSTPOTTER_DEFAULT_CONFIG: Lazy<RustpotterConfig> = Lazy::new(|| {
RustpotterConfig {
fmt: AudioFmt::default(),
detector: DetectorConfig {
avg_threshold: 0.,
threshold: 0.5,
min_scores: 15,
score_ref: 0.22,
band_size: 5,
vad_mode: None,
score_mode: ScoreMode::Max,
eager: false,
// comparator_band_size: 5,
// comparator_ref: 0.22
},
filters: FiltersConfig {
gain_normalizer: GainNormalizationConfig {
enabled: true,
gain_ref: None,
min_gain: 0.7,
max_gain: 1.0,
},
band_pass: BandPassConfig {
enabled: true,
low_cutoff: 80.,
high_cutoff: 400.,
},
},
}
});
// PICOVOICE
pub const COMMANDS_PATH: &str = "commands/";
pub const KEYWORDS_PATH: &str = "picovoice/keywords/";
pub const DEFAULT_KEYWORD: &str = "jarvis_windows.ppn";
pub const DEFAULT_SENSITIVITY: f32 = 1.0;
// VOSK
// pub const VOSK_MODEL_PATH: &str = const_concat!(PUBLIC_PATH, "/vosk/model_small");
pub const VOSK_FETCH_PHRASE: &str = "джарвис";
pub const VOSK_MODEL_PATH: &str = "vosk/model_small";
pub const VOSK_MIN_RATIO: f64 = 70.0;
// ETC
pub const CMD_RATIO_THRESHOLD: f64 = 65f64;
pub const CMS_WAIT_DELAY: std::time::Duration = std::time::Duration::from_secs(15);
pub const ASSISTANT_GREET_PHRASES: [&str; 3] = ["greet1", "greet2", "greet3"];
pub const ASSISTANT_PHRASES_TBR: [&str; 17] = [
"джарвис",
"сэр",
"слушаю сэр",
"всегда к услугам",
"произнеси",
"ответь",
"покажи",
"скажи",
"давай",
"да сэр",
"к вашим услугам сэр",
"всегда к вашим услугам сэр",
"запрос выполнен сэр",
"выполнен сэр",
"есть",
"загружаю сэр",
"очень тонкое замечание сэр",
];

43
crates/jarvis-core/src/config/structs.rs Normal file
View file

@ -0,0 +1,43 @@
use std::fmt;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
pub enum WakeWordEngine {
Rustpotter,
Vosk,
Porcupine,
}
impl fmt::Display for WakeWordEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[derive(Serialize, Deserialize, Debug)]
pub enum SpeechToTextEngine {
Vosk,
}
impl fmt::Display for SpeechToTextEngine {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{:?}", self)
}
}
#[derive(PartialEq, Debug)]
pub enum RecorderType {
Cpal,
PvRecorder,
PortAudio,
}
#[derive(PartialEq, Debug)]
pub enum AudioType {
Rodio,
Kira,
}
// pub enum TextToSpeechEngine {}
// pub enum IntentRecognitionEngine {}

58
crates/jarvis-core/src/db.rs Normal file
View file

@ -0,0 +1,58 @@
pub mod structs;
use crate::{config, APP_CONFIG_DIR};
use log::info;
use std::fs::File;
use std::io::{BufReader, Read};
use std::path::PathBuf;
use serde_json;
fn get_db_file_path() -> PathBuf {
PathBuf::from(format!(
"{}/{}",
APP_CONFIG_DIR.get().unwrap().display(),
config::DB_FILE_NAME
))
}
pub fn init_settings() -> structs::Settings {
let mut db = None;
let db_file_path = get_db_file_path();
info!(
"Loading settings db file located at: {}",
db_file_path.display()
);
if db_file_path.exists() {
// try load existing settings
if let Ok(mut db_file) = File::open(db_file_path) {
let reader = BufReader::new(db_file);
if let Ok(parsed_json) = serde_json::from_reader(reader) {
info!("Settings loaded.");
db = Some(parsed_json);
}
}
}
if db.is_none() {
// create default settings db file
warn!("No settings file found or there was an error parsing it. Creating default struct.");
db = Some(structs::Settings::default());
}
db.unwrap()
}
pub fn save_settings(settings: &structs::Settings) -> Result<(), std::io::Error> {
let db_file_path = get_db_file_path();
std::fs::write(
db_file_path,
serde_json::to_string_pretty(&settings).unwrap(),
)?;
info!("Settings saved.");
Ok(())
}

39
crates/jarvis-core/src/db/structs.rs Normal file
View file

@ -0,0 +1,39 @@
use crate::config;
use serde::{Deserialize, Serialize};
use crate::config::structs::SpeechToTextEngine;
use crate::config::structs::WakeWordEngine;
#[derive(Serialize, Deserialize, Debug)]
pub struct Settings {
pub microphone: i32,
pub voice: String,
pub wake_word_engine: WakeWordEngine,
pub speech_to_text_engine: SpeechToTextEngine,
pub api_keys: ApiKeys,
}
impl Default for Settings {
fn default() -> Settings {
Settings {
microphone: -1,
voice: String::from(""),
wake_word_engine: config::DEFAULT_WAKE_WORD_ENGINE,
speech_to_text_engine: config::DEFAULT_SPEECH_TO_TEXT_ENGINE,
api_keys: ApiKeys {
picovoice: String::from(""),
openai: String::from(""),
},
}
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct ApiKeys {
pub picovoice: String,
pub openai: String,
}

28
crates/jarvis-core/src/lib.rs Normal file
View file

@ -0,0 +1,28 @@
use once_cell::sync::{Lazy, OnceCell};
use platform_dirs::AppDirs;
use std::path::PathBuf;
#[macro_use]
extern crate log;
pub mod audio;
pub mod commands;
pub mod config;
pub mod db;
pub mod listener;
pub mod recorder;
pub mod stt;
// shared statics
pub static APP_DIR: Lazy<PathBuf> = Lazy::new(|| std::env::current_dir().unwrap());
pub static SOUND_DIR: Lazy<PathBuf> = Lazy::new(|| APP_DIR.clone().join("sound"));
pub static APP_DIRS: OnceCell<AppDirs> = OnceCell::new();
pub static APP_CONFIG_DIR: OnceCell<PathBuf> = OnceCell::new();
pub static APP_LOG_DIR: OnceCell<PathBuf> = OnceCell::new();
pub static DB: OnceCell<db::structs::Settings> = OnceCell::new();
pub static COMMANDS_LIST: OnceCell<Vec<commands::AssistantCommand>> = OnceCell::new();
// re-exports
pub use commands::AssistantCommand;
pub use config::structs::*;
pub use db::structs::Settings;

65
crates/jarvis-core/src/listener.rs Normal file
View file

@ -0,0 +1,65 @@
// mod porcupine;
mod rustpotter;
mod vosk;
use once_cell::sync::OnceCell;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::config::structs::WakeWordEngine;
use crate::{config, stt};
use crate::DB;
// store wake-word engine being used
static WAKE_WORD_ENGINE: OnceCell<WakeWordEngine> = OnceCell::new();
// track listening state
static LISTENING: AtomicBool = AtomicBool::new(false);
pub fn init() -> Result<(), ()> {
if !WAKE_WORD_ENGINE.get().is_none() {
return Ok(());
} // already initialized
// store current engine
WAKE_WORD_ENGINE
.set(DB.get().unwrap().wake_word_engine)
.unwrap();
// load given wake-word engine
match WAKE_WORD_ENGINE.get().unwrap() {
WakeWordEngine::Porcupine => {
// Init Porcupine wake-word engine
info!("Initializing Porcupine wake-word engine.");
// return porcupine::init();
unimplemented!("f*ck picovoice");
}
WakeWordEngine::Rustpotter => {
// Init Rustpotter wake-word engine
info!("Initializing Rustpotter wake-word engine.");
return rustpotter::init();
}
WakeWordEngine::Vosk => {
// Init Vosk as wake-word engine (very slow, though)
info!("Initializing Vosk as wake-word engine.");
warn!("Using Vosk as wake-word engine is highly not recommended, because it's very slow for this task.");
return vosk::init();
}
}
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
match WAKE_WORD_ENGINE.get().unwrap() {
WakeWordEngine::Porcupine => {
// porcupine::data_callback(frame_buffer)
unimplemented!("f*ck picovoice");
},
WakeWordEngine::Rustpotter => rustpotter::data_callback(frame_buffer),
WakeWordEngine::Vosk => vosk::data_callback(frame_buffer),
}
}

56
crates/jarvis-core/src/listener/porcupine.rs Normal file
View file

@ -0,0 +1,56 @@
use std::path::Path;
use once_cell::sync::OnceCell;
use porcupine::{Porcupine, PorcupineBuilder};
use crate::config;
use crate::DB;
// store porcupine instance
static PORCUPINE: OnceCell<Porcupine> = OnceCell::new();
pub fn init() -> Result<(), ()> {
let picovoice_api_key: String;
// retrieve picovoice api key
picovoice_api_key = DB.get().unwrap().api_keys.picovoice.clone();
if picovoice_api_key.trim().is_empty() {
warn!("Picovoice API key is not set.");
return Err(());
}
// create porcupine instance with the given API key
match PorcupineBuilder::new_with_keyword_paths(
picovoice_api_key,
&[Path::new(config::KEYWORDS_PATH).join(config::DEFAULT_KEYWORD)],
)
.sensitivities(&[config::DEFAULT_SENSITIVITY]) // set sensitivity
.init()
{
Ok(pinstance) => {
// success
info!("Porcupine successfully initialized with the given API key.");
// store
PORCUPINE.set(pinstance);
}
Err(msg) => {
error!("Porcupine failed to initialize, either API key is not valid or there is no internet connection.");
error!("Error details: {}", msg);
return Err(());
}
}
Ok(())
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
if let Ok(keyword_index) = PORCUPINE.get().unwrap().process(&frame_buffer) {
if keyword_index >= 0 {
return Some(keyword_index);
}
}
None
}

68
crates/jarvis-core/src/listener/rustpotter.rs Normal file
View file

@ -0,0 +1,68 @@
use std::path::Path;
use std::sync::Mutex;
use once_cell::sync::OnceCell;
use rustpotter::{
AudioFmt, BandPassConfig, DetectorConfig, FiltersConfig, GainNormalizationConfig, Rustpotter,
RustpotterConfig, ScoreMode,
};
use crate::config;
use crate::DB;
// store rustpotter instance
static RUSTPOTTER: OnceCell<Mutex<Rustpotter>> = OnceCell::new();
pub fn init() -> Result<(), ()> {
let rustpotter_config = config::RUSTPOTTER_DEFAULT_CONFIG;
// create rustpotter instance
match Rustpotter::new(&rustpotter_config) {
Ok(mut rinstance) => {
// success
// wake word files list
// @TODO. Make it configurable via GUI for custom user voice.
let rustpotter_wake_word_files: [&str; 1] = [
"rustpotter/jarvis-default.rpw",
// "rustpotter/jarvis-community-1.rpw",
// "rustpotter/jarvis-community-2.rpw",
// "rustpotter/jarvis-community-3.rpw",
// "rustpotter/jarvis-community-4.rpw",
// "rustpotter/jarvis-community-5.rpw",
];
// load wake word files
for rpw in rustpotter_wake_word_files {
rinstance.add_wakeword_from_file(rpw, rpw).unwrap(); // @TODO: Change wakeword key to something else?
}
// store
RUSTPOTTER.set(Mutex::new(rinstance));
}
Err(msg) => {
error!("Rustpotter failed to initialize.\nError details: {}", msg);
return Err(());
}
}
Ok(())
}
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
let mut lock = RUSTPOTTER.get().unwrap().lock();
let rustpotter = lock.as_mut().unwrap();
let detection = rustpotter.process_samples(frame_buffer.to_vec()); // @TODO. Temp crutch. Fix optimization issue, frame_buffer should not be copied to a new vector!
if let Some(detection) = detection {
if detection.score > config::RUSPOTTER_MIN_SCORE {
info!("Rustpotter detection info:\n{:?}", detection);
return Some(0);
} else {
info!("Rustpotter detection info:\n{:?}", detection)
}
}
None
}

36
crates/jarvis-core/src/listener/vosk.rs Normal file
View file

@ -0,0 +1,36 @@
use crate::{config, stt};
pub fn init() -> Result<(), ()> {
Ok(()) // nothing to init for Vosk
}
// @TODO. Make it better somehow (more accurate or with higher sensitivity).
pub fn data_callback(frame_buffer: &[i16]) -> Option<i32> {
// recognize & convert to sequence
let recognized_phrase = stt::recognize(&frame_buffer, true).unwrap_or("".into());
if !recognized_phrase.trim().is_empty() {
info!("Vosk wake-word debug info:");
info!("rec: {}", recognized_phrase);
let recognized_phrases = recognized_phrase.split_whitespace();
for phrase in recognized_phrases {
let recognized_phrase_chars = phrase.trim().to_lowercase().chars().collect::<Vec<_>>();
// compare
let compare_ratio = seqdiff::ratio(
&config::VOSK_FETCH_PHRASE.chars().collect::<Vec<_>>(),
&recognized_phrase_chars,
);
info!("og phrase: {:?}", &config::VOSK_FETCH_PHRASE);
info!("recognized phrase: {:?}", &recognized_phrase_chars);
info!("compare ratio: {}", compare_ratio);
if compare_ratio >= config::VOSK_MIN_RATIO {
info!("Phrase activated.");
return Some(0);
}
}
}
None
}

150
crates/jarvis-core/src/recorder.rs Normal file
View file

@ -0,0 +1,150 @@
mod pvrecorder;
// mod cpal;
// mod portaudio;
use once_cell::sync::OnceCell;
use crate::{config, config::structs::RecorderType, DB};
static RECORDER_TYPE: OnceCell<RecorderType> = OnceCell::new();
static FRAME_LENGTH: OnceCell<u32> = OnceCell::new();
pub fn init() -> Result<(), ()> {
// set default recorder type
// @TODO. Make it configurable?
RECORDER_TYPE.set(config::DEFAULT_RECORDER_TYPE).unwrap();
// some info
info!("Loading recorder ...");
info!("Available audio_devices are:\n{:?}", get_audio_devices());
// load given recorder
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
// Init Pv Recorder
info!("Initializing PvRecorder recording backend.");
FRAME_LENGTH.set(512u32).unwrap(); // pvrecorder requires frame buffer of 512
let selected_microphone = get_selected_microphone_index();
match pvrecorder::init_microphone(
selected_microphone,
FRAME_LENGTH.get().unwrap().to_owned(),
) {
false => {
error!("Recorder initialization failed.");
return Err(());
}
_ => {
info!(
"Recorder initialization success. Listening to microphone ({}): {}",
selected_microphone,
get_audio_device_name(selected_microphone)
);
}
}
}
RecorderType::PortAudio => {
// Init PortAudio
info!("Initializing PortAudio recording backend");
todo!();
// match portaudio::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
// false => {
// // Switch to PortAudio recorder
// error!("PortAudio audio backend failed.");
// },
// _ => ()
// }
}
RecorderType::Cpal => {
// Init CPAL
info!("Initializing CPAL recording backend");
todo!();
// match cpal::init_microphone(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst)) {
// false => {
// // Switch to CPAL recorder
// error!("CPAL audio backend failed.");
// },
// _ => ()
// }
}
}
Ok(())
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
pvrecorder::read_microphone(frame_buffer);
}
RecorderType::PortAudio => {
todo!();
// portaudio::read_microphone(frame_buffer);
}
RecorderType::Cpal => {
// cpal::read_microphone(frame_buffer);
panic!("Cpal should be used via callback assignment");
}
}
}
pub fn start_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => {
return pvrecorder::start_recording(
get_selected_microphone_index(),
FRAME_LENGTH.get().unwrap().to_owned(),
);
}
RecorderType::PortAudio => {
todo!();
// portaudio::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
}
RecorderType::Cpal => {
todo!();
// cpal::start_recording(get_selected_microphone_index(), FRAME_LENGTH.load(Ordering::SeqCst));
}
}
}
pub fn stop_recording() -> Result<(), ()> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => pvrecorder::stop_recording(),
RecorderType::PortAudio => {
todo!();
// portaudio::stop_recording();
}
RecorderType::Cpal => {
todo!();
// cpal::stop_recording();
}
}
}
pub fn get_selected_microphone_index() -> i32 {
DB.get().unwrap().microphone
}
pub fn get_audio_devices() -> Vec<String> {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => pvrecorder::list_audio_devices(),
RecorderType::PortAudio => {
todo!();
}
RecorderType::Cpal => {
todo!();
}
}
}
pub fn get_audio_device_name(idx: i32) -> String {
match RECORDER_TYPE.get().unwrap() {
RecorderType::PvRecorder => pvrecorder::get_audio_device_name(idx),
RecorderType::PortAudio => {
todo!();
}
RecorderType::Cpal => {
todo!();
}
}
}

191
crates/jarvis-core/src/recorder/cpal.rs Normal file
View file

@ -0,0 +1,191 @@
/*
Abandoned temporary.
Problems with frame size.
*/
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{BufferSize, StreamConfig, SampleRate, Host, Device, Stream, SampleFormat};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::Arc;
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
use crate::tauri_commands::cpal_data_callback;
static HOST: OnceCell<Host> = OnceCell::new();
thread_local!(static RECORDER: OnceCell<ArcSwap<Stream>> = OnceCell::new());
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
// init host & frame buffer for the callback
if HOST.get().is_none() {
HOST.set(cpal::default_host());
// FRAME_BUFFER.set(Mutex::new(vec![0; FRAME_LENGTH.load(Ordering::SeqCst) as usize]));
}
// init microphone
RECORDER.with(|recorder| {
match recorder.get().is_none() {
true => {
if let Some(device) = get_device(device_index as usize) {
// store
recorder.set(ArcSwap::from_pointee(create_stream(device, frame_length)));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
} else {
false
}
},
false => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
stop_recording();
}
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
if let Some(device) = get_device(device_index as usize) {
// store
recorder.get().unwrap().store(Arc::new(create_stream(device, frame_length)));
// success
return true
} else {
return false
}
}
// success
true
}
}
})
}
fn create_stream(device: Device, frame_length: u32) -> Stream {
// get default input stream config
// let default_config = device.default_input_config().unwrap();
// create config for the stream
// let config: StreamConfig = StreamConfig {
// channels: default_config.channels(),
// sample_rate: SampleRate(16000),
// buffer_size: BufferSize::Fixed(frame_length)
// };
let config = device
.default_input_config()
.expect("Failed to load default input config");
let channels = config.channels();
let err_fn = move |err| {
eprintln!("an error occurred on stream: {}", err);
};
match config.sample_format() {
SampleFormat::F32 => device.build_input_stream(
&config.into(),
move |data: &[f32], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
SampleFormat::U16 => device.build_input_stream(
&config.into(),
move |data: &[u16], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
SampleFormat::I16 => device.build_input_stream(
&config.into(),
move |data: &[i16], info| {
cpal_data_callback(data, channels);
},
err_fn,
None
),
_ => todo!()
}.unwrap()
}
pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
let mut result = Vec::with_capacity(input_data.len() / 2);
result.extend(
input_data
.chunks_exact(2)
.map(|chunk| chunk[0] / 2 + chunk[1] / 2),
);
result
}
fn get_device(device_index: usize) -> Option<Device> {
if let Some(device) = HOST.get().unwrap().input_devices().expect("Get devices error ...").nth(device_index) {
Some(device)
} else {
if let Some(default) = HOST.get().unwrap().default_input_device() {
Some(default)
} else {
error!("No default input device ...");
None
}
}
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.with(|recorder| {
match recorder.get().unwrap().load().play() {
Err(msg) => {
error!("[CPAL] Audio stream PLAY error ... {:?}", msg);
},
_ => ()
};
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
});
}
pub fn stop_recording() {
// ensure microphone is initialized
RECORDER.with(|recorder| {
if !recorder.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// pause instead of stop
match recorder.get().unwrap().load().pause() {
Err(msg) => {
error!("[CPAL] Audio stream PAUSE error ... {:?}", msg);
},
_ => ()
};
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
});
}

205
crates/jarvis-core/src/recorder/portaudio.rs Normal file
View file

@ -0,0 +1,205 @@
/*
Abandoned temporary.
*/
use portaudio as pa;
use pa::{DeviceIndex, Stream};
use log::{info, warn, error};
use once_cell::sync::OnceCell;
use std::sync::{Arc, Mutex};
use arc_swap::ArcSwap;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
thread_local!(static RECORDER: OnceCell<ArcSwap<Mutex<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>>>> = OnceCell::new());
static SELECTED_MICROPHONE_IDX: AtomicI32 = AtomicI32::new(0);
static FRAME_LENGTH: AtomicU32 = AtomicU32::new(0);
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
const CHANNELS: i32 = 1;
const SAMPLE_RATE: f64 = 16_000.0;
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
RECORDER.with(|r| {
match r.get().is_none() {
true => {
match create_stream(device_index, frame_length) {
Ok(stream) => {
// store
r.set(ArcSwap::from_pointee(Mutex::new(stream)));
// remember current configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
true
},
Err(msg) => {
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
// fail
false
}
}
},
_ => {
// check if re-initialization required (i.e. selecetd microphoneor frame-length was changed )
if SELECTED_MICROPHONE_IDX.load(Ordering::SeqCst) != device_index
||
FRAME_LENGTH.load(Ordering::SeqCst) != frame_length {
warn!("Selected microphone or frame length was changed, re-initializing ...");
// initialize again with new device index
if IS_RECORDING.load(Ordering::SeqCst) {
// RECORDER.get().unwrap().load().stop().expect("Failed to start audio recording!");
stop_recording();
}
// store
match create_stream(device_index, frame_length) {
Ok(stream) => {
// store new stream
r.get().unwrap().store(Arc::new(Mutex::new(stream)));
// remember new configuration
SELECTED_MICROPHONE_IDX.store(device_index, Ordering::SeqCst);
FRAME_LENGTH.store(frame_length, Ordering::SeqCst);
// success
return true
},
Err(msg) => {
error!("Failed to initialize portaudio.\nError details: {:?}", msg);
// fail
return false
}
}
}
// success
true
}
}
})
}
fn create_stream(device_index: i32, frame_length: u32) -> Result<Stream<pa::Blocking<pa::stream::Buffer>, pa::Input<i16>>, pa::Error> {
let pa_recorder: Result<pa::PortAudio, pa::Error> = pa::PortAudio::new();
match pa_recorder {
Ok(pa) => {
let input_settings = match get_input_settings(DeviceIndex(device_index as u32), &pa, SAMPLE_RATE, frame_length, CHANNELS) {
Ok(settings) => settings,
Err(error) => panic!("{}", String::from(error))
};
// Construct a stream with input and output sample types of i16
match pa.open_blocking_stream(input_settings) {
Ok(strm) => Ok(strm),
Err(error) => panic!("{}", error.to_string()),
}
},
Err(msg) => Err(msg)
}
}
fn get_input_latency(audio_port: &pa::PortAudio, input_index: pa::DeviceIndex) -> Result<f64, String>
{
let input_device_information = audio_port.device_info(input_index).or_else(|error| Err(String::from(format!("{}", error))));
Ok(input_device_information.unwrap().default_low_input_latency)
}
fn get_input_stream_parameters(input_index: pa::DeviceIndex, latency: f64, channels: i32) -> Result<pa::StreamParameters<i16>, String>
{
const INTERLEAVED: bool = true;
Ok(pa::StreamParameters::<i16>::new(input_index, channels, INTERLEAVED, latency))
}
fn get_input_settings(input_index: pa::DeviceIndex, audio_port: &pa::PortAudio, sample_rate: f64, frames: u32, channels: i32) -> Result<pa::InputStreamSettings<i16>, String>
{
Ok(
pa::InputStreamSettings::new(
(get_input_stream_parameters(
input_index,
(get_input_latency(
&audio_port,
input_index,
))?,
channels
))?,
sample_rate,
frames,
)
)
}
// We'll use this function to wait for read/write availability.
fn wait_for_stream<F>(f: F, name: &str) -> u32
where
F: Fn() -> Result<pa::StreamAvailable, pa::error::Error>,
{
loop {
match f() {
Ok(available) => match available {
pa::StreamAvailable::Frames(frames) => return frames as u32,
pa::StreamAvailable::InputOverflowed => println!("Input stream has overflowed"),
pa::StreamAvailable::OutputUnderflowed => {
println!("Output stream has underflowed")
}
},
Err(err) => panic!(
"An error occurred while waiting for the {} stream: {}",
name, err
),
}
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
RECORDER.with(|r| {
if !r.get().is_none() {
let cell = r.get().unwrap().load();
let mut lock = cell.lock();
let stream = lock.as_mut().unwrap();
// read to frame buffer
let in_frames = wait_for_stream(|| stream.read_available(), "Read");
if in_frames > 0 {
// let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
// println!("Read {:?} frames from the input stream.", in_frames);
let input_samples = stream.read(in_frames).expect("Cannot read frames ...");
println!("Read: {} (required {})", input_samples.len(), frame_buffer.len());
frame_buffer.copy_from_slice(input_samples.chunks(frame_buffer.len()).last().unwrap());
}
// r.get().unwrap().load().read(frame_buffer).expect("Failed to read audio frame");
}
});
}
pub fn start_recording(device_index: i32, frame_length: u32) {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
RECORDER.with(|r| {
r.get().unwrap().load().lock().unwrap().start().expect("Failed to start audio recording!");
IS_RECORDING.store(true, Ordering::SeqCst);
info!("START recording from microphone ...");
});
}
pub fn stop_recording() {
RECORDER.with(|r| {
if !r.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// stop recording
let pa = r.get().unwrap().load();
r.get().unwrap().load().lock().unwrap().stop().expect("Failed to stop audio recording!");
IS_RECORDING.store(false, Ordering::SeqCst);
info!("STOP recording from microphone ...");
}
});
}

129
crates/jarvis-core/src/recorder/pvrecorder.rs Normal file
View file

@ -0,0 +1,129 @@
use once_cell::sync::OnceCell;
use pv_recorder::{PvRecorder, PvRecorderBuilder};
use std::sync::atomic::{AtomicBool, Ordering};
static RECORDER: OnceCell<PvRecorder> = OnceCell::new();
static IS_RECORDING: AtomicBool = AtomicBool::new(false);
pub fn init_microphone(device_index: i32, frame_length: u32) -> bool {
match RECORDER.get().is_none() {
true => {
let pv_recorder = PvRecorderBuilder::new(frame_length as i32)
.device_index(device_index)
// .frame_length(frame_length as i32)
.init();
match pv_recorder {
Ok(pv) => {
// store
RECORDER.set(pv);
// success
true
}
Err(msg) => {
error!("Failed to initialize pvrecorder.\nError details: {:?}", msg);
// fail
false
}
}
}
_ => true, // already initialized
}
}
pub fn read_microphone(frame_buffer: &mut [i16]) {
// ensure microphone is initialized
if !RECORDER.get().is_none() {
// read to frame buffer
let frame = RECORDER.get().unwrap().read();
match frame {
Ok(f) => {
frame_buffer.copy_from_slice(f.as_slice());
}
Err(msg) => {
// @TODO: Fix? PvRecorder always wait for PCM buffer size of 512.
error!("Failed to read audio frame. {:?}", msg);
}
}
}
}
pub fn start_recording(device_index: i32, frame_length: u32) -> Result<(), ()> {
// ensure microphone is initialized
init_microphone(device_index, frame_length);
// start recording
match RECORDER.get().unwrap().start() {
Ok(_) => {
info!("START recording from microphone ...");
// change recording state
IS_RECORDING.store(true, Ordering::SeqCst);
// success
Ok(())
}
Err(msg) => {
error!("Failed to start audio recording!");
// fail
Err(())
}
}
}
pub fn stop_recording() -> Result<(), ()> {
// ensure microphone is initialized & recording is in process
if !RECORDER.get().is_none() && IS_RECORDING.load(Ordering::SeqCst) {
// stop recording
match RECORDER.get().unwrap().stop() {
Ok(_) => {
info!("STOP recording from microphone ...");
// change recording state
IS_RECORDING.store(false, Ordering::SeqCst);
// success
return Ok(());
}
Err(msg) => {
error!("Failed to stop audio recording!");
// fail
return Err(());
}
}
}
Ok(()) // if already stopped or not yet initialized
}
pub fn list_audio_devices() -> Vec<String> {
let audio_devices = PvRecorderBuilder::default().get_available_devices();
match audio_devices {
Ok(audio_devices) => audio_devices,
Err(err) => panic!("Failed to get audio devices: {}", err),
}
}
pub fn get_audio_device_name(idx: i32) -> String {
let audio_devices = list_audio_devices();
let mut first_device: String = String::new();
for (_idx, device) in audio_devices.iter().enumerate() {
if idx as usize == _idx {
return device.to_string();
}
if _idx == 0 {
first_device = device.to_string()
}
}
// return first device as default, if none were matched
first_device
}

36
crates/jarvis-core/src/stt.rs Normal file
View file

@ -0,0 +1,36 @@
mod vosk;
use crate::config;
use once_cell::sync::OnceCell;
use crate::config::structs::SpeechToTextEngine;
static STT_TYPE: OnceCell<SpeechToTextEngine> = OnceCell::new();
pub fn init() -> Result<(), ()> {
if !STT_TYPE.get().is_none() {
return Ok(());
} // already initialized
// set default stt type
// @TODO. Make it configurable?
STT_TYPE.set(config::DEFAULT_SPEECH_TO_TEXT_ENGINE).unwrap();
// load given recorder
match STT_TYPE.get().unwrap() {
SpeechToTextEngine::Vosk => {
// Init Vosk
info!("Initializing Vosk STT backend.");
vosk::init_vosk();
info!("STT backend initialized.");
}
}
Ok(())
}
pub fn recognize(data: &[i16], partial: bool) -> Option<String> {
match STT_TYPE.get().unwrap() {
SpeechToTextEngine::Vosk => vosk::recognize(data, partial),
}
}

92
crates/jarvis-core/src/stt/vosk.rs Normal file
View file

@ -0,0 +1,92 @@
use once_cell::sync::OnceCell;
use vosk::{DecodingState, Model, Recognizer};
use std::sync::Mutex;
use crate::config::VOSK_MODEL_PATH;
static MODEL: OnceCell<Model> = OnceCell::new();
static RECOGNIZER: OnceCell<Mutex<Recognizer>> = OnceCell::new();
pub fn init_vosk() {
if !RECOGNIZER.get().is_none() {
return;
} // already initialized
let model = Model::new(VOSK_MODEL_PATH).unwrap();
let mut recognizer = Recognizer::new(&model, 16000.0).unwrap();
recognizer.set_max_alternatives(10);
recognizer.set_words(true);
recognizer.set_partial_words(true);
MODEL.set(model);
RECOGNIZER.set(Mutex::new(recognizer));
}
pub fn recognize(data: &[i16], include_partial: bool) -> Option<String> {
let state = RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.accept_waveform(data);
match state {
Ok(ds) => {
match ds {
DecodingState::Running => {
if include_partial {
Some(
RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.partial_result()
.partial
.into(),
)
} else {
None
}
}
DecodingState::Finalized => {
// Result will always be multiple because we called set_max_alternatives
Some(
RECOGNIZER
.get()
.unwrap()
.lock()
.unwrap()
.result()
.multiple()
.unwrap()
.alternatives
.first()
.unwrap()
.text
.into(),
)
}
DecodingState::Failed => None,
}
},
Err(err) => {
error!("Vosk accept waveform error.\nError details: {}", err);
None
}
}
}
// pub fn stereo_to_mono(input_data: &[i16]) -> Vec<i16> {
// let mut result = Vec::with_capacity(input_data.len() / 2);
// result.extend(
// input_data
// .chunks_exact(2)
// .map(|chunk| chunk[0] / 2 + chunk[1] / 2),
// );
// result
// }