diff --git a/Cargo.lock b/Cargo.lock index 649a047..be1b99a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,7 +105,7 @@ dependencies = [ "cpal", "env_logger", "log", - "rustfft", + "realfft", "textplots", ] @@ -772,6 +772,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "realfft" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390252372b7f2aac8360fc5e72eba10136b166d6faeed97e6d0c8324eb99b2b1" +dependencies = [ + "rustfft", +] + [[package]] name = "regex" version = "1.11.1" diff --git a/Cargo.toml b/Cargo.toml index f3caad9..ebaf86a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,6 @@ log = "0.4" color-eyre = "0.6" cpal = { version = "0.15", features = ["jack"] } -rustfft = "6.2" +realfft = "3.4" textplots = "0.8" diff --git a/src/main.rs b/src/main.rs index 6ccc695..b8879b0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,16 +1,16 @@ -use std::{sync::mpsc, thread::sleep, time::Duration}; +use std::{cmp::Ordering, sync::mpsc, thread::sleep, time::Duration}; -use color_eyre::eyre::bail; +use color_eyre::eyre::{bail, Result}; use cpal::{ traits::{DeviceTrait, HostTrait, StreamTrait}, BufferSize, HostId, SupportedBufferSize, }; use log::{debug, error, info, trace}; -use rustfft::{ - num_complex::{Complex, ComplexFloat}, - FftPlanner, +use realfft::{ + num_complex::{Complex, Complex32, ComplexFloat}, + RealFftPlanner, RealToComplex, }; -use textplots::{Chart, Plot, Shape}; +use textplots::{Chart, LabelBuilder, LabelFormat, Plot, Shape}; fn main() -> color_eyre::Result<()> { env_logger::init(); @@ -44,27 +44,27 @@ fn main() -> color_eyre::Result<()> { config.channels = 1; debug!("Using config {config:?}"); - let mut planner = FftPlanner::::new(); + let mut planner = RealFftPlanner::::new(); let fft = planner.plan_fft_forward(buffer_size as usize); - let mut fft_data = vec![Complex::new(0.0, 0.0); buffer_size as usize]; - let mut fft_scratch = vec![Complex::new(0.0, 0.0); buffer_size as usize]; + let mut fft_input = fft.make_input_vec(); + let mut fft_output = fft.make_output_vec(); let (sender, receiver) = mpsc::channel(); let stream = device.build_input_stream( &config, - move |data: &[f32], _| { - for i in 0..data.len() { - fft_data[i] = Complex::new(data[i], 0.0); - } - let mut fft_output = vec![Complex::new(0.0, 0.0); data.len()]; - fft.process_outofplace_with_scratch( - &mut fft_data[..], - &mut fft_output[..], - &mut fft_scratch[..], - ); - if let Err(e) = sender.send(fft_output) { - error!("Unable to send fft output: {e}"); + move |data: &[f32], _| match AudioSyncPacket::from_sample( + data, + fft.as_ref(), + &mut fft_input, + &mut fft_output, + config.sample_rate.0, + ) { + Ok(packet) => { + if let Err(e) = sender.send(packet) { + error!("Unable to send packet: {e}"); + } } + Err(e) => error!("Unable to process audio data: {e}"), }, |err| { error!("input stream error: {err}"); @@ -74,15 +74,149 @@ fn main() -> color_eyre::Result<()> { stream.play()?; + let mut previous_sample_smth = 0.0; loop { - let data = receiver.recv()?; + let mut data = receiver.recv()?; + data.sample_smth = 0.8f32.mul_add(previous_sample_smth, 0.2 * data.sample_raw); + previous_sample_smth = data.sample_smth; + let plot_data: Vec<_> = data - .into_iter() + .fft_result + .iter() .enumerate() - .map(|(i, datum)| ((i as f32).log10(), datum.abs() / 1024.0.sqrt())) + .map(|(i, datum)| (i as f32, f32::from(*datum))) .collect(); - Chart::new_with_y_range(200, 50, 0.0, 1024.0.log10(), 0.0, 2.0) + Chart::new_with_y_range(250, 50, 0.0, 16.0, 0.0, 255.0) .lineplot(&Shape::Bars(&plot_data[..])) .display(); } } + +/// new "V2" audiosync struct - 44 Bytes +#[repr(C, packed(1))] +struct AudioSyncPacket { + /// "00002" for protocol version 2 (includes '\0' for c-style string termination) + header: [u8; 6], + /// Optional sound pressure as fixed point (8bit integer, 8bit fractional) + preasure: [u8; 2], + /// either "sample_raw" or "raw_sample_agc" depending on sound_agc setting + sample_raw: f32, + /// either "sample_avg" or "sample_agc" depending on sound_agc setting + sample_smth: f32, + /// 0 = no peak, >= 1 = peak detected, In future, this will also provide peak Magnitude + sample_peak: u8, + /// Optional rolling counter to track duplicate / out of order packets + frame_counter: u8, + /// 16 GEQ channels, each channel has one byte + fft_result: [u8; 16], + /// Optional number of zero crossings seen in 23ms + zero_crossing_count: u16, + /// Largest FFT result (raw value, can go up to 4096) + fft_magnitude: f32, + /// frequency (Hz) of largest FFT result + fft_major_peak: f32, +} + +const _: () = assert!(size_of::() == 44); + +impl AudioSyncPacket { + /// Create an `AudioSyncPacket` with only required information + pub const fn new( + sample_raw: f32, + sample_smth: f32, + sample_peak: u8, + fft_result: [u8; 16], + fft_magnitude: f32, + fft_major_peak: f32, + ) -> Self { + Self { + header: *b"00002\0", + preasure: [0, 0], + sample_raw, + sample_smth, + sample_peak, + frame_counter: 0, + fft_result, + zero_crossing_count: 0, + fft_magnitude, + fft_major_peak, + } + } + + pub fn from_sample( + data: &[f32], + fft: &dyn RealToComplex, + fft_input: &mut [f32], + fft_output: &mut [Complex32], + sample_rate: u32, + ) -> Result { + // take the absolute value of the audio data + let abs_data: Vec<_> = data.iter().map(|sample| sample.abs()).collect(); + + // calculate peak (raw level and peak level) + let raw_level = abs_data.iter().sum::() / data.len() as f32; + let peak_level = abs_data + .iter() + .max_by(|lhs, rhs| lhs.partial_cmp(rhs).unwrap_or(Ordering::Equal)) + .copied() + .unwrap_or(0.0); + // scale peak level to u8 + let peak_level = (peak_level * f32::from(u8::MAX)) as u8; + + // calculate fft + fft_input[..].copy_from_slice(data); + fft.process(fft_input, fft_output)?; + let real_fft_output: Vec<_> = fft_output.iter().map(|value| value.abs()).collect(); + + // find dominant frequency + let (peak_index, peak) = real_fft_output + .iter() + .enumerate() + .max_by(|(_, lhs), (_, rhs)| lhs.partial_cmp(rhs).unwrap_or(Ordering::Equal)) + .map(|(i, value)| (i, *value)) + .unwrap_or_default(); + let peak_frequency = (peak_index * sample_rate as usize) as f32 / data.len() as f32; + + // select 16 [frequency bins](https://github.com/MoonModules/WLED/blob/fc173b3bc00694e59b653ca230133052b5476c05/usermods/audioreactive/audio_reactive.h#L733-L760) + let bins = [ + 86, 129, 216, 301, 430, 560, 818, 1120, 1421, 1895, 2412, 3015, 3704, 4479, 7106, 9259, + ] + .map(|frequency| frequency * data.len() / sample_rate as usize); + let mut fft_values = [0.0; 16]; + fft_values[0] = real_fft_output[1..=bins[0]].iter().sum::() / (bins[0]) as f32; + for i in 1..fft_values.len() { + fft_values[i] = real_fft_output[bins[i - 1] + 1..=bins[i]] + .iter() + .sum::() + / (bins[i] - bins[i - 1] + 1) as f32; + } + fft_values[14] *= 0.88; + fft_values[15] *= 0.7; + + // scale the fft values to u8 + let mut fft_values_u8 = [0u8; 16]; + let mut fft_max = fft_values + .iter() + .max_by(|lhs, rhs| lhs.partial_cmp(rhs).unwrap_or(Ordering::Equal)) + .copied() + .unwrap_or(1.0); + if fft_max == 0.0 { + fft_max = 1.0; + } + for i in 0..fft_values.len() { + fft_values_u8[i] = (fft_values[i] * f32::from(u8::MAX) / fft_max) as u8; + } + + // calculate fft magnitude sum + let fft_magnitude_sum = real_fft_output.iter().sum::(); + + Ok(Self::new( + raw_level, + 0.0, + peak_level, + fft_values_u8, + fft_magnitude_sum, + peak_frequency, + )) + } +}