From b7c082279f413024107e082ae4700221ea7aecdc Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 26 Jun 2018 14:34:56 -0700 Subject: [PATCH 1/2] Support channels in AudioBuffer; clean up AudioBufferSourceNode --- examples/audio_decoder.rs | 2 +- examples/play_noise.rs | 7 +- servo-media/src/audio/block.rs | 20 ++++ servo-media/src/audio/buffer_source_node.rs | 119 +++++++++++++++----- 4 files changed, 116 insertions(+), 32 deletions(-) diff --git a/examples/audio_decoder.rs b/examples/audio_decoder.rs index e3ee8b52..ff805bfc 100644 --- a/examples/audio_decoder.rs +++ b/examples/audio_decoder.rs @@ -53,7 +53,7 @@ fn run_example(servo_media: Arc) { context.message_node( buffer_source, AudioNodeMessage::AudioBufferSourceNode(AudioBufferSourceNodeMessage::SetBuffer( - decoded_audio.lock().unwrap().to_vec(), + decoded_audio.lock().unwrap().to_vec().into(), )), ); let _ = context.resume(); diff --git a/examples/play_noise.rs b/examples/play_noise.rs index 0168ab40..75c7f7d0 100644 --- a/examples/play_noise.rs +++ b/examples/play_noise.rs @@ -12,9 +12,10 @@ fn run_example(servo_media: Arc) { let buffer_source = context.create_node(AudioNodeType::AudioBufferSourceNode(Default::default())); let dest = context.dest_node(); context.connect_ports(buffer_source.output(0), dest.input(0)); - let mut buffer = Vec::with_capacity(4096); + let mut buffers = vec![Vec::with_capacity(4096), Vec::with_capacity(4096)]; for _ in 0..4096 { - buffer.push(rand::random::()); + buffers[0].push(rand::random::()); + buffers[1].push(rand::random::()); } context.message_node( buffer_source, @@ -22,7 +23,7 @@ fn run_example(servo_media: Arc) { ); context.message_node( buffer_source, - AudioNodeMessage::AudioBufferSourceNode(AudioBufferSourceNodeMessage::SetBuffer(buffer)), + AudioNodeMessage::AudioBufferSourceNode(AudioBufferSourceNodeMessage::SetBuffer(buffers.into())), ); let _ = context.resume(); thread::sleep(time::Duration::from_millis(5000)); diff --git a/servo-media/src/audio/block.rs b/servo-media/src/audio/block.rs index be3d1fbe..254e91ae 100644 --- a/servo-media/src/audio/block.rs +++ b/servo-media/src/audio/block.rs @@ -72,6 +72,18 @@ impl Default for Block { } impl Block { + + /// Empty block with no channels, for pushing + /// new channels to. + /// + /// Must be used with push_chan + pub fn empty() -> Self { + Block { + channels: 0, + ..Default::default() + } + } + /// This provides the entire buffer as a mutable slice of u8 pub fn as_mut_byte_slice(&mut self) -> &mut [u8] { self.data_mut().as_mut_byte_slice().expect("casting failed") @@ -138,6 +150,14 @@ impl Block { self.buffer[frame + offset] } + pub fn push_chan(&mut self, data: &[f32]) { + assert!(!self.repeat); + assert!(!self.is_silence() || self.channels == 0); + assert!(data.len() == FRAMES_PER_BLOCK_USIZE); + self.buffer.extend(data); + self.channels += 1; + } + /// upmix/downmix the channels if necessary /// /// Currently only supports upmixing from 1 diff --git a/servo-media/src/audio/buffer_source_node.rs b/servo-media/src/audio/buffer_source_node.rs index cda6e9a7..ec200f49 100644 --- a/servo-media/src/audio/buffer_source_node.rs +++ b/servo-media/src/audio/buffer_source_node.rs @@ -1,5 +1,5 @@ use audio::node::ChannelCountMode; -use audio::block::{Chunk, Tick, FRAMES_PER_BLOCK}; +use audio::block::{Block, Chunk, Tick, FRAMES_PER_BLOCK}; use audio::node::{AudioNodeEngine, BlockInfo}; use audio::param::Param; @@ -7,7 +7,7 @@ use audio::param::Param; pub enum AudioBufferSourceNodeMessage { /// Set the data block holding the audio sample data to be played. // XXX handle channels - SetBuffer(Vec), + SetBuffer(AudioBuffer), /// Schedules a sound to playback at an exact time. Start(f64), /// Schedules a sound to stop playback at an exact time. @@ -17,7 +17,7 @@ pub enum AudioBufferSourceNodeMessage { /// This specifies options for constructing an AudioBufferSourceNode. pub struct AudioBufferSourceNodeOptions { /// The audio asset to be played. - pub buffer: Option>, + pub buffer: Option, /// The initial value for the detune AudioParam. pub detune: f32, /// The initial value for the loop_enabled attribute. @@ -51,7 +51,7 @@ impl Default for AudioBufferSourceNodeOptions { #[allow(dead_code)] pub struct AudioBufferSourceNode { /// A data block holding the audio sample data to be played. - buffer: Option>, + buffer: Option, /// AudioParam to modulate the speed at which is rendered the audio stream. detune: Param, /// Indicates if the region of audio data designated by loopStart and loopEnd @@ -115,40 +115,103 @@ impl AudioNodeEngine for AudioBufferSourceNode { fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { debug_assert!(inputs.len() == 0); - inputs.blocks.push(Default::default()); - - if self.buffer.is_none() { + let buffer = if let Some(buffer) = self.buffer.as_ref() { + buffer + } else { + inputs.blocks.push(Default::default()); return inputs; - } + }; - let buffer = self.buffer.as_ref().unwrap(); - - if self.playback_offset >= buffer.len() || self.should_play_at(info.frame) == (false, true) - { + if self.playback_offset >= buffer.len() || + self.should_play_at(info.frame) == (false, true) { + inputs.blocks.push(Default::default()); return inputs; } - { - let samples_to_copy = match self.stop_at { - Some(stop_at) => { - let ticks_to_stop = stop_at - info.frame; - (if ticks_to_stop > FRAMES_PER_BLOCK { - FRAMES_PER_BLOCK - } else { - ticks_to_stop - }).0 as usize + let samples_to_copy = match self.stop_at { + Some(stop_at) => { + let ticks_to_stop = stop_at - info.frame; + if ticks_to_stop > FRAMES_PER_BLOCK { + FRAMES_PER_BLOCK.0 as usize + } else { + ticks_to_stop.0 as usize } - None => FRAMES_PER_BLOCK.0 as usize, - }; - let data = inputs.blocks[0].data_mut(); - let (data, _) = data.split_at_mut(samples_to_copy); - let next_offset = self.playback_offset + samples_to_copy; - data.copy_from_slice(&buffer[self.playback_offset..next_offset]); - self.playback_offset = next_offset; + } + None => FRAMES_PER_BLOCK.0 as usize, + }; + + let next_offset = self.playback_offset + samples_to_copy; + + + if samples_to_copy == FRAMES_PER_BLOCK.0 as usize { + // copy entire chan + let mut block = Block::empty(); + for chan in 0..buffer.chans() { + block.push_chan(&buffer.buffers[chan as usize][self.playback_offset..next_offset]); + } + inputs.blocks.push(block) + } else { + // silent fill and copy + let mut block = Block::default(); + block.repeat(buffer.chans()); + block.explicit_repeat(); + for chan in 0..buffer.chans() { + let data = block.data_chan_mut(chan); + data.copy_from_slice(&buffer.buffers[chan as usize][self.playback_offset..next_offset]); + } + inputs.blocks.push(block) } + self.playback_offset = next_offset; inputs } make_message_handler!(AudioBufferSourceNode); } + +pub struct AudioBuffer { + /// Invariant: all buffers must be of the same length + pub buffers: Vec> +} + +impl AudioBuffer { + pub fn new(chan: u8, len: usize) -> Self { + assert!(chan > 0); + let mut buffers = Vec::with_capacity(chan as usize); + let single = vec![0.; len]; + buffers.resize(chan as usize, single); + AudioBuffer { buffers } + } + + pub fn from_buffers(buffers: Vec>) -> Self { + for buf in &buffers { + assert!(buf.len() == buffers[0].len()) + } + + Self { + buffers + } + } + + pub fn len(&self) -> usize { + self.buffers[0].len() + } + + pub fn chans(&self) -> u8 { + self.buffers.len() as u8 + } +} + +impl From> for AudioBuffer { + fn from(vec: Vec) -> Self { + Self { + buffers: vec![vec] + } + } +} + +impl From>> for AudioBuffer { + fn from(vec: Vec>) -> Self { + AudioBuffer::from_buffers(vec) + } +} From be8820847a268126a732c4aba11f41301354b0ac Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 26 Jun 2018 15:08:49 -0700 Subject: [PATCH 2/2] Add ChannelMergerNode --- servo-media/src/audio/block.rs | 18 ++++++++----- servo-media/src/audio/channel_node.rs | 37 ++++++++++++++++++++++++++ servo-media/src/audio/node.rs | 2 +- servo-media/src/audio/render_thread.rs | 3 ++- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/servo-media/src/audio/block.rs b/servo-media/src/audio/block.rs index 254e91ae..cc71040e 100644 --- a/servo-media/src/audio/block.rs +++ b/servo-media/src/audio/block.rs @@ -122,6 +122,16 @@ impl Block { &mut self.buffer[start..start + FRAMES_PER_BLOCK_USIZE] } + #[inline] + pub fn data_chan(&self, chan: u8) -> &[f32] { + let offset = if self.repeat { + 0 + } else { + chan as usize * FRAMES_PER_BLOCK_USIZE + }; + &self.buffer[offset..offset + FRAMES_PER_BLOCK_USIZE] + } + pub fn take(&mut self) -> Block { let mut new = Block::default(); new.channels = self.channels; @@ -141,13 +151,7 @@ impl Block { } pub fn data_chan_frame(&self, frame: usize, chan: u8) -> f32 { - let offset = if self.repeat { - 0 - } else { - chan as usize * FRAMES_PER_BLOCK_USIZE - }; - - self.buffer[frame + offset] + self.data_chan(chan)[frame] } pub fn push_chan(&mut self, data: &[f32]) { diff --git a/servo-media/src/audio/channel_node.rs b/servo-media/src/audio/channel_node.rs index bd175ebb..542a83df 100644 --- a/servo-media/src/audio/channel_node.rs +++ b/servo-media/src/audio/channel_node.rs @@ -37,6 +37,43 @@ impl AudioNodeEngine for ChannelMergerNode { inputs } + fn output_count(&self) -> u32 { + self.channels as u32 + } + + fn channel_count_mode(&self) -> ChannelCountMode { + ChannelCountMode::Explicit + } + +} + +pub struct ChannelSplitterNode { + channels: u8 +} + +impl ChannelSplitterNode { + pub fn new(params: ChannelNodeOptions) -> Self { + ChannelSplitterNode { + channels: params.channels + } + } +} + +impl AudioNodeEngine for ChannelSplitterNode { + fn process(&mut self, mut inputs: Chunk, _: &BlockInfo) -> Chunk { + debug_assert!(inputs.len() == 1); + + let original = inputs.blocks.pop().unwrap(); + + for chan in 0..original.chan_count() { + let mut block = Block::empty(); + block.push_chan(original.data_chan(chan)); + inputs.blocks.push(block); + } + + inputs + } + fn input_count(&self) -> u32 { self.channels as u32 } diff --git a/servo-media/src/audio/node.rs b/servo-media/src/audio/node.rs index ff8b63fb..5eba62a9 100644 --- a/servo-media/src/audio/node.rs +++ b/servo-media/src/audio/node.rs @@ -11,7 +11,7 @@ pub enum AudioNodeType { AudioBuffer, AudioBufferSourceNode(AudioBufferSourceNodeOptions), ChannelMergerNode(ChannelNodeOptions), - ChannelSplitterNode, + ChannelSplitterNode(ChannelNodeOptions), ConstantSourceNode, ConvolverNode, DelayNode, diff --git a/servo-media/src/audio/render_thread.rs b/servo-media/src/audio/render_thread.rs index de8af3c5..04921e6b 100644 --- a/servo-media/src/audio/render_thread.rs +++ b/servo-media/src/audio/render_thread.rs @@ -1,6 +1,6 @@ use audio::block::{Chunk, Tick, FRAMES_PER_BLOCK}; use audio::buffer_source_node::AudioBufferSourceNode; -use audio::channel_node::ChannelMergerNode; +use audio::channel_node::{ChannelMergerNode, ChannelSplitterNode}; use audio::context::{ProcessingState, StateChangeResult}; use audio::destination_node::DestinationNode; use audio::gain_node::GainNode; @@ -73,6 +73,7 @@ impl AudioRenderThread { AudioNodeType::GainNode(options) => Box::new(GainNode::new(options)), AudioNodeType::OscillatorNode(options) => Box::new(OscillatorNode::new(options)), AudioNodeType::ChannelMergerNode(options) => Box::new(ChannelMergerNode::new(options)), + AudioNodeType::ChannelSplitterNode(options) => Box::new(ChannelSplitterNode::new(options)), _ => unimplemented!(), }; self.graph.add_node(node)