diff --git a/servo-media/src/audio/block.rs b/servo-media/src/audio/block.rs index 1eb78332..be3d1fbe 100644 --- a/servo-media/src/audio/block.rs +++ b/servo-media/src/audio/block.rs @@ -1,3 +1,5 @@ +use std::f32::consts::SQRT_2; +use audio::node::ChannelInterpretation; use audio::graph::PortIndex; use byte_slice_cast::*; use smallvec::SmallVec; @@ -126,15 +128,232 @@ impl Block { self.buffer.is_empty() } + pub fn data_chan_frame(&self, frame: usize, chan: u8) -> f32 { + let offset = if self.repeat { + 0 + } else { + chan as usize * FRAMES_PER_BLOCK_USIZE + }; + + self.buffer[frame + offset] + } + /// upmix/downmix the channels if necessary /// /// Currently only supports upmixing from 1 - pub fn mix(&mut self, channels: u8) { + pub fn mix(&mut self, channels: u8, interpretation: ChannelInterpretation) { + // If we're not changing the number of channels, we + // don't actually need to mix if self.channels == channels { return } - assert!(self.channels == 1 && channels > 1); + // Silent buffers stay silent + if self.is_silence() { + self.channels = channels; + return + } + + if interpretation == ChannelInterpretation::Discrete { + // discrete downmixes by truncation, upmixes by adding + // silent channels + + // If we're discrete, have a repeat, and are downmixing, + // just truncate by changing the channel value + if self.repeat && self.channels > channels { + self.channels = channels; + } else { + // otherwise resize the buffer, silent-filling when necessary + self.resize_silence(channels); + } + } else { + // For speakers, we have to do special things based on the + // interpretation of the channels for each kind of speakers + + // The layout of each speaker kind is: + // + // - Mono: [The mono channel] + // - Stereo: [L, R] + // - Quad: [L, R, SL, SR] + // - 5.1: [L, R, C, LFE, SL, SR] + + match (self.channels, channels) { + // Upmixing + // https://webaudio.github.io/web-audio-api/#UpMix-sub + + // mono + (1, 2) => { + // output.{L, R} = input + self.repeat(2); + } + (1, 4) => { + // output.{L, R} = input + self.repeat(2); + // output.{SL, SR} = 0 + self.resize_silence(4); + } + (1, 6) => { + let mut v = Vec::with_capacity(channels as usize * FRAMES_PER_BLOCK_USIZE); + // output.{L, R} = 0 + v.resize(2 * FRAMES_PER_BLOCK_USIZE, 0.); + // output.C = input + v.extend(&self.buffer); + self.buffer = v; + // output.{LFE, SL, SR} = 0 + self.resize_silence(6); + } + + // stereo + (2, 4) | (2, 6) => { + // output.{L, R} = input.{L, R} + // (5.1) output.{C, LFE} = 0 + // output.{SL, SR} = 0 + self.resize_silence(channels); + } + + // quad + (4, 6) => { + // we can avoid this and instead calculate offsets + // based off whether or not this is `repeat`, but + // a `repeat` quad block should be rare + self.explicit_repeat(); + + let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE); + // output.{L, R} = input.{L, R} + v.extend(&self.buffer[0 .. 2 * FRAMES_PER_BLOCK_USIZE]); + // output.{C, LFE} = 0 + v.resize(4 * FRAMES_PER_BLOCK_USIZE, 0.); + // output.{SL, R} = input.{SL, SR} + v.extend(&self.buffer[2 * FRAMES_PER_BLOCK_USIZE ..]); + self.buffer = v; + self.channels = channels; + } + + // Downmixing + // https://webaudio.github.io/web-audio-api/#down-mix + + // mono + (2, 1) => { + let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output = 0.5 * (input.L + input.R); + v[frame] = 0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 1)); + } + self.buffer = v; + self.channels = 1; + self.repeat = false; + } + (4, 1) => { + let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output = 0.5 * (input.L + input.R + input.SL + input.SR); + v[frame] = 0.25 * (self.data_chan_frame(frame, 0) + + self.data_chan_frame(frame, 1) + + self.data_chan_frame(frame, 2) + + self.data_chan_frame(frame, 3)); + } + self.buffer = v; + self.channels = 1; + self.repeat = false; + } + (6, 1) => { + let mut v = Vec::with_capacity(FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output = sqrt(0.5) * (input.L + input.R) + input.C + 0.5 * (input.SL + input.SR) + v[frame] = + // sqrt(0.5) * (input.L + input.R) + SQRT_2 * (self.data_chan_frame(frame, 0) + + self.data_chan_frame(frame, 1)) + + // input.C + self.data_chan_frame(frame, 2) + + // (ignore LFE) + // + 0 * self.buffer[frame + 3 * FRAMES_PER_BLOCK_USIZE] + // 0.5 * (input.SL + input.SR) + 0.5 * (self.data_chan_frame(frame, 4) + + self.data_chan_frame(frame, 5)); + } + self.buffer = v; + self.channels = 1; + self.repeat = false; + } + + // stereo + (4, 2) => { + let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output.L = 0.5 * (input.L + input.SL) + v[frame] = + 0.5 * (self.data_chan_frame(frame, 0) + self.data_chan_frame(frame, 2)); + // output.R = 0.5 * (input.R + input.SR) + v[frame + FRAMES_PER_BLOCK_USIZE] = + 0.5 * (self.data_chan_frame(frame, 1) + self.data_chan_frame(frame, 3)); + } + self.buffer = v; + self.channels = 2; + self.repeat = false; + } + (6, 2) => { + let mut v = Vec::with_capacity(2 * FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output.L = L + sqrt(0.5) * (input.C + input.SL) + v[frame] = + self.data_chan_frame(frame, 0) + + SQRT_2 * (self.data_chan_frame(frame, 2) + self.data_chan_frame(frame, 4)); + // output.R = R + sqrt(0.5) * (input.C + input.SR) + v[frame + FRAMES_PER_BLOCK_USIZE] = + self.data_chan_frame(frame, 1) + + SQRT_2 * (self.data_chan_frame(frame, 2) + self.data_chan_frame(frame, 5)); + } + self.buffer = v; + self.channels = 2; + self.repeat = false; + } + + // quad + (6, 4) => { + let mut v = Vec::with_capacity(6 * FRAMES_PER_BLOCK_USIZE); + for frame in 0..FRAMES_PER_BLOCK_USIZE { + // output.L = L + sqrt(0.5) * input.C + v[frame] = + self.data_chan_frame(frame, 0) + + SQRT_2 * self.data_chan_frame(frame, 2); + // output.R = R + sqrt(0.5) * input.C + v[frame + FRAMES_PER_BLOCK_USIZE] = + self.data_chan_frame(frame, 1) + + SQRT_2 * self.data_chan_frame(frame, 2); + // output.SL = input.SL + v[frame + 2 * FRAMES_PER_BLOCK_USIZE] = + self.data_chan_frame(frame, 4); + // output.SR = input.SR + v[frame + 3 * FRAMES_PER_BLOCK_USIZE] = + self.data_chan_frame(frame, 5); + } + self.buffer = v; + self.channels = 4; + self.repeat = false; + } + + // If it's not a known kind of speaker configuration, treat as + // discrete + _ => { + self.mix(channels, ChannelInterpretation::Discrete); + } + } + debug_assert!(self.channels == channels); + } + } + + /// Resize to add or remove channels, fill extra channels with silence + fn resize_silence(&mut self, channels: u8) { + self.explicit_repeat(); + self.buffer.resize(FRAMES_PER_BLOCK_USIZE * channels as usize, 0.); + self.channels = channels; + } + + /// Take a single-channel block and repeat the + /// channel + pub fn repeat(&mut self, channels: u8) { + debug_assert!(self.channels == 1); self.channels = channels; if !self.is_silence() { self.repeat = true; @@ -156,7 +375,6 @@ impl Block { } } - /// An iterator over frames in a block pub struct FrameIterator<'a> { frame: Tick, diff --git a/servo-media/src/audio/buffer_source_node.rs b/servo-media/src/audio/buffer_source_node.rs index e138ea78..cda6e9a7 100644 --- a/servo-media/src/audio/buffer_source_node.rs +++ b/servo-media/src/audio/buffer_source_node.rs @@ -1,3 +1,4 @@ +use audio::node::ChannelCountMode; use audio::block::{Chunk, Tick, FRAMES_PER_BLOCK}; use audio::node::{AudioNodeEngine, BlockInfo}; use audio::param::Param; @@ -107,6 +108,10 @@ impl AudioNodeEngine for AudioBufferSourceNode { 0 } + fn channel_count_mode(&self) -> ChannelCountMode { + ChannelCountMode::Max + } + fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { debug_assert!(inputs.len() == 0); diff --git a/servo-media/src/audio/channel_node.rs b/servo-media/src/audio/channel_node.rs index dbefc32c..bd175ebb 100644 --- a/servo-media/src/audio/channel_node.rs +++ b/servo-media/src/audio/channel_node.rs @@ -1,3 +1,4 @@ +use audio::node::ChannelCountMode; use audio::block::FRAMES_PER_BLOCK_USIZE; use audio::node::AudioNodeEngine; use audio::block::{Block, Chunk}; @@ -24,7 +25,7 @@ impl AudioNodeEngine for ChannelMergerNode { debug_assert!(inputs.len() == self.channels as usize); let mut block = Block::default(); - block.mix(self.channels); + block.repeat(self.channels); block.explicit_repeat(); for (i, channel) in block.data_mut().chunks_mut(FRAMES_PER_BLOCK_USIZE).enumerate() { @@ -39,4 +40,9 @@ impl AudioNodeEngine for ChannelMergerNode { fn input_count(&self) -> u32 { self.channels as u32 } + + fn channel_count_mode(&self) -> ChannelCountMode { + ChannelCountMode::Explicit + } + } diff --git a/servo-media/src/audio/destination_node.rs b/servo-media/src/audio/destination_node.rs index 340da8f8..6a1fa981 100644 --- a/servo-media/src/audio/destination_node.rs +++ b/servo-media/src/audio/destination_node.rs @@ -1,3 +1,4 @@ +use audio::node::ChannelCountMode; use audio::node::{AudioNodeEngine, BlockInfo}; use audio::block::Chunk; @@ -28,4 +29,8 @@ impl AudioNodeEngine for DestinationNode { // gst_audio::AudioInfo::new 2 } + + fn channel_count_mode(&self) -> ChannelCountMode { + ChannelCountMode::Explicit + } } diff --git a/servo-media/src/audio/gain_node.rs b/servo-media/src/audio/gain_node.rs index 8f866333..dbec327e 100644 --- a/servo-media/src/audio/gain_node.rs +++ b/servo-media/src/audio/gain_node.rs @@ -1,3 +1,4 @@ +use audio::node::ChannelCountMode; use audio::block::Chunk; use audio::block::Tick; use audio::node::AudioNodeEngine; @@ -62,5 +63,9 @@ impl AudioNodeEngine for GainNode { inputs } + fn channel_count_mode(&self) -> ChannelCountMode { + ChannelCountMode::Max + } + make_message_handler!(GainNode); } diff --git a/servo-media/src/audio/graph.rs b/servo-media/src/audio/graph.rs index 15584dc4..f84c411b 100644 --- a/servo-media/src/audio/graph.rs +++ b/servo-media/src/audio/graph.rs @@ -1,13 +1,13 @@ use audio::block::{Block, Chunk}; use audio::destination_node::DestinationNode; -use audio::node::AudioNodeEngine; -use audio::node::BlockInfo; +use audio::node::{AudioNodeEngine, BlockInfo, ChannelCountMode}; use petgraph::Direction; use petgraph::graph::DefaultIx; use petgraph::stable_graph::NodeIndex; use petgraph::stable_graph::StableGraph; use petgraph::visit::{DfsPostOrder, EdgeRef}; use std::cell::{Ref, RefCell, RefMut}; +use std::cmp; #[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] /// A unique identifier for nodes in the graph. Stable @@ -138,6 +138,12 @@ impl AudioGraph { chunk .blocks .resize(curr.input_count() as usize, Default::default()); + + let mut max = 0; // max channel count + let mode = curr.channel_count_mode(); + let count = curr.channel_count(); + let interpretation = curr.channel_interpretation(); + // all edges from this node point to its dependencies for edge in self.graph.edges(ix) { let edge = edge.weight(); @@ -148,12 +154,27 @@ impl AudioGraph { .borrow_mut() .take() .expect("Cache should have been filled from traversal"); - block.mix(curr.channel_count()); + if mode == ChannelCountMode::Explicit { + block.mix(count, interpretation); + } else { + max = cmp::max(max, block.chan_count()); + } chunk[edge.input_idx] = block; } + + if mode != ChannelCountMode::Explicit { + if mode == ChannelCountMode::ClampedMax { + max = cmp::min(max, count); + } + + for block in &mut chunk.blocks { + block.mix(max, interpretation); + } + } } + // actually run the node engine let mut out = curr.process(chunk, info); diff --git a/servo-media/src/audio/node.rs b/servo-media/src/audio/node.rs index 2d5c523f..ff8b63fb 100644 --- a/servo-media/src/audio/node.rs +++ b/servo-media/src/audio/node.rs @@ -27,6 +27,20 @@ pub enum AudioNodeType { WaveShaperNode, } +#[derive(Copy, Clone, PartialEq, Eq)] +pub enum ChannelCountMode { + Max, + ClampedMax, + Explicit +} + + +#[derive(Copy, Clone, PartialEq, Eq)] +pub enum ChannelInterpretation { + Discrete, + Speakers +} + #[derive(Copy, Clone)] pub struct BlockInfo { pub sample_rate: f32, @@ -60,6 +74,14 @@ pub trait AudioNodeEngine: Send { 1 } + fn channel_count_mode(&self) -> ChannelCountMode { + ChannelCountMode::Max + } + + fn channel_interpretation(&self) -> ChannelInterpretation { + ChannelInterpretation::Speakers + } + /// If we're the destination node, extract the contained data fn destination_data(&mut self) -> Option { None diff --git a/servo-media/src/audio/oscillator_node.rs b/servo-media/src/audio/oscillator_node.rs index 42a75089..f6080f88 100644 --- a/servo-media/src/audio/oscillator_node.rs +++ b/servo-media/src/audio/oscillator_node.rs @@ -1,3 +1,4 @@ +use audio::node::ChannelCountMode; use audio::block::{Chunk, Tick}; use audio::node::{AudioNodeEngine, BlockInfo}; use audio::param::{Param, UserAutomationEvent}; @@ -141,5 +142,9 @@ impl AudioNodeEngine for OscillatorNode { 0 } + fn channel_count_mode(&self) -> ChannelCountMode { + ChannelCountMode::Max + } + make_message_handler!(OscillatorNode); } diff --git a/servo-media/src/backends/gstreamer/audio_sink.rs b/servo-media/src/backends/gstreamer/audio_sink.rs index 395fbbe5..31b2bbb8 100644 --- a/servo-media/src/backends/gstreamer/audio_sink.rs +++ b/servo-media/src/backends/gstreamer/audio_sink.rs @@ -124,7 +124,7 @@ impl AudioSink for GStreamerAudioSink { // sometimes nothing reaches the output if chunk.len() == 0 { chunk.blocks.push(Default::default()); - chunk.blocks[0].mix(channels as u8); + chunk.blocks[0].repeat(channels as u8); } debug_assert!(chunk.len() == 1); let mut data = chunk.blocks[0].interleave();