diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 00000000..d331d77f --- /dev/null +++ b/.cargo/config @@ -0,0 +1,2 @@ +[alias] +ex = "run -p examples --bin" \ No newline at end of file diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e8cdcded..500122ba 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -30,4 +30,8 @@ path = "play_noise.rs" [[bin]] name = "channels" -path = "channels.rs" \ No newline at end of file +path = "channels.rs" + +[[bin]] +name = "channelsum" +path = "channelsum.rs" \ No newline at end of file diff --git a/examples/channelsum.rs b/examples/channelsum.rs new file mode 100644 index 00000000..48eb6861 --- /dev/null +++ b/examples/channelsum.rs @@ -0,0 +1,58 @@ +extern crate servo_media; + +use servo_media::audio::channel_node::ChannelNodeOptions; +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{AudioNodeMessage, AudioNodeType, AudioScheduledSourceNodeMessage}; +use servo_media::ServoMedia; +use std::sync::Arc; +use std::{thread, time}; + +fn run_example(servo_media: Arc) { + let context = servo_media.create_audio_context(Default::default()); + let mut options = Default::default(); + let osc = context.create_node(AudioNodeType::OscillatorNode(options)); + options.freq = 213.; + let osc2 = context.create_node(AudioNodeType::OscillatorNode(options)); + options.freq = 100.; + let osc3 = context.create_node(AudioNodeType::OscillatorNode(options)); + let mut options = GainNodeOptions::default(); + options.gain = 0.7; + let gain = context.create_node(AudioNodeType::GainNode(options)); + + let options = ChannelNodeOptions { channels: 2 }; + let merger = context.create_node(AudioNodeType::ChannelMergerNode(options)); + + let dest = context.dest_node(); + context.connect_ports(osc.output(0), merger.input(0)); + context.connect_ports(osc2.output(0), merger.input(1)); + context.connect_ports(merger.output(0), gain.input(0)); + context.connect_ports(osc3.output(0), gain.input(0)); + context.connect_ports(gain.output(0), dest.input(0)); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc2, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc3, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + let _ = context.resume(); + + thread::sleep(time::Duration::from_millis(2000)); + context.message_node(dest, AudioNodeMessage::SetChannelCount(1)); + thread::sleep(time::Duration::from_millis(2000)); + let _ = context.close(); + +} + +fn main() { + if let Ok(servo_media) = ServoMedia::get() { + run_example(servo_media); + } else { + unreachable!(); + } +} diff --git a/servo-media/src/audio/block.rs b/servo-media/src/audio/block.rs index 1de3873c..fcc2288a 100644 --- a/servo-media/src/audio/block.rs +++ b/servo-media/src/audio/block.rs @@ -89,6 +89,26 @@ impl Block { self.data_mut().as_mut_byte_slice().expect("casting failed") } + /// Zero-gain sum with another buffer + /// + /// Used after mixing multiple inputs to a single port + pub fn sum(mut self, mut other: Self) -> Self { + if self.is_silence() { + other + } else { + debug_assert!(self.channels == other.channels); + if self.repeat ^ other.repeat { + self.explicit_repeat(); + other.explicit_repeat(); + } + debug_assert!(self.buffer.len() == other.buffer.len()); + for (a, b) in self.buffer.iter_mut().zip(other.buffer.iter()) { + *a += b + } + self + } + } + /// If this is in "silence" mode without a buffer, allocate a silent buffer pub fn explicit_silence(&mut self) { if self.buffer.is_empty() { @@ -111,6 +131,7 @@ impl Block { } self.buffer = new; + self.repeat = false; } else if self.is_silence() { self.buffer.resize(FRAMES_PER_BLOCK_USIZE * self.channels as usize, 0.); } diff --git a/servo-media/src/audio/graph.rs b/servo-media/src/audio/graph.rs index 3bcdb602..8ba9aa47 100644 --- a/servo-media/src/audio/graph.rs +++ b/servo-media/src/audio/graph.rs @@ -1,3 +1,4 @@ +use smallvec::SmallVec; use audio::block::{Block, Chunk}; use audio::destination_node::DestinationNode; use audio::node::{AudioNodeEngine, BlockInfo, ChannelCountMode}; @@ -5,7 +6,7 @@ use petgraph::Direction; use petgraph::graph::DefaultIx; use petgraph::stable_graph::NodeIndex; use petgraph::stable_graph::StableGraph; -use petgraph::visit::{DfsPostOrder, EdgeRef}; +use petgraph::visit::{DfsPostOrder, EdgeRef, Reversed}; use std::cell::{Ref, RefCell, RefMut}; use std::cmp; @@ -63,11 +64,41 @@ pub struct Node { node: RefCell>, } -/// Edges go *to* the output port from the input port, +/// An edge in the graph /// -/// The edge direction is the *reverse* of the direction of sound -/// since we need to do a postorder DFS traversal starting at the output +/// This connects one or more pair of ports between two +/// nodes, each connection represented by a `Connection`. +/// WebAudio allows for multiple connections to/from the same port +/// however it does not allow for duplicate connections between pairs +/// of ports pub struct Edge { + connections: SmallVec<[Connection; 1]> +} + +impl Edge { + /// Find if there are connections between two given ports, return the index + fn has_between(&self, + output_idx: PortIndex, + input_idx: PortIndex) -> bool { + self.connections.iter() + .find(|e| e.input_idx == input_idx && e.output_idx == output_idx) + .is_some() + } + + fn remove_by_output(&mut self, output_idx: PortIndex) { + self.connections.retain(|i| i.output_idx != output_idx) + } + + fn remove_by_pair(&mut self, + output_idx: PortIndex, + input_idx: PortIndex) { + self.connections + .retain(|i| i.output_idx != output_idx || i.input_idx != input_idx) + } +} + +/// A single connection between ports +struct Connection { /// The index of the port on the input node /// This is actually the /output/ of this edge input_idx: PortIndex, @@ -93,24 +124,22 @@ impl AudioGraph { /// Connect an output port to an input port /// - /// While conceptually the edge goes *from* the output port *to* the input port, - /// the internal implementation reverses the direction of the edge + /// The edge goes *from* the output port *to* the input port, connecting two nodes pub fn add_edge(&mut self, out: PortId, inp: PortId) { - // Output ports can only have a single edge associated with them. - // Remove all others - let old = self - .graph - .edges_directed(out.node().0, Direction::Incoming) - .find(|e| e.weight().input_idx == inp.1) - .map(|e| e.id()); - if let Some(old) = old { - self.graph.remove_edge(old); + let edge = self.graph.edges(out.node().0) + .find(|e| e.target() == inp.node().0) + .map(|e| e.id()); + if let Some(e) = edge { + // .find(|e| e.weight().has_between(out.1, inp.1)); + let w = self.graph.edge_weight_mut(e).expect("This edge is known to exist"); + if w.has_between(out.1, inp.1) { + return; + } + w.connections.push(Connection::new(inp.1, out.1)) + } else { + // add a new edge + self.graph.add_edge(out.node().0, inp.node().0, Edge::new(inp.1, out.1)); } - // add a new edge - // XXXManishearth it is actually possible for two nodes to have - // multiple edges between them between - // different ports. We should represent this somehow. - self.graph.add_edge(inp.node().0, out.node().0, Edge::new(inp.1, out.1)); } /// Disconnect all outgoing connections from a node @@ -118,7 +147,7 @@ impl AudioGraph { /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect pub fn disconnect_all(&mut self, node: NodeId) { let edges = self.graph - .edges_directed(node.0, Direction::Incoming) + .edges(node.0) .map(|e| e.id()) .collect::>(); for edge in edges { @@ -126,19 +155,18 @@ impl AudioGraph { } } - /// Disconnect all outgoing connections from a node's output - /// - /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-output + // /// Disconnect all outgoing connections from a node's output + // /// + // /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-output pub fn disconnect_output(&mut self, out: PortId) { - // XXXManishearth we don't support multiple connections through - // a single output yet - let edge = self - .graph - .edges_directed(out.node().0, Direction::Incoming) - .find(|e| e.weight().output_idx == out.1) - .map(|e| e.id()); - if let Some(edge) = edge { - self.graph.remove_edge(edge); + let candidates: Vec<_> = self.graph.edges(out.node().0) + .map(|e| (e.id(), e.target())).collect(); + for (edge, to) in candidates { + let mut e = self.graph.remove_edge(edge).expect("Edge index is known to exist"); + e.remove_by_output(out.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, to, e); + } } } @@ -146,13 +174,12 @@ impl AudioGraph { /// /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode pub fn disconnect_between(&mut self, from: NodeId, to: NodeId) { - let edges = self.graph - .edges(to.0) - .filter(|e| e.target() == from.0) - .map(|e| e.id()) - .collect::>(); - for edge in edges { - self.graph.remove_edge(edge); + let edge = self.graph + .edges(from.0) + .find(|e| e.target() == to.0) + .map(|e| e.id()); + if let Some(i) = edge { + self.graph.remove_edge(i); } } @@ -162,27 +189,33 @@ impl AudioGraph { pub fn disconnect_output_between(&mut self, out: PortId, to: NodeId) { let edge = self .graph - .edges_directed(out.node().0, Direction::Incoming) - .find(|e| e.weight().output_idx == out.1 && e.source() == to.0) + .edges(out.node().0) + .find(|e| e.target() == to.0) .map(|e| e.id()); if let Some(edge) = edge { - self.graph.remove_edge(edge); + let mut e = self.graph.remove_edge(edge).expect("Edge index is known to exist"); + e.remove_by_output(out.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, to.0, e); + } } } - /// Disconnect all outgoing connections from a node's output to another node's input - /// - /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode-output-input + // /// Disconnect all outgoing connections from a node's output to another node's input + // /// + // /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode-output-input pub fn disconnect_output_between_to(&mut self, out: PortId, inp: PortId) { let edge = self .graph - .edges_directed(out.node().0, Direction::Incoming) - .find(|e| e.weight().output_idx == out.1 && - e.source() == inp.node().0 && - e.weight().input_idx == inp.1) + .edges(out.node().0) + .find(|e| e.target() == inp.node().0) .map(|e| e.id()); if let Some(edge) = edge { - self.graph.remove_edge(edge); + let mut e = self.graph.remove_edge(edge).expect("Edge index is known to exist"); + e.remove_by_pair(out.1, inp.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, inp.node().0, e); + } } } @@ -200,49 +233,83 @@ impl AudioGraph { // children's output // // This will only visit each node once - let mut visit = DfsPostOrder::new(&self.graph, self.dest_id.0); - while let Some(ix) = visit.next(&self.graph) { + let reversed = Reversed(&self.graph); + let mut visit = DfsPostOrder::new(reversed, self.dest_id.0); + + let mut blocks: SmallVec<[SmallVec<[Block; 1]>; 1]> = SmallVec::new(); + let mut output_counts: SmallVec<[u32; 1]> = SmallVec::new(); + + while let Some(ix) = visit.next(reversed) { let mut curr = self.graph[ix].node.borrow_mut(); + let mut chunk = Chunk::default(); + chunk + .blocks + .resize(curr.input_count() as usize, Default::default()); // if we have inputs, collect all the computed blocks // and construct a Chunk if curr.input_count() > 0 { - // set the chunk to the correct size - chunk - .blocks - .resize(curr.input_count() as usize, Default::default()); + // set up scratch space to store all the blocks + blocks.clear(); + blocks.resize(curr.input_count() as usize, Default::default()); - let mut max = 0; // max channel count let mode = curr.channel_count_mode(); let count = curr.channel_count(); let interpretation = curr.channel_interpretation(); - // all edges from this node point to its dependencies - for edge in self.graph.edges(ix) { + // all edges to this node are from its dependencies + for edge in self.graph.edges_directed(ix, Direction::Incoming) { let edge = edge.weight(); - // XXXManishearth we can have multiple edges - // hitting the same input port, we should deal with that - let mut block = edge - .cache - .borrow_mut() - .take() - .expect("Cache should have been filled from traversal"); - if mode == ChannelCountMode::Explicit { - block.mix(count, interpretation); - } else { - max = cmp::max(max, block.chan_count()); + for connection in &edge.connections { + let block = connection + .cache + .borrow_mut() + .take() + .expect("Cache should have been filled from traversal"); + blocks[connection.input_idx.0 as usize].push(block); } - chunk[edge.input_idx] = block; } - if mode != ChannelCountMode::Explicit { - if mode == ChannelCountMode::ClampedMax { - max = cmp::min(max, count); - } - - for block in &mut chunk.blocks { - block.mix(max, interpretation); + for (i, mut blocks) in blocks.drain().enumerate() { + if blocks.len() == 0 { + if mode == ChannelCountMode::Explicit { + // It's silence, but mix it anyway + chunk.blocks[i].mix(count, interpretation); + } + } else if blocks.len() == 1 { + chunk.blocks[i] = blocks.pop().expect("`blocks` had length 1"); + match mode { + ChannelCountMode::Explicit => { + chunk.blocks[i].mix(count, interpretation); + } + ChannelCountMode::ClampedMax => { + if chunk.blocks[i].chan_count() > count { + chunk.blocks[i].mix(count, interpretation); + } + } + // It's one channel, it maxes itself + ChannelCountMode::Max => () + } + } else { + let mix_count = match mode { + ChannelCountMode::Explicit => count, + _ => { + let mut max = 0; // max channel count + for block in &blocks { + max = cmp::max(max, block.chan_count()); + } + if mode == ChannelCountMode::ClampedMax { + max = cmp::min(max, count); + } + max + } + }; + let block = blocks.into_iter().fold(Block::default(), |acc, mut block| { + block.mix(mix_count, interpretation); + acc.sum(block) + }); + chunk.blocks[i] = block; } } } @@ -256,11 +323,34 @@ impl AudioGraph { continue; } - // all the edges to this node come from nodes which depend on it, + // Count how many output connections fan out from each port + // This is so that we don't have to needlessly clone audio buffers + // + // If this is inefficient, we can instead maintain this data + // cached on the node + output_counts.clear(); + output_counts.resize(curr.output_count() as usize, 0); + for edge in self.graph.edges(ix) { + let edge = edge.weight(); + for conn in &edge.connections { + output_counts[conn.output_idx.0 as usize] += 1; + } + } + + // all the edges from this node go to nodes which depend on it, // i.e. the nodes it outputs to. Store the blocks for retrieval. - for edge in self.graph.edges_directed(ix, Direction::Incoming) { + for edge in self.graph.edges(ix) { let edge = edge.weight(); - *edge.cache.borrow_mut() = Some(out[edge.output_idx].take()); + for conn in &edge.connections { + output_counts[conn.output_idx.0 as usize] -= 1; + // if there are no consumers left after this, take the data + let block = if output_counts[conn.output_idx.0 as usize] == 0 { + out[conn.output_idx].take() + } else { + out[conn.output_idx].clone() + }; + *conn.cache.borrow_mut() = Some(block); + } } } @@ -292,10 +382,17 @@ impl Node { impl Edge { pub fn new(input_idx: PortIndex, output_idx: PortIndex) -> Self { Edge { + connections: SmallVec::from_buf([Connection::new(input_idx, output_idx)]) + } + } +} + +impl Connection { + pub fn new(input_idx: PortIndex, output_idx: PortIndex) -> Self { + Connection { input_idx, output_idx, cache: RefCell::new(None), } } } -