From 0a2da13529fa5a6c63a7eccf2ce4a2a22c22231a Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 3 Jul 2018 14:05:18 -0700 Subject: [PATCH 1/5] Invert edge direction (part of #68) We currently have the edges going in the reverse direction as the audio, which is confusing. The reason we did this was so that we could use the DFSPostOrder iterator. This inverts their direction so that edges now go from source to sink, and uses a `Reversed` adaptor to fix it. --- servo-media/src/audio/graph.rs | 35 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/servo-media/src/audio/graph.rs b/servo-media/src/audio/graph.rs index 3bcdb602..6a377318 100644 --- a/servo-media/src/audio/graph.rs +++ b/servo-media/src/audio/graph.rs @@ -5,7 +5,7 @@ use petgraph::Direction; use petgraph::graph::DefaultIx; use petgraph::stable_graph::NodeIndex; use petgraph::stable_graph::StableGraph; -use petgraph::visit::{DfsPostOrder, EdgeRef}; +use petgraph::visit::{DfsPostOrder, EdgeRef, Reversed}; use std::cell::{Ref, RefCell, RefMut}; use std::cmp; @@ -65,8 +65,7 @@ pub struct Node { /// Edges go *to* the output port from the input port, /// -/// The edge direction is the *reverse* of the direction of sound -/// since we need to do a postorder DFS traversal starting at the output +/// The edge direction is the direction of sound pub struct Edge { /// The index of the port on the input node /// This is actually the /output/ of this edge @@ -93,14 +92,13 @@ impl AudioGraph { /// Connect an output port to an input port /// - /// While conceptually the edge goes *from* the output port *to* the input port, - /// the internal implementation reverses the direction of the edge + /// The edge goes *from* the output port *to* the input port, connecting two nodes pub fn add_edge(&mut self, out: PortId, inp: PortId) { // Output ports can only have a single edge associated with them. // Remove all others let old = self .graph - .edges_directed(out.node().0, Direction::Incoming) + .edges(out.node().0) .find(|e| e.weight().input_idx == inp.1) .map(|e| e.id()); if let Some(old) = old { @@ -110,7 +108,7 @@ impl AudioGraph { // XXXManishearth it is actually possible for two nodes to have // multiple edges between them between // different ports. We should represent this somehow. - self.graph.add_edge(inp.node().0, out.node().0, Edge::new(inp.1, out.1)); + self.graph.add_edge(out.node().0, inp.node().0, Edge::new(inp.1, out.1)); } /// Disconnect all outgoing connections from a node @@ -118,7 +116,7 @@ impl AudioGraph { /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect pub fn disconnect_all(&mut self, node: NodeId) { let edges = self.graph - .edges_directed(node.0, Direction::Incoming) + .edges(node.0) .map(|e| e.id()) .collect::>(); for edge in edges { @@ -134,7 +132,7 @@ impl AudioGraph { // a single output yet let edge = self .graph - .edges_directed(out.node().0, Direction::Incoming) + .edges(out.node().0) .find(|e| e.weight().output_idx == out.1) .map(|e| e.id()); if let Some(edge) = edge { @@ -147,7 +145,7 @@ impl AudioGraph { /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode pub fn disconnect_between(&mut self, from: NodeId, to: NodeId) { let edges = self.graph - .edges(to.0) + .edges_directed(to.0, Direction::Incoming) .filter(|e| e.target() == from.0) .map(|e| e.id()) .collect::>(); @@ -162,7 +160,7 @@ impl AudioGraph { pub fn disconnect_output_between(&mut self, out: PortId, to: NodeId) { let edge = self .graph - .edges_directed(out.node().0, Direction::Incoming) + .edges(out.node().0) .find(|e| e.weight().output_idx == out.1 && e.source() == to.0) .map(|e| e.id()); if let Some(edge) = edge { @@ -176,7 +174,7 @@ impl AudioGraph { pub fn disconnect_output_between_to(&mut self, out: PortId, inp: PortId) { let edge = self .graph - .edges_directed(out.node().0, Direction::Incoming) + .edges(out.node().0) .find(|e| e.weight().output_idx == out.1 && e.source() == inp.node().0 && e.weight().input_idx == inp.1) @@ -200,8 +198,9 @@ impl AudioGraph { // children's output // // This will only visit each node once - let mut visit = DfsPostOrder::new(&self.graph, self.dest_id.0); - while let Some(ix) = visit.next(&self.graph) { + let reversed = Reversed(&self.graph); + let mut visit = DfsPostOrder::new(reversed, self.dest_id.0); + while let Some(ix) = visit.next(reversed) { let mut curr = self.graph[ix].node.borrow_mut(); let mut chunk = Chunk::default(); // if we have inputs, collect all the computed blocks @@ -217,8 +216,8 @@ impl AudioGraph { let count = curr.channel_count(); let interpretation = curr.channel_interpretation(); - // all edges from this node point to its dependencies - for edge in self.graph.edges(ix) { + // all edges to this node are from its dependencies + for edge in self.graph.edges_directed(ix, Direction::Incoming) { let edge = edge.weight(); // XXXManishearth we can have multiple edges // hitting the same input port, we should deal with that @@ -256,9 +255,9 @@ impl AudioGraph { continue; } - // all the edges to this node come from nodes which depend on it, + // all the edges from this node go to nodes which depend on it, // i.e. the nodes it outputs to. Store the blocks for retrieval. - for edge in self.graph.edges_directed(ix, Direction::Incoming) { + for edge in self.graph.edges(ix) { let edge = edge.weight(); *edge.cache.borrow_mut() = Some(out[edge.output_idx].take()); } From 516e5b2f4ee7cb8154ec42f8e30e347c94c260dd Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 3 Jul 2018 14:09:16 -0700 Subject: [PATCH 2/5] Add a cargo ex alias for easily running examples --- .cargo/config | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .cargo/config diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 00000000..d331d77f --- /dev/null +++ b/.cargo/config @@ -0,0 +1,2 @@ +[alias] +ex = "run -p examples --bin" \ No newline at end of file From 69f9573e53c8a3928d377e2c3e39f6132423ffac Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 3 Jul 2018 16:49:48 -0700 Subject: [PATCH 3/5] Support fan-in, fan-out, multiple connections between nodes This adds support for ports to fan in and fan out, mixing on fan-in. This is done by making individual `Edge`s hold multiple `Connection`s, which enumerate different port pairs. Typically each edge will only have a single connection. fixes #68 --- servo-media/src/audio/block.rs | 20 +++ servo-media/src/audio/graph.rs | 238 +++++++++++++++++++++++---------- 2 files changed, 188 insertions(+), 70 deletions(-) diff --git a/servo-media/src/audio/block.rs b/servo-media/src/audio/block.rs index 1de3873c..00fc4199 100644 --- a/servo-media/src/audio/block.rs +++ b/servo-media/src/audio/block.rs @@ -89,6 +89,26 @@ impl Block { self.data_mut().as_mut_byte_slice().expect("casting failed") } + /// Zero-gain sum with another buffer + /// + /// Used after mixing multiple inputs to a single port + pub fn sum(mut self, mut other: Self) -> Self { + if self.is_silence() { + other + } else { + debug_assert!(self.channels == other.channels); + if self.repeat ^ other.repeat { + self.explicit_repeat(); + other.explicit_repeat(); + } + debug_assert!(self.buffer.len() == other.buffer.len()); + for (a, b) in self.buffer.iter_mut().zip(other.buffer.iter()) { + *a += b + } + self + } + } + /// If this is in "silence" mode without a buffer, allocate a silent buffer pub fn explicit_silence(&mut self) { if self.buffer.is_empty() { diff --git a/servo-media/src/audio/graph.rs b/servo-media/src/audio/graph.rs index 6a377318..8ba9aa47 100644 --- a/servo-media/src/audio/graph.rs +++ b/servo-media/src/audio/graph.rs @@ -1,3 +1,4 @@ +use smallvec::SmallVec; use audio::block::{Block, Chunk}; use audio::destination_node::DestinationNode; use audio::node::{AudioNodeEngine, BlockInfo, ChannelCountMode}; @@ -63,10 +64,41 @@ pub struct Node { node: RefCell>, } -/// Edges go *to* the output port from the input port, +/// An edge in the graph /// -/// The edge direction is the direction of sound +/// This connects one or more pair of ports between two +/// nodes, each connection represented by a `Connection`. +/// WebAudio allows for multiple connections to/from the same port +/// however it does not allow for duplicate connections between pairs +/// of ports pub struct Edge { + connections: SmallVec<[Connection; 1]> +} + +impl Edge { + /// Find if there are connections between two given ports, return the index + fn has_between(&self, + output_idx: PortIndex, + input_idx: PortIndex) -> bool { + self.connections.iter() + .find(|e| e.input_idx == input_idx && e.output_idx == output_idx) + .is_some() + } + + fn remove_by_output(&mut self, output_idx: PortIndex) { + self.connections.retain(|i| i.output_idx != output_idx) + } + + fn remove_by_pair(&mut self, + output_idx: PortIndex, + input_idx: PortIndex) { + self.connections + .retain(|i| i.output_idx != output_idx || i.input_idx != input_idx) + } +} + +/// A single connection between ports +struct Connection { /// The index of the port on the input node /// This is actually the /output/ of this edge input_idx: PortIndex, @@ -94,21 +126,20 @@ impl AudioGraph { /// /// The edge goes *from* the output port *to* the input port, connecting two nodes pub fn add_edge(&mut self, out: PortId, inp: PortId) { - // Output ports can only have a single edge associated with them. - // Remove all others - let old = self - .graph - .edges(out.node().0) - .find(|e| e.weight().input_idx == inp.1) - .map(|e| e.id()); - if let Some(old) = old { - self.graph.remove_edge(old); + let edge = self.graph.edges(out.node().0) + .find(|e| e.target() == inp.node().0) + .map(|e| e.id()); + if let Some(e) = edge { + // .find(|e| e.weight().has_between(out.1, inp.1)); + let w = self.graph.edge_weight_mut(e).expect("This edge is known to exist"); + if w.has_between(out.1, inp.1) { + return; + } + w.connections.push(Connection::new(inp.1, out.1)) + } else { + // add a new edge + self.graph.add_edge(out.node().0, inp.node().0, Edge::new(inp.1, out.1)); } - // add a new edge - // XXXManishearth it is actually possible for two nodes to have - // multiple edges between them between - // different ports. We should represent this somehow. - self.graph.add_edge(out.node().0, inp.node().0, Edge::new(inp.1, out.1)); } /// Disconnect all outgoing connections from a node @@ -124,19 +155,18 @@ impl AudioGraph { } } - /// Disconnect all outgoing connections from a node's output - /// - /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-output + // /// Disconnect all outgoing connections from a node's output + // /// + // /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-output pub fn disconnect_output(&mut self, out: PortId) { - // XXXManishearth we don't support multiple connections through - // a single output yet - let edge = self - .graph - .edges(out.node().0) - .find(|e| e.weight().output_idx == out.1) - .map(|e| e.id()); - if let Some(edge) = edge { - self.graph.remove_edge(edge); + let candidates: Vec<_> = self.graph.edges(out.node().0) + .map(|e| (e.id(), e.target())).collect(); + for (edge, to) in candidates { + let mut e = self.graph.remove_edge(edge).expect("Edge index is known to exist"); + e.remove_by_output(out.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, to, e); + } } } @@ -144,13 +174,12 @@ impl AudioGraph { /// /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode pub fn disconnect_between(&mut self, from: NodeId, to: NodeId) { - let edges = self.graph - .edges_directed(to.0, Direction::Incoming) - .filter(|e| e.target() == from.0) - .map(|e| e.id()) - .collect::>(); - for edge in edges { - self.graph.remove_edge(edge); + let edge = self.graph + .edges(from.0) + .find(|e| e.target() == to.0) + .map(|e| e.id()); + if let Some(i) = edge { + self.graph.remove_edge(i); } } @@ -161,26 +190,32 @@ impl AudioGraph { let edge = self .graph .edges(out.node().0) - .find(|e| e.weight().output_idx == out.1 && e.source() == to.0) + .find(|e| e.target() == to.0) .map(|e| e.id()); if let Some(edge) = edge { - self.graph.remove_edge(edge); + let mut e = self.graph.remove_edge(edge).expect("Edge index is known to exist"); + e.remove_by_output(out.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, to.0, e); + } } } - /// Disconnect all outgoing connections from a node's output to another node's input - /// - /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode-output-input + // /// Disconnect all outgoing connections from a node's output to another node's input + // /// + // /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode-output-input pub fn disconnect_output_between_to(&mut self, out: PortId, inp: PortId) { let edge = self .graph .edges(out.node().0) - .find(|e| e.weight().output_idx == out.1 && - e.source() == inp.node().0 && - e.weight().input_idx == inp.1) + .find(|e| e.target() == inp.node().0) .map(|e| e.id()); if let Some(edge) = edge { - self.graph.remove_edge(edge); + let mut e = self.graph.remove_edge(edge).expect("Edge index is known to exist"); + e.remove_by_pair(out.1, inp.1); + if !e.connections.is_empty() { + self.graph.add_edge(out.node().0, inp.node().0, e); + } } } @@ -200,18 +235,24 @@ impl AudioGraph { // This will only visit each node once let reversed = Reversed(&self.graph); let mut visit = DfsPostOrder::new(reversed, self.dest_id.0); + + let mut blocks: SmallVec<[SmallVec<[Block; 1]>; 1]> = SmallVec::new(); + let mut output_counts: SmallVec<[u32; 1]> = SmallVec::new(); + while let Some(ix) = visit.next(reversed) { let mut curr = self.graph[ix].node.borrow_mut(); + let mut chunk = Chunk::default(); + chunk + .blocks + .resize(curr.input_count() as usize, Default::default()); // if we have inputs, collect all the computed blocks // and construct a Chunk if curr.input_count() > 0 { - // set the chunk to the correct size - chunk - .blocks - .resize(curr.input_count() as usize, Default::default()); + // set up scratch space to store all the blocks + blocks.clear(); + blocks.resize(curr.input_count() as usize, Default::default()); - let mut max = 0; // max channel count let mode = curr.channel_count_mode(); let count = curr.channel_count(); let interpretation = curr.channel_interpretation(); @@ -219,29 +260,56 @@ impl AudioGraph { // all edges to this node are from its dependencies for edge in self.graph.edges_directed(ix, Direction::Incoming) { let edge = edge.weight(); - // XXXManishearth we can have multiple edges - // hitting the same input port, we should deal with that - let mut block = edge - .cache - .borrow_mut() - .take() - .expect("Cache should have been filled from traversal"); - if mode == ChannelCountMode::Explicit { - block.mix(count, interpretation); - } else { - max = cmp::max(max, block.chan_count()); + for connection in &edge.connections { + let block = connection + .cache + .borrow_mut() + .take() + .expect("Cache should have been filled from traversal"); + blocks[connection.input_idx.0 as usize].push(block); } - chunk[edge.input_idx] = block; } - if mode != ChannelCountMode::Explicit { - if mode == ChannelCountMode::ClampedMax { - max = cmp::min(max, count); - } - - for block in &mut chunk.blocks { - block.mix(max, interpretation); + for (i, mut blocks) in blocks.drain().enumerate() { + if blocks.len() == 0 { + if mode == ChannelCountMode::Explicit { + // It's silence, but mix it anyway + chunk.blocks[i].mix(count, interpretation); + } + } else if blocks.len() == 1 { + chunk.blocks[i] = blocks.pop().expect("`blocks` had length 1"); + match mode { + ChannelCountMode::Explicit => { + chunk.blocks[i].mix(count, interpretation); + } + ChannelCountMode::ClampedMax => { + if chunk.blocks[i].chan_count() > count { + chunk.blocks[i].mix(count, interpretation); + } + } + // It's one channel, it maxes itself + ChannelCountMode::Max => () + } + } else { + let mix_count = match mode { + ChannelCountMode::Explicit => count, + _ => { + let mut max = 0; // max channel count + for block in &blocks { + max = cmp::max(max, block.chan_count()); + } + if mode == ChannelCountMode::ClampedMax { + max = cmp::min(max, count); + } + max + } + }; + let block = blocks.into_iter().fold(Block::default(), |acc, mut block| { + block.mix(mix_count, interpretation); + acc.sum(block) + }); + chunk.blocks[i] = block; } } } @@ -255,11 +323,34 @@ impl AudioGraph { continue; } + // Count how many output connections fan out from each port + // This is so that we don't have to needlessly clone audio buffers + // + // If this is inefficient, we can instead maintain this data + // cached on the node + output_counts.clear(); + output_counts.resize(curr.output_count() as usize, 0); + for edge in self.graph.edges(ix) { + let edge = edge.weight(); + for conn in &edge.connections { + output_counts[conn.output_idx.0 as usize] += 1; + } + } + // all the edges from this node go to nodes which depend on it, // i.e. the nodes it outputs to. Store the blocks for retrieval. for edge in self.graph.edges(ix) { let edge = edge.weight(); - *edge.cache.borrow_mut() = Some(out[edge.output_idx].take()); + for conn in &edge.connections { + output_counts[conn.output_idx.0 as usize] -= 1; + // if there are no consumers left after this, take the data + let block = if output_counts[conn.output_idx.0 as usize] == 0 { + out[conn.output_idx].take() + } else { + out[conn.output_idx].clone() + }; + *conn.cache.borrow_mut() = Some(block); + } } } @@ -291,10 +382,17 @@ impl Node { impl Edge { pub fn new(input_idx: PortIndex, output_idx: PortIndex) -> Self { Edge { + connections: SmallVec::from_buf([Connection::new(input_idx, output_idx)]) + } + } +} + +impl Connection { + pub fn new(input_idx: PortIndex, output_idx: PortIndex) -> Self { + Connection { input_idx, output_idx, cache: RefCell::new(None), } } } - From 14d6cd75482d4b00e77ce21f96d627bdf586d200 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 3 Jul 2018 17:30:31 -0700 Subject: [PATCH 4/5] Add channel summing example --- examples/Cargo.toml | 6 ++++- examples/channelsum.rs | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 examples/channelsum.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e8cdcded..500122ba 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -30,4 +30,8 @@ path = "play_noise.rs" [[bin]] name = "channels" -path = "channels.rs" \ No newline at end of file +path = "channels.rs" + +[[bin]] +name = "channelsum" +path = "channelsum.rs" \ No newline at end of file diff --git a/examples/channelsum.rs b/examples/channelsum.rs new file mode 100644 index 00000000..48eb6861 --- /dev/null +++ b/examples/channelsum.rs @@ -0,0 +1,58 @@ +extern crate servo_media; + +use servo_media::audio::channel_node::ChannelNodeOptions; +use servo_media::audio::gain_node::GainNodeOptions; +use servo_media::audio::node::{AudioNodeMessage, AudioNodeType, AudioScheduledSourceNodeMessage}; +use servo_media::ServoMedia; +use std::sync::Arc; +use std::{thread, time}; + +fn run_example(servo_media: Arc) { + let context = servo_media.create_audio_context(Default::default()); + let mut options = Default::default(); + let osc = context.create_node(AudioNodeType::OscillatorNode(options)); + options.freq = 213.; + let osc2 = context.create_node(AudioNodeType::OscillatorNode(options)); + options.freq = 100.; + let osc3 = context.create_node(AudioNodeType::OscillatorNode(options)); + let mut options = GainNodeOptions::default(); + options.gain = 0.7; + let gain = context.create_node(AudioNodeType::GainNode(options)); + + let options = ChannelNodeOptions { channels: 2 }; + let merger = context.create_node(AudioNodeType::ChannelMergerNode(options)); + + let dest = context.dest_node(); + context.connect_ports(osc.output(0), merger.input(0)); + context.connect_ports(osc2.output(0), merger.input(1)); + context.connect_ports(merger.output(0), gain.input(0)); + context.connect_ports(osc3.output(0), gain.input(0)); + context.connect_ports(gain.output(0), dest.input(0)); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc2, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + osc3, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + let _ = context.resume(); + + thread::sleep(time::Duration::from_millis(2000)); + context.message_node(dest, AudioNodeMessage::SetChannelCount(1)); + thread::sleep(time::Duration::from_millis(2000)); + let _ = context.close(); + +} + +fn main() { + if let Ok(servo_media) = ServoMedia::get() { + run_example(servo_media); + } else { + unreachable!(); + } +} From c34173aefe4af5a3cde453c40d25ff37d400abad Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 3 Jul 2018 17:31:20 -0700 Subject: [PATCH 5/5] Unset repeat after explicit_repeat() --- servo-media/src/audio/block.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/servo-media/src/audio/block.rs b/servo-media/src/audio/block.rs index 00fc4199..fcc2288a 100644 --- a/servo-media/src/audio/block.rs +++ b/servo-media/src/audio/block.rs @@ -131,6 +131,7 @@ impl Block { } self.buffer = new; + self.repeat = false; } else if self.is_silence() { self.buffer.resize(FRAMES_PER_BLOCK_USIZE * self.channels as usize, 0.); }