From 749bcf3bc84bc7ff28b165552a9d98b41a894bde Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 18 Jul 2018 16:18:53 -0700 Subject: [PATCH 1/4] Add unused support for param indexed inputs This adds all the types for param indexed inputs, and makes everything work with them, but doesn't actually do anything with connections to such inputs. --- audio/src/block.rs | 18 +++++++---- audio/src/graph.rs | 74 ++++++++++++++++++++++++++++++++++++---------- audio/src/lib.rs | 2 +- audio/src/param.rs | 2 +- 4 files changed, 74 insertions(+), 22 deletions(-) diff --git a/audio/src/block.rs b/audio/src/block.rs index 02e074f4..ee8f9d3e 100644 --- a/audio/src/block.rs +++ b/audio/src/block.rs @@ -1,5 +1,5 @@ use byte_slice_cast::*; -use graph::PortIndex; +use graph::{PortIndex, PortKind}; use node::ChannelInterpretation; use smallvec::SmallVec; use std::f32::consts::SQRT_2; @@ -503,16 +503,24 @@ impl<'a> FrameRef<'a> { // operator impls -impl IndexMut> for Chunk { +impl IndexMut> for Chunk { fn index_mut(&mut self, i: PortIndex) -> &mut Block { - &mut self.blocks[i.0 as usize] + if let PortIndex::Port(i) = i { + &mut self.blocks[i as usize] + } else { + panic!("attempted to index chunk with param") + } } } -impl Index> for Chunk { +impl Index> for Chunk { type Output = Block; fn index(&self, i: PortIndex) -> &Block { - &self.blocks[i.0 as usize] + if let PortIndex::Port(i) = i { + &self.blocks[i as usize] + } else { + panic!("attempted to index chunk with param") + } } } diff --git a/audio/src/graph.rs b/audio/src/graph.rs index d9e75c68..44b2f03b 100644 --- a/audio/src/graph.rs +++ b/audio/src/graph.rs @@ -1,3 +1,4 @@ +use param::ParamType; use block::{Block, Chunk}; use destination_node::DestinationNode; use node::{AudioNodeEngine, BlockInfo, ChannelCountMode}; @@ -8,7 +9,7 @@ use petgraph::visit::{DfsPostOrder, EdgeRef, Reversed}; use petgraph::Direction; use smallvec::SmallVec; use std::cell::{RefCell, RefMut}; -use std::cmp; +use std::{cmp, fmt, hash}; #[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] /// A unique identifier for nodes in the graph. Stable @@ -17,10 +18,13 @@ pub struct NodeId(NodeIndex); impl NodeId { pub fn input(self, port: u32) -> PortId { - PortId(self, PortIndex(port, InputPort)) + PortId(self, PortIndex::Port(port)) + } + pub fn param(self, param: ParamType) -> PortId { + PortId(self, PortIndex::Param(param)) } pub fn output(self, port: u32) -> PortId { - PortId(self, PortIndex(port, OutputPort)) + PortId(self, PortIndex::Port(port)) } } @@ -34,17 +38,25 @@ impl NodeId { /// Kind is a zero sized type and is useful for distinguishing /// between input and output ports (which may otherwise share indices) #[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] -pub struct PortIndex(pub u32, pub Kind); +pub enum PortIndex { + Port(u32), + Param(Kind::ParamId) +} -impl PortId { +impl PortId { pub fn node(&self) -> NodeId { self.0 } } +pub trait PortKind { + type ParamId: Copy + Eq + PartialEq + Ord + + PartialOrd + hash::Hash + fmt::Debug; +} + /// An identifier for a port. #[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] -pub struct PortId(NodeId, PortIndex); +pub struct PortId(NodeId, PortIndex); #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)] /// Marker type for denoting that the port is an input port @@ -55,6 +67,19 @@ pub struct InputPort; /// of the node it is connected to pub struct OutputPort; +impl PortKind for InputPort { + type ParamId = ParamType; +} + +impl PortKind for OutputPort { + // Params are only a feature of input ports. By using a never type here + // we ensure that the PortIndex enum has zero overhead for outputs, + // taking up no extra discriminant space and eliminating PortIndex::Param + // branches entirely from the compiled code + type ParamId = !; +} + + pub struct AudioGraph { graph: StableGraph, dest_id: NodeId, @@ -291,7 +316,18 @@ impl AudioGraph { .borrow_mut() .take() .expect("Cache should have been filled from traversal"); - blocks[connection.input_idx.0 as usize].push(block); + + match connection.input_idx { + PortIndex::Port(idx) => { + blocks[idx as usize].push(block); + } + PortIndex::Param(_) => { + // XXXManishearth do something here + + } + } + + } } @@ -356,7 +392,11 @@ impl AudioGraph { for edge in self.graph.edges(ix) { let edge = edge.weight(); for conn in &edge.connections { - output_counts[conn.output_idx.0 as usize] += 1; + if let PortIndex::Port(idx) = conn.output_idx { + output_counts[idx as usize] += 1; + } else { + unreachable!() + } } } @@ -365,14 +405,18 @@ impl AudioGraph { for edge in self.graph.edges(ix) { let edge = edge.weight(); for conn in &edge.connections { - output_counts[conn.output_idx.0 as usize] -= 1; - // if there are no consumers left after this, take the data - let block = if output_counts[conn.output_idx.0 as usize] == 0 { - out[conn.output_idx].take() + if let PortIndex::Port(idx) = conn.output_idx { + output_counts[idx as usize] -= 1; + // if there are no consumers left after this, take the data + let block = if output_counts[idx as usize] == 0 { + out[conn.output_idx].take() + } else { + out[conn.output_idx].clone() + }; + *conn.cache.borrow_mut() = Some(block); } else { - out[conn.output_idx].clone() - }; - *conn.cache.borrow_mut() = Some(block); + unreachable!() + } } } } diff --git a/audio/src/lib.rs b/audio/src/lib.rs index ed8303a6..0b4afdc1 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(fnbox)] +#![feature(fnbox, never_type)] #[macro_use] extern crate servo_media_derive; diff --git a/audio/src/param.rs b/audio/src/param.rs index 70d60385..e7a70eb8 100644 --- a/audio/src/param.rs +++ b/audio/src/param.rs @@ -1,7 +1,7 @@ use block::Tick; use node::BlockInfo; -#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] pub enum ParamType { Frequency, Detune, From 7b2f8369d2eeee86720cf6110e989b4947cc99cd Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 18 Jul 2018 16:41:21 -0700 Subject: [PATCH 2/4] Fill in support for mixing in inputs connected to audio params --- audio/src/graph.rs | 12 +++++---- audio/src/param.rs | 62 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/audio/src/graph.rs b/audio/src/graph.rs index 44b2f03b..f2e01091 100644 --- a/audio/src/graph.rs +++ b/audio/src/graph.rs @@ -1,7 +1,7 @@ use param::ParamType; use block::{Block, Chunk}; use destination_node::DestinationNode; -use node::{AudioNodeEngine, BlockInfo, ChannelCountMode}; +use node::{AudioNodeEngine, BlockInfo, ChannelCountMode, ChannelInterpretation}; use petgraph::graph::DefaultIx; use petgraph::stable_graph::NodeIndex; use petgraph::stable_graph::StableGraph; @@ -311,7 +311,7 @@ impl AudioGraph { for edge in self.graph.edges_directed(ix, Direction::Incoming) { let edge = edge.weight(); for connection in &edge.connections { - let block = connection + let mut block = connection .cache .borrow_mut() .take() @@ -321,9 +321,11 @@ impl AudioGraph { PortIndex::Port(idx) => { blocks[idx as usize].push(block); } - PortIndex::Param(_) => { - // XXXManishearth do something here - + PortIndex::Param(param) => { + // param inputs are downmixed to mono + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + block.mix(1, ChannelInterpretation::Speakers); + curr.get_param(param).add_block(block) } } diff --git a/audio/src/param.rs b/audio/src/param.rs index e7a70eb8..416910ae 100644 --- a/audio/src/param.rs +++ b/audio/src/param.rs @@ -1,3 +1,4 @@ +use block::Block; use block::Tick; use node::BlockInfo; @@ -12,7 +13,6 @@ pub enum ParamType { /// An AudioParam. /// /// https://webaudio.github.io/web-audio-api/#AudioParam -#[derive(Debug)] pub struct Param { val: f32, kind: ParamRate, @@ -20,6 +20,12 @@ pub struct Param { current_event: usize, event_start_time: Tick, event_start_value: f32, + /// Cache of inputs from connect()ed nodes + blocks: Vec, + /// The value of all connect()ed inputs mixed together, for this frame + block_mix_val: f32, + /// If true, `blocks` has been summed together into a single block + summed: bool, } #[derive(Copy, Clone, Eq, PartialEq, Debug)] @@ -39,19 +45,48 @@ impl Param { current_event: 0, event_start_time: Tick(0), event_start_value: val, + blocks: Vec::new(), + block_mix_val: 0., + summed: false, } } /// Update the value of this param to the next /// + /// Invariant: This should be called with monotonically increasing + /// ticks, and Tick(0) should never be skipped. + /// /// Returns true if anything changed pub fn update(&mut self, block: &BlockInfo, tick: Tick) -> bool { - if tick.0 != 0 && self.kind == ParamRate::KRate { + if tick.0 == 0 { + self.summed = true; + if let Some(first) = self.blocks.pop() { + // first sum them together + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + let block = self.blocks.drain(..) + .fold(first, |acc, block| acc.sum(block)); + self.blocks.push(block); + + } + } else if self.kind == ParamRate::KRate { return false; } + + // Even if the timeline does nothing, it's still possible + // that there were connected inputs, so we should not + // directly return `false` after this point, instead returning + // `changed` + let changed = if let Some(block) = self.blocks.get(0) { + // store to be summed with `val` later + self.block_mix_val = block.data_chan(0)[tick.0 as usize]; + true + } else { + false + }; + if self.events.len() <= self.current_event { - return false; + return changed; } let current_tick = block.absolute_tick(tick); @@ -86,7 +121,7 @@ impl Param { move_next = true; } else { // This is a SetTarget event before its start time, ignore - return false; + return changed; } } } @@ -99,7 +134,7 @@ impl Param { // may need to move multiple times continue; } else { - return false; + return changed; } } break; @@ -114,7 +149,10 @@ impl Param { } pub fn value(&self) -> f32 { - self.val + // the data from connect()ed audionodes is first mixed + // together in update(), and then mixed with the actual param value + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + self.val + self.block_mix_val } pub fn set_rate(&mut self, rate: ParamRate) { @@ -156,6 +194,18 @@ impl Param { // XXXManishearth handle inserting events with a time before that // of the current one } + + pub(crate) fn add_block(&mut self, block: Block) { + debug_assert!(block.chan_count() == 1); + // summed only becomes true during a node's process() call, + // but add_block is called during graph traversal before processing, + // so if summed is true that means we've moved on to the next block + // and should clear our inputs + if self.summed { + self.blocks.clear(); + } + self.blocks.push(block) + } } #[derive(Clone, Copy, Eq, PartialEq, Debug)] From 17cbde04ac71bbb41105a9fae23265ddb1be4e49 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 18 Jul 2018 17:01:09 -0700 Subject: [PATCH 3/4] Add example for connecting params to nodes --- examples/Cargo.toml | 4 +++ examples/params_connect.rs | 52 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 examples/params_connect.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index d9f6e17e..7d2ec79c 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -19,6 +19,10 @@ path = "params.rs" name = "params_settarget" path = "params_settarget.rs" +[[bin]] +name = "params_connect" +path = "params_connect.rs" + [[bin]] name = "play" path = "play.rs" diff --git a/examples/params_connect.rs b/examples/params_connect.rs new file mode 100644 index 00000000..b99a1568 --- /dev/null +++ b/examples/params_connect.rs @@ -0,0 +1,52 @@ +extern crate servo_media; + +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::audio::node::{ + AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage, +}; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::ServoMedia; +use std::sync::Arc; +use std::{thread, time}; + +fn run_example(servo_media: Arc) { + let context = servo_media.create_audio_context(Default::default()); + let mut options = OscillatorNodeOptions::default(); + options.freq = 2.0; + let lfo = context.create_node(AudioNodeInit::OscillatorNode(options)); + let osc = context.create_node(AudioNodeInit::OscillatorNode(Default::default())); + let gain = context.create_node(AudioNodeInit::GainNode(Default::default())); + let dest = context.dest_node(); + context.connect_ports(lfo.output(0), gain.param(ParamType::Gain)); + context.connect_ports(gain.output(0), dest.input(0)); + context.connect_ports(osc.output(0), gain.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + lfo, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + thread::sleep(time::Duration::from_millis(3000)); + // 0.75s - 1.75s: Linearly ramp frequency to 880Hz + context.message_node( + gain, + AudioNodeMessage::SetParam( + ParamType::Gain, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 0., 6.), + ), + ); + + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); +} + +fn main() { + if let Ok(servo_media) = ServoMedia::get() { + run_example(servo_media); + } else { + unreachable!(); + } +} From eeac67437ca0d2fa75ae4e3ff518388d674f46a9 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 18 Jul 2018 19:54:29 -0700 Subject: [PATCH 4/4] Add disconnect support --- audio/src/graph.rs | 38 +++++++++++++++++++++++++++++++++++--- audio/src/render_thread.rs | 4 ++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/audio/src/graph.rs b/audio/src/graph.rs index f2e01091..d6b9eb1f 100644 --- a/audio/src/graph.rs +++ b/audio/src/graph.rs @@ -117,6 +117,10 @@ impl Edge { self.connections.retain(|i| i.output_idx != output_idx) } + fn remove_by_input(&mut self, input_idx: PortIndex) { + self.connections.retain(|i| i.input_idx != input_idx) + } + fn remove_by_pair( &mut self, output_idx: PortIndex, @@ -244,9 +248,37 @@ impl AudioGraph { } } - // /// Disconnect all outgoing connections from a node's output to another node's input - // /// - // /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode-output-input + /// Disconnect all outgoing connections from a node to another node's input + /// + /// Only used in WebAudio for disconnecting audio params + /// + /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationparam + pub fn disconnect_to( + &mut self, + node: NodeId, + inp: PortId, + ) { + let edge = self + .graph + .edges(node.0) + .find(|e| e.target() == inp.node().0) + .map(|e| e.id()); + if let Some(edge) = edge { + let mut e = self + .graph + .remove_edge(edge) + .expect("Edge index is known to exist"); + e.remove_by_input(inp.1); + if !e.connections.is_empty() { + self.graph.add_edge(node.0, inp.node().0, e); + } + } + } + + /// Disconnect all outgoing connections from a node's output to another node's input + /// + /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode-output-input + /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationparam-output pub fn disconnect_output_between_to( &mut self, out: PortId, diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index e4ebd749..513671d3 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -25,6 +25,7 @@ pub enum AudioRenderThreadMsg { DisconnectAllFrom(NodeId), DisconnectOutput(PortId), DisconnectBetween(NodeId, NodeId), + DisconnectTo(NodeId, PortId), DisconnectOutputBetween(PortId, NodeId), DisconnectOutputBetweenTo(PortId, PortId), } @@ -136,6 +137,9 @@ impl AudioRenderThread { AudioRenderThreadMsg::DisconnectBetween(from, to) => { context.graph.disconnect_between(from, to) } + AudioRenderThreadMsg::DisconnectTo(from, to) => { + context.graph.disconnect_to(from, to) + } AudioRenderThreadMsg::DisconnectOutputBetween(from, to) => { context.graph.disconnect_output_between(from, to) }