diff --git a/audio/src/block.rs b/audio/src/block.rs index 02e074f4..ee8f9d3e 100644 --- a/audio/src/block.rs +++ b/audio/src/block.rs @@ -1,5 +1,5 @@ use byte_slice_cast::*; -use graph::PortIndex; +use graph::{PortIndex, PortKind}; use node::ChannelInterpretation; use smallvec::SmallVec; use std::f32::consts::SQRT_2; @@ -503,16 +503,24 @@ impl<'a> FrameRef<'a> { // operator impls -impl IndexMut> for Chunk { +impl IndexMut> for Chunk { fn index_mut(&mut self, i: PortIndex) -> &mut Block { - &mut self.blocks[i.0 as usize] + if let PortIndex::Port(i) = i { + &mut self.blocks[i as usize] + } else { + panic!("attempted to index chunk with param") + } } } -impl Index> for Chunk { +impl Index> for Chunk { type Output = Block; fn index(&self, i: PortIndex) -> &Block { - &self.blocks[i.0 as usize] + if let PortIndex::Port(i) = i { + &self.blocks[i as usize] + } else { + panic!("attempted to index chunk with param") + } } } diff --git a/audio/src/graph.rs b/audio/src/graph.rs index d9e75c68..d6b9eb1f 100644 --- a/audio/src/graph.rs +++ b/audio/src/graph.rs @@ -1,6 +1,7 @@ +use param::ParamType; use block::{Block, Chunk}; use destination_node::DestinationNode; -use node::{AudioNodeEngine, BlockInfo, ChannelCountMode}; +use node::{AudioNodeEngine, BlockInfo, ChannelCountMode, ChannelInterpretation}; use petgraph::graph::DefaultIx; use petgraph::stable_graph::NodeIndex; use petgraph::stable_graph::StableGraph; @@ -8,7 +9,7 @@ use petgraph::visit::{DfsPostOrder, EdgeRef, Reversed}; use petgraph::Direction; use smallvec::SmallVec; use std::cell::{RefCell, RefMut}; -use std::cmp; +use std::{cmp, fmt, hash}; #[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] /// A unique identifier for nodes in the graph. Stable @@ -17,10 +18,13 @@ pub struct NodeId(NodeIndex); impl NodeId { pub fn input(self, port: u32) -> PortId { - PortId(self, PortIndex(port, InputPort)) + PortId(self, PortIndex::Port(port)) + } + pub fn param(self, param: ParamType) -> PortId { + PortId(self, PortIndex::Param(param)) } pub fn output(self, port: u32) -> PortId { - PortId(self, PortIndex(port, OutputPort)) + PortId(self, PortIndex::Port(port)) } } @@ -34,17 +38,25 @@ impl NodeId { /// Kind is a zero sized type and is useful for distinguishing /// between input and output ports (which may otherwise share indices) #[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] -pub struct PortIndex(pub u32, pub Kind); +pub enum PortIndex { + Port(u32), + Param(Kind::ParamId) +} -impl PortId { +impl PortId { pub fn node(&self) -> NodeId { self.0 } } +pub trait PortKind { + type ParamId: Copy + Eq + PartialEq + Ord + + PartialOrd + hash::Hash + fmt::Debug; +} + /// An identifier for a port. #[derive(Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash, Debug)] -pub struct PortId(NodeId, PortIndex); +pub struct PortId(NodeId, PortIndex); #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Debug)] /// Marker type for denoting that the port is an input port @@ -55,6 +67,19 @@ pub struct InputPort; /// of the node it is connected to pub struct OutputPort; +impl PortKind for InputPort { + type ParamId = ParamType; +} + +impl PortKind for OutputPort { + // Params are only a feature of input ports. By using a never type here + // we ensure that the PortIndex enum has zero overhead for outputs, + // taking up no extra discriminant space and eliminating PortIndex::Param + // branches entirely from the compiled code + type ParamId = !; +} + + pub struct AudioGraph { graph: StableGraph, dest_id: NodeId, @@ -92,6 +117,10 @@ impl Edge { self.connections.retain(|i| i.output_idx != output_idx) } + fn remove_by_input(&mut self, input_idx: PortIndex) { + self.connections.retain(|i| i.input_idx != input_idx) + } + fn remove_by_pair( &mut self, output_idx: PortIndex, @@ -219,9 +248,37 @@ impl AudioGraph { } } - // /// Disconnect all outgoing connections from a node's output to another node's input - // /// - // /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode-output-input + /// Disconnect all outgoing connections from a node to another node's input + /// + /// Only used in WebAudio for disconnecting audio params + /// + /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationparam + pub fn disconnect_to( + &mut self, + node: NodeId, + inp: PortId, + ) { + let edge = self + .graph + .edges(node.0) + .find(|e| e.target() == inp.node().0) + .map(|e| e.id()); + if let Some(edge) = edge { + let mut e = self + .graph + .remove_edge(edge) + .expect("Edge index is known to exist"); + e.remove_by_input(inp.1); + if !e.connections.is_empty() { + self.graph.add_edge(node.0, inp.node().0, e); + } + } + } + + /// Disconnect all outgoing connections from a node's output to another node's input + /// + /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationnode-output-input + /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationparam-output pub fn disconnect_output_between_to( &mut self, out: PortId, @@ -286,12 +343,25 @@ impl AudioGraph { for edge in self.graph.edges_directed(ix, Direction::Incoming) { let edge = edge.weight(); for connection in &edge.connections { - let block = connection + let mut block = connection .cache .borrow_mut() .take() .expect("Cache should have been filled from traversal"); - blocks[connection.input_idx.0 as usize].push(block); + + match connection.input_idx { + PortIndex::Port(idx) => { + blocks[idx as usize].push(block); + } + PortIndex::Param(param) => { + // param inputs are downmixed to mono + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + block.mix(1, ChannelInterpretation::Speakers); + curr.get_param(param).add_block(block) + } + } + + } } @@ -356,7 +426,11 @@ impl AudioGraph { for edge in self.graph.edges(ix) { let edge = edge.weight(); for conn in &edge.connections { - output_counts[conn.output_idx.0 as usize] += 1; + if let PortIndex::Port(idx) = conn.output_idx { + output_counts[idx as usize] += 1; + } else { + unreachable!() + } } } @@ -365,14 +439,18 @@ impl AudioGraph { for edge in self.graph.edges(ix) { let edge = edge.weight(); for conn in &edge.connections { - output_counts[conn.output_idx.0 as usize] -= 1; - // if there are no consumers left after this, take the data - let block = if output_counts[conn.output_idx.0 as usize] == 0 { - out[conn.output_idx].take() + if let PortIndex::Port(idx) = conn.output_idx { + output_counts[idx as usize] -= 1; + // if there are no consumers left after this, take the data + let block = if output_counts[idx as usize] == 0 { + out[conn.output_idx].take() + } else { + out[conn.output_idx].clone() + }; + *conn.cache.borrow_mut() = Some(block); } else { - out[conn.output_idx].clone() - }; - *conn.cache.borrow_mut() = Some(block); + unreachable!() + } } } } diff --git a/audio/src/lib.rs b/audio/src/lib.rs index ed8303a6..0b4afdc1 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(fnbox)] +#![feature(fnbox, never_type)] #[macro_use] extern crate servo_media_derive; diff --git a/audio/src/param.rs b/audio/src/param.rs index 70d60385..416910ae 100644 --- a/audio/src/param.rs +++ b/audio/src/param.rs @@ -1,7 +1,8 @@ +use block::Block; use block::Tick; use node::BlockInfo; -#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] pub enum ParamType { Frequency, Detune, @@ -12,7 +13,6 @@ pub enum ParamType { /// An AudioParam. /// /// https://webaudio.github.io/web-audio-api/#AudioParam -#[derive(Debug)] pub struct Param { val: f32, kind: ParamRate, @@ -20,6 +20,12 @@ pub struct Param { current_event: usize, event_start_time: Tick, event_start_value: f32, + /// Cache of inputs from connect()ed nodes + blocks: Vec, + /// The value of all connect()ed inputs mixed together, for this frame + block_mix_val: f32, + /// If true, `blocks` has been summed together into a single block + summed: bool, } #[derive(Copy, Clone, Eq, PartialEq, Debug)] @@ -39,19 +45,48 @@ impl Param { current_event: 0, event_start_time: Tick(0), event_start_value: val, + blocks: Vec::new(), + block_mix_val: 0., + summed: false, } } /// Update the value of this param to the next /// + /// Invariant: This should be called with monotonically increasing + /// ticks, and Tick(0) should never be skipped. + /// /// Returns true if anything changed pub fn update(&mut self, block: &BlockInfo, tick: Tick) -> bool { - if tick.0 != 0 && self.kind == ParamRate::KRate { + if tick.0 == 0 { + self.summed = true; + if let Some(first) = self.blocks.pop() { + // first sum them together + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + let block = self.blocks.drain(..) + .fold(first, |acc, block| acc.sum(block)); + self.blocks.push(block); + + } + } else if self.kind == ParamRate::KRate { return false; } + + // Even if the timeline does nothing, it's still possible + // that there were connected inputs, so we should not + // directly return `false` after this point, instead returning + // `changed` + let changed = if let Some(block) = self.blocks.get(0) { + // store to be summed with `val` later + self.block_mix_val = block.data_chan(0)[tick.0 as usize]; + true + } else { + false + }; + if self.events.len() <= self.current_event { - return false; + return changed; } let current_tick = block.absolute_tick(tick); @@ -86,7 +121,7 @@ impl Param { move_next = true; } else { // This is a SetTarget event before its start time, ignore - return false; + return changed; } } } @@ -99,7 +134,7 @@ impl Param { // may need to move multiple times continue; } else { - return false; + return changed; } } break; @@ -114,7 +149,10 @@ impl Param { } pub fn value(&self) -> f32 { - self.val + // the data from connect()ed audionodes is first mixed + // together in update(), and then mixed with the actual param value + // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output + self.val + self.block_mix_val } pub fn set_rate(&mut self, rate: ParamRate) { @@ -156,6 +194,18 @@ impl Param { // XXXManishearth handle inserting events with a time before that // of the current one } + + pub(crate) fn add_block(&mut self, block: Block) { + debug_assert!(block.chan_count() == 1); + // summed only becomes true during a node's process() call, + // but add_block is called during graph traversal before processing, + // so if summed is true that means we've moved on to the next block + // and should clear our inputs + if self.summed { + self.blocks.clear(); + } + self.blocks.push(block) + } } #[derive(Clone, Copy, Eq, PartialEq, Debug)] diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index e4ebd749..513671d3 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -25,6 +25,7 @@ pub enum AudioRenderThreadMsg { DisconnectAllFrom(NodeId), DisconnectOutput(PortId), DisconnectBetween(NodeId, NodeId), + DisconnectTo(NodeId, PortId), DisconnectOutputBetween(PortId, NodeId), DisconnectOutputBetweenTo(PortId, PortId), } @@ -136,6 +137,9 @@ impl AudioRenderThread { AudioRenderThreadMsg::DisconnectBetween(from, to) => { context.graph.disconnect_between(from, to) } + AudioRenderThreadMsg::DisconnectTo(from, to) => { + context.graph.disconnect_to(from, to) + } AudioRenderThreadMsg::DisconnectOutputBetween(from, to) => { context.graph.disconnect_output_between(from, to) } diff --git a/examples/Cargo.toml b/examples/Cargo.toml index d9f6e17e..7d2ec79c 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -19,6 +19,10 @@ path = "params.rs" name = "params_settarget" path = "params_settarget.rs" +[[bin]] +name = "params_connect" +path = "params_connect.rs" + [[bin]] name = "play" path = "play.rs" diff --git a/examples/params_connect.rs b/examples/params_connect.rs new file mode 100644 index 00000000..b99a1568 --- /dev/null +++ b/examples/params_connect.rs @@ -0,0 +1,52 @@ +extern crate servo_media; + +use servo_media::audio::oscillator_node::OscillatorNodeOptions; +use servo_media::audio::node::{ + AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage, +}; +use servo_media::audio::param::{ParamType, RampKind, UserAutomationEvent}; +use servo_media::ServoMedia; +use std::sync::Arc; +use std::{thread, time}; + +fn run_example(servo_media: Arc) { + let context = servo_media.create_audio_context(Default::default()); + let mut options = OscillatorNodeOptions::default(); + options.freq = 2.0; + let lfo = context.create_node(AudioNodeInit::OscillatorNode(options)); + let osc = context.create_node(AudioNodeInit::OscillatorNode(Default::default())); + let gain = context.create_node(AudioNodeInit::GainNode(Default::default())); + let dest = context.dest_node(); + context.connect_ports(lfo.output(0), gain.param(ParamType::Gain)); + context.connect_ports(gain.output(0), dest.input(0)); + context.connect_ports(osc.output(0), gain.input(0)); + let _ = context.resume(); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + lfo, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + thread::sleep(time::Duration::from_millis(3000)); + // 0.75s - 1.75s: Linearly ramp frequency to 880Hz + context.message_node( + gain, + AudioNodeMessage::SetParam( + ParamType::Gain, + UserAutomationEvent::RampToValueAtTime(RampKind::Linear, 0., 6.), + ), + ); + + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); +} + +fn main() { + if let Ok(servo_media) = ServoMedia::get() { + run_example(servo_media); + } else { + unreachable!(); + } +}