diff --git a/audio/src/context.rs b/audio/src/context.rs index 2542a279..6eaa84aa 100644 --- a/audio/src/context.rs +++ b/audio/src/context.rs @@ -1,3 +1,4 @@ +use AudioBackend; use decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecoderOptions}; use graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; use node::{AudioNodeInit, AudioNodeMessage}; @@ -7,7 +8,6 @@ use std::cell::Cell; use std::marker::PhantomData; use std::sync::mpsc::{self, Sender}; use std::thread::Builder; -use AudioBackend; /// Describes the state of the audio context on the control thread. #[derive(Clone, Copy, Debug, PartialEq)] @@ -56,9 +56,9 @@ impl Default for RealTimeAudioContextOptions { /// User-specified options for an offline audio context. pub struct OfflineAudioContextOptions { /// The number of channels for this offline audio context. - pub channels: u32, + pub channels: u8, /// The length of the rendered audio buffer in sample-frames. - pub length: u32, + pub length: usize, /// Number of samples that will be rendered in one second, measured in Hz. pub sample_rate: f32, } @@ -111,14 +111,13 @@ pub struct AudioContext { backend: PhantomData, } -impl AudioContext { +impl AudioContext { /// Constructs a new audio context. pub fn new(options: AudioContextOptions) -> Self { - let options = match options { - AudioContextOptions::RealTimeAudioContext(options) => options, - AudioContextOptions::OfflineAudioContext(_) => unimplemented!(), + let sample_rate = match options { + AudioContextOptions::RealTimeAudioContext(ref options) => options.sample_rate, + AudioContextOptions::OfflineAudioContext(ref options) => options.sample_rate }; - let sample_rate = options.sample_rate; let (sender, receiver) = mpsc::channel(); let sender_ = sender.clone(); @@ -127,7 +126,7 @@ impl AudioContext { Builder::new() .name("AudioRenderThread".to_owned()) .spawn(move || { - AudioRenderThread::::start(receiver, sender_, options.sample_rate, graph) + AudioRenderThread::::start(receiver, sender_, sample_rate, graph, options) .expect("Could not start AudioRenderThread"); }) .unwrap(); @@ -246,6 +245,12 @@ impl AudioContext { }) .unwrap(); } + + pub fn set_eos_callback(&self, callback: Box>) + Send + Sync + 'static>) { + let _ = self + .sender + .send(AudioRenderThreadMsg::SetSinkEosCallback(callback)); + } } impl Drop for AudioContext { diff --git a/audio/src/lib.rs b/audio/src/lib.rs index 0b4afdc1..0d3cfb65 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(cell_update)] #![feature(fnbox, never_type)] #[macro_use] @@ -19,6 +20,7 @@ pub mod destination_node; pub mod gain_node; pub mod graph; pub mod node; +pub mod offline_sink; pub mod oscillator_node; pub mod param; pub mod render_thread; diff --git a/audio/src/offline_sink.rs b/audio/src/offline_sink.rs new file mode 100644 index 00000000..60012047 --- /dev/null +++ b/audio/src/offline_sink.rs @@ -0,0 +1,89 @@ +use block::{Chunk, FRAMES_PER_BLOCK_USIZE}; +use render_thread::AudioRenderThreadMsg; +use sink::AudioSink; +use std::cell::{Cell, RefCell}; +use std::sync::mpsc::Sender; + +pub struct ProcessedAudio(Box<[f32]>); + +impl AsRef<[f32]> for ProcessedAudio { + fn as_ref(&self) -> &[f32] { + &self.0 + } +} + +pub struct OfflineAudioSink { + buffer: RefCell>>, + channel_count: usize, + has_enough_data: Cell, + length: usize, + rendered_blocks: Cell, + eos_callback: RefCell>) + Send + Sync + 'static>>>, +} + +impl OfflineAudioSink { + pub fn new(channel_count: usize, length: usize) -> Self { + Self { + buffer: RefCell::new(None), + channel_count, + has_enough_data: Cell::new(false), + length, + rendered_blocks: Cell::new(0), + eos_callback: RefCell::new(None), + } + } +} + +impl AudioSink for OfflineAudioSink { + fn init(&self, _: f32, _: Sender) -> Result<(), ()> { + Ok(()) + } + + fn play(&self) -> Result<(), ()> { + self.has_enough_data.set(false); + Ok(()) + } + + fn stop(&self) -> Result<(), ()> { + self.has_enough_data.set(true); + Ok(()) + } + + fn has_enough_data(&self) -> bool { + self.has_enough_data.get() + || (self.rendered_blocks.get() >= (self.length / FRAMES_PER_BLOCK_USIZE)) + } + + fn push_data(&self, chunk: Chunk) -> Result<(), ()> { + { + let offset = self.rendered_blocks.get() * FRAMES_PER_BLOCK_USIZE; + let mut buffer = self.buffer.borrow_mut(); + if buffer.is_none() { + *buffer = Some(vec![0.; self.channel_count * self.length]); + } + if let Some(ref mut buffer) = *buffer { + for channel_number in 0..self.channel_count { + let channel_offset = offset + (channel_number * self.length); + let mut channel_data = + &mut buffer[channel_offset..channel_offset + FRAMES_PER_BLOCK_USIZE]; + channel_data.copy_from_slice(chunk.blocks[0].data_chan(channel_number as u8)); + } + }; + self.rendered_blocks.update(|blocks| blocks + 1); + } + + if self.rendered_blocks.get() >= (self.length / FRAMES_PER_BLOCK_USIZE) { + if let Some(callback) = self.eos_callback.borrow_mut().take() { + let processed_audio = + ProcessedAudio(self.buffer.borrow_mut().take().unwrap().into_boxed_slice()); + callback(Box::new(processed_audio)); + } + } + + Ok(()) + } + + fn set_eos_callback(&self, callback: Box>) + Send + Sync + 'static>) { + *self.eos_callback.borrow_mut() = Some(callback); + } +} diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index 513671d3..337b0246 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -1,12 +1,13 @@ use block::{Chunk, Tick, FRAMES_PER_BLOCK}; use buffer_source_node::AudioBufferSourceNode; use channel_node::{ChannelMergerNode, ChannelSplitterNode}; -use context::{ProcessingState, StateChangeResult}; +use context::{AudioContextOptions, ProcessingState, StateChangeResult}; use destination_node::DestinationNode; use gain_node::GainNode; use graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; use node::BlockInfo; use node::{AudioNodeEngine, AudioNodeInit, AudioNodeMessage}; +use offline_sink::OfflineAudioSink; use oscillator_node::OscillatorNode; use sink::AudioSink; use std::sync::mpsc::{Receiver, Sender}; @@ -28,26 +29,83 @@ pub enum AudioRenderThreadMsg { DisconnectTo(NodeId, PortId), DisconnectOutputBetween(PortId, NodeId), DisconnectOutputBetweenTo(PortId, PortId), + + SetSinkEosCallback(Box>) + Send + Sync + 'static>), +} + +pub enum Sink { + RealTime(B::Sink), + Offline(OfflineAudioSink), +} + +impl AudioSink for Sink { + fn init(&self, sample_rate: f32, sender: Sender) -> Result<(), ()> { + match *self { + Sink::RealTime(ref sink) => sink.init(sample_rate, sender), + Sink::Offline(ref sink) => sink.init(sample_rate, sender), + } + } + + fn play(&self) -> Result<(), ()> { + match *self { + Sink::RealTime(ref sink) => sink.play(), + Sink::Offline(ref sink) => sink.play(), + } + } + + fn stop(&self) -> Result<(), ()> { + match *self { + Sink::RealTime(ref sink) => sink.stop(), + Sink::Offline(ref sink) => sink.stop(), + } + } + + fn has_enough_data(&self) -> bool { + match *self { + Sink::RealTime(ref sink) => sink.has_enough_data(), + Sink::Offline(ref sink) => sink.has_enough_data(), + } + } + + fn push_data(&self, chunk: Chunk) -> Result<(), ()> { + match *self { + Sink::RealTime(ref sink) => sink.push_data(chunk), + Sink::Offline(ref sink) => sink.push_data(chunk), + } + } + + fn set_eos_callback(&self, callback: Box>) + Send + Sync + 'static>) { + match *self { + Sink::RealTime(ref sink) => sink.set_eos_callback(callback), + Sink::Offline(ref sink) => sink.set_eos_callback(callback), + } + } } pub struct AudioRenderThread { pub graph: AudioGraph, - pub sink: B::Sink, + pub sink: Sink, pub state: ProcessingState, pub sample_rate: f32, pub current_time: f64, pub current_frame: Tick, } -impl AudioRenderThread { +impl AudioRenderThread { /// Start the audio render thread pub fn start( event_queue: Receiver, sender: Sender, sample_rate: f32, graph: AudioGraph, + options: AudioContextOptions, ) -> Result<(), ()> { - let sink = B::make_sink()?; + let sink = match options { + AudioContextOptions::RealTimeAudioContext(_) => Sink::RealTime(B::make_sink()?), + AudioContextOptions::OfflineAudioContext(options) => Sink::Offline( + OfflineAudioSink::new(options.channels as usize, options.length), + ), + }; let mut graph = Self { graph, @@ -129,7 +187,6 @@ impl AudioRenderThread { // Do nothing. This will simply unblock the thread so we // can restart the non-blocking event loop. } - AudioRenderThreadMsg::DisconnectAllFrom(id) => { context.graph.disconnect_all_from(id) } @@ -146,6 +203,9 @@ impl AudioRenderThread { AudioRenderThreadMsg::DisconnectOutputBetweenTo(from, to) => { context.graph.disconnect_output_between_to(from, to) } + AudioRenderThreadMsg::SetSinkEosCallback(callback) => { + context.sink.set_eos_callback(callback); + } }; break_loop diff --git a/audio/src/sink.rs b/audio/src/sink.rs index 5047294f..03af31aa 100644 --- a/audio/src/sink.rs +++ b/audio/src/sink.rs @@ -12,6 +12,7 @@ pub trait AudioSink { fn stop(&self) -> Result<(), ()>; fn has_enough_data(&self) -> bool; fn push_data(&self, chunk: Chunk) -> Result<(), ()>; + fn set_eos_callback(&self, callback: Box>) + Send + Sync + 'static>); } pub struct DummyAudioSink; @@ -32,4 +33,5 @@ impl AudioSink for DummyAudioSink { fn push_data(&self, _: Chunk) -> Result<(), ()> { Ok(()) } + fn set_eos_callback(&self, _: Box>) + Send + Sync + 'static>) {} } diff --git a/backends/gstreamer/src/audio_sink.rs b/backends/gstreamer/src/audio_sink.rs index 1265322e..a24a5e7a 100644 --- a/backends/gstreamer/src/audio_sink.rs +++ b/backends/gstreamer/src/audio_sink.rs @@ -181,6 +181,8 @@ impl AudioSink for GStreamerAudioSink { .map(|_| ()) .map_err(|_| ()) } + + fn set_eos_callback(&self, _: Box>) + Send + Sync + 'static>) {} } impl Drop for GStreamerAudioSink { diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 7d2ec79c..16fd6556 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -11,6 +11,18 @@ servo-media = { path = "../servo-media" } name = "audio_decoder" path = "audio_decoder.rs" +[[bin]] +name = "channels" +path = "channels.rs" + +[[bin]] +name = "channelsum" +path = "channelsum.rs" + +[[bin]] +name = "offline" +path = "offline_context.rs" + [[bin]] name = "params" path = "params.rs" @@ -30,11 +42,3 @@ path = "play.rs" [[bin]] name = "play_noise" path = "play_noise.rs" - -[[bin]] -name = "channels" -path = "channels.rs" - -[[bin]] -name = "channelsum" -path = "channelsum.rs" diff --git a/examples/offline_context.rs b/examples/offline_context.rs new file mode 100644 index 00000000..9a0f6204 --- /dev/null +++ b/examples/offline_context.rs @@ -0,0 +1,68 @@ +extern crate servo_media; + +use servo_media::audio::block::FRAMES_PER_BLOCK_USIZE; +use servo_media::audio::buffer_source_node::AudioBufferSourceNodeMessage; +use servo_media::audio::context::{AudioContextOptions, OfflineAudioContextOptions}; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::ServoMedia; +use std::{thread, time}; +use std::sync::{Arc, Mutex}; +use std::sync::mpsc; + +fn run_example(servo_media: Arc) { + // Create offline context to process 1024 blocks of a oscillator node produced + // sine wave. + let mut options = ::default(); + options.channels = 2; + options.length = 1024 * FRAMES_PER_BLOCK_USIZE; + let options = AudioContextOptions::OfflineAudioContext(options); + let context = servo_media.create_audio_context(options); + let processed_audio = Arc::new(Mutex::new(Vec::new())); + let processed_audio_ = processed_audio.clone(); + let (sender, receiver) = mpsc::channel(); + let sender = Mutex::new(sender); + context.set_eos_callback(Box::new(move |buffer| { + processed_audio.lock().unwrap().extend_from_slice((*buffer).as_ref()); + sender.lock().unwrap().send(()).unwrap(); + })); + let osc = context.create_node(AudioNodeInit::OscillatorNode(Default::default())); + let dest = context.dest_node(); + context.connect_ports(osc.output(0), dest.input(0)); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + let _ = context.resume(); + // Block until we processed the data. + receiver.recv().unwrap(); + // Close offline context. + let _ = context.close(); + // Create audio context to play the processed audio. + let context = servo_media.create_audio_context(Default::default()); + let buffer_source = + context.create_node(AudioNodeInit::AudioBufferSourceNode(Default::default())); + let dest = context.dest_node(); + context.connect_ports(buffer_source.output(0), dest.input(0)); + context.message_node( + buffer_source, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + buffer_source, + AudioNodeMessage::AudioBufferSourceNode(AudioBufferSourceNodeMessage::SetBuffer(Some( + processed_audio_.lock().unwrap().to_vec().into(), + ))), + ); + let _ = context.resume(); + thread::sleep(time::Duration::from_millis(5000)); + let _ = context.close(); + +} + +fn main() { + if let Ok(servo_media) = ServoMedia::get() { + run_example(servo_media); + } else { + unreachable!() + } +}