From 028f8e3fcef652298bb24aabb7ac9eb1740e7d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 23 Jul 2018 11:17:00 +0200 Subject: [PATCH 1/8] Add OfflineAudioContext placeholder --- audio/src/lib.rs | 1 + audio/src/offline_context.rs | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 audio/src/offline_context.rs diff --git a/audio/src/lib.rs b/audio/src/lib.rs index 0b4afdc1..3eaaebc8 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -19,6 +19,7 @@ pub mod destination_node; pub mod gain_node; pub mod graph; pub mod node; +pub mod offline_context; pub mod oscillator_node; pub mod param; pub mod render_thread; diff --git a/audio/src/offline_context.rs b/audio/src/offline_context.rs new file mode 100644 index 00000000..66897f12 --- /dev/null +++ b/audio/src/offline_context.rs @@ -0,0 +1,25 @@ +use block::Chunk; +use render_thread::AudioRenderThreadMsg; +use sink::AudioSink; +use std::sync::mpsc::Sender; + +pub struct OfflineAudioContext { +} + +impl AudioSink for OfflineAudioContext { + fn init(&self, _: f32, _: Sender) -> Result<(), ()> { + Ok(()) + } + fn play(&self) -> Result<(), ()> { + Ok(()) + } + fn stop(&self) -> Result<(), ()> { + Ok(()) + } + fn has_enough_data(&self) -> bool { + true + } + fn push_data(&self, _: Chunk) -> Result<(), ()> { + Ok(()) + } +} From 1b35315a39cdbace63ac29130fea5758f692f4d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 23 Jul 2018 14:01:59 +0200 Subject: [PATCH 2/8] Use backend provided or offline context sink --- audio/src/context.rs | 13 ++++++------- audio/src/offline_context.rs | 9 +++++++++ audio/src/render_thread.rs | 20 ++++++++++++++++---- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/audio/src/context.rs b/audio/src/context.rs index 2542a279..75feb72c 100644 --- a/audio/src/context.rs +++ b/audio/src/context.rs @@ -1,3 +1,4 @@ +use AudioBackend; use decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecoderOptions}; use graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; use node::{AudioNodeInit, AudioNodeMessage}; @@ -7,7 +8,6 @@ use std::cell::Cell; use std::marker::PhantomData; use std::sync::mpsc::{self, Sender}; use std::thread::Builder; -use AudioBackend; /// Describes the state of the audio context on the control thread. #[derive(Clone, Copy, Debug, PartialEq)] @@ -111,14 +111,13 @@ pub struct AudioContext { backend: PhantomData, } -impl AudioContext { +impl AudioContext { /// Constructs a new audio context. pub fn new(options: AudioContextOptions) -> Self { - let options = match options { - AudioContextOptions::RealTimeAudioContext(options) => options, - AudioContextOptions::OfflineAudioContext(_) => unimplemented!(), + let sample_rate = match options { + AudioContextOptions::RealTimeAudioContext(ref options) => options.sample_rate, + AudioContextOptions::OfflineAudioContext(ref options) => options.sample_rate }; - let sample_rate = options.sample_rate; let (sender, receiver) = mpsc::channel(); let sender_ = sender.clone(); @@ -127,7 +126,7 @@ impl AudioContext { Builder::new() .name("AudioRenderThread".to_owned()) .spawn(move || { - AudioRenderThread::::start(receiver, sender_, options.sample_rate, graph) + AudioRenderThread::::start(receiver, sender_, sample_rate, graph, options) .expect("Could not start AudioRenderThread"); }) .unwrap(); diff --git a/audio/src/offline_context.rs b/audio/src/offline_context.rs index 66897f12..7478defe 100644 --- a/audio/src/offline_context.rs +++ b/audio/src/offline_context.rs @@ -4,6 +4,15 @@ use sink::AudioSink; use std::sync::mpsc::Sender; pub struct OfflineAudioContext { + length: usize, +} + +impl OfflineAudioContext { + pub fn new(length: usize) -> Self { + Self { + length + } + } } impl AudioSink for OfflineAudioContext { diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index 513671d3..0dc5128c 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -1,14 +1,16 @@ use block::{Chunk, Tick, FRAMES_PER_BLOCK}; use buffer_source_node::AudioBufferSourceNode; use channel_node::{ChannelMergerNode, ChannelSplitterNode}; -use context::{ProcessingState, StateChangeResult}; +use context::{AudioContextOptions, ProcessingState, StateChangeResult}; use destination_node::DestinationNode; use gain_node::GainNode; use graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; use node::BlockInfo; use node::{AudioNodeEngine, AudioNodeInit, AudioNodeMessage}; +use offline_context::OfflineAudioContext; use oscillator_node::OscillatorNode; use sink::AudioSink; +use std::marker::PhantomData; use std::sync::mpsc::{Receiver, Sender}; use AudioBackend; @@ -32,22 +34,31 @@ pub enum AudioRenderThreadMsg { pub struct AudioRenderThread { pub graph: AudioGraph, - pub sink: B::Sink, + pub sink: Box, pub state: ProcessingState, pub sample_rate: f32, pub current_time: f64, pub current_frame: Tick, + pub backend: PhantomData, } -impl AudioRenderThread { +impl AudioRenderThread { /// Start the audio render thread pub fn start( event_queue: Receiver, sender: Sender, sample_rate: f32, graph: AudioGraph, + options: AudioContextOptions, ) -> Result<(), ()> { - let sink = B::make_sink()?; + let sink: Box = match options { + AudioContextOptions::RealTimeAudioContext(_) => { + Box::new(B::make_sink()?) + }, + AudioContextOptions::OfflineAudioContext(options) => { + Box::new(OfflineAudioContext::new(options.length as usize)) + }, + }; let mut graph = Self { graph, @@ -56,6 +67,7 @@ impl AudioRenderThread { sample_rate, current_time: 0., current_frame: Tick(0), + backend: PhantomData, }; graph.sink.init(sample_rate, sender)?; From 653cafa9dd011695250164c40785d89b3b717a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 23 Jul 2018 15:09:22 +0200 Subject: [PATCH 3/8] Implement AudioSink for OfflineAudioContext --- audio/src/lib.rs | 1 + audio/src/offline_context.rs | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/audio/src/lib.rs b/audio/src/lib.rs index 3eaaebc8..b1797a01 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(cell_update)] #![feature(fnbox, never_type)] #[macro_use] diff --git a/audio/src/offline_context.rs b/audio/src/offline_context.rs index 7478defe..63d8574d 100644 --- a/audio/src/offline_context.rs +++ b/audio/src/offline_context.rs @@ -1,16 +1,21 @@ use block::Chunk; use render_thread::AudioRenderThreadMsg; use sink::AudioSink; +use std::cell::Cell; use std::sync::mpsc::Sender; pub struct OfflineAudioContext { + has_enough_data: Cell, length: usize, + rendered_bytes: Cell, } impl OfflineAudioContext { pub fn new(length: usize) -> Self { Self { - length + has_enough_data: Cell::new(false), + length, + rendered_bytes: Cell::new(0), } } } @@ -20,15 +25,18 @@ impl AudioSink for OfflineAudioContext { Ok(()) } fn play(&self) -> Result<(), ()> { + self.has_enough_data.set(false); Ok(()) } fn stop(&self) -> Result<(), ()> { + self.has_enough_data.set(true); Ok(()) } fn has_enough_data(&self) -> bool { - true + self.has_enough_data.get() && (self.rendered_bytes.get() < self.length) } - fn push_data(&self, _: Chunk) -> Result<(), ()> { + fn push_data(&self, chunk: Chunk) -> Result<(), ()> { + self.rendered_bytes.update(|bytes| bytes + chunk.len()); Ok(()) } } From dbf94e464ff5fc12cff64c3d8c1d2ad8098f837e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 23 Jul 2018 17:47:01 +0200 Subject: [PATCH 4/8] Store processed audio and add example --- audio/src/context.rs | 4 ++-- audio/src/offline_context.rs | 28 +++++++++++++++++++++------- audio/src/render_thread.rs | 2 +- examples/Cargo.toml | 20 ++++++++++++-------- examples/offline_context.rs | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 18 deletions(-) create mode 100644 examples/offline_context.rs diff --git a/audio/src/context.rs b/audio/src/context.rs index 75feb72c..9ab89cf3 100644 --- a/audio/src/context.rs +++ b/audio/src/context.rs @@ -56,9 +56,9 @@ impl Default for RealTimeAudioContextOptions { /// User-specified options for an offline audio context. pub struct OfflineAudioContextOptions { /// The number of channels for this offline audio context. - pub channels: u32, + pub channels: u8, /// The length of the rendered audio buffer in sample-frames. - pub length: u32, + pub length: usize, /// Number of samples that will be rendered in one second, measured in Hz. pub sample_rate: f32, } diff --git a/audio/src/offline_context.rs b/audio/src/offline_context.rs index 63d8574d..ce7239d2 100644 --- a/audio/src/offline_context.rs +++ b/audio/src/offline_context.rs @@ -1,21 +1,24 @@ -use block::Chunk; +use block::{Chunk, FRAMES_PER_BLOCK_USIZE}; use render_thread::AudioRenderThreadMsg; use sink::AudioSink; -use std::cell::Cell; +use std::cell::{Cell, RefCell}; use std::sync::mpsc::Sender; pub struct OfflineAudioContext { + buffers: RefCell>>, has_enough_data: Cell, length: usize, - rendered_bytes: Cell, + rendered_blocks: Cell, } impl OfflineAudioContext { - pub fn new(length: usize) -> Self { + pub fn new(number_of_channels: u8, length: usize) -> Self { + let buffers = vec![Vec::with_capacity(length * FRAMES_PER_BLOCK_USIZE); number_of_channels as usize]; Self { + buffers: RefCell::new(buffers), has_enough_data: Cell::new(false), length, - rendered_bytes: Cell::new(0), + rendered_blocks: Cell::new(0), } } } @@ -24,19 +27,30 @@ impl AudioSink for OfflineAudioContext { fn init(&self, _: f32, _: Sender) -> Result<(), ()> { Ok(()) } + fn play(&self) -> Result<(), ()> { self.has_enough_data.set(false); Ok(()) } + fn stop(&self) -> Result<(), ()> { self.has_enough_data.set(true); Ok(()) } + fn has_enough_data(&self) -> bool { - self.has_enough_data.get() && (self.rendered_bytes.get() < self.length) + self.has_enough_data.get() || (self.rendered_blocks.get() >= (self.length / FRAMES_PER_BLOCK_USIZE)) } + fn push_data(&self, chunk: Chunk) -> Result<(), ()> { - self.rendered_bytes.update(|bytes| bytes + chunk.len()); + let mut buffers = self.buffers.borrow_mut(); + let channel_count = buffers.len(); + for chan in 0..channel_count { + buffers[chan].extend_from_slice(chunk.blocks[0].data_chan(chan as u8)); + } + + self.rendered_blocks.update(|blocks| blocks + 1); + Ok(()) } } diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index 0dc5128c..acc1dd0b 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -56,7 +56,7 @@ impl AudioRenderThread { Box::new(B::make_sink()?) }, AudioContextOptions::OfflineAudioContext(options) => { - Box::new(OfflineAudioContext::new(options.length as usize)) + Box::new(OfflineAudioContext::new(options.channels, options.length)) }, }; diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 7d2ec79c..16fd6556 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -11,6 +11,18 @@ servo-media = { path = "../servo-media" } name = "audio_decoder" path = "audio_decoder.rs" +[[bin]] +name = "channels" +path = "channels.rs" + +[[bin]] +name = "channelsum" +path = "channelsum.rs" + +[[bin]] +name = "offline" +path = "offline_context.rs" + [[bin]] name = "params" path = "params.rs" @@ -30,11 +42,3 @@ path = "play.rs" [[bin]] name = "play_noise" path = "play_noise.rs" - -[[bin]] -name = "channels" -path = "channels.rs" - -[[bin]] -name = "channelsum" -path = "channelsum.rs" diff --git a/examples/offline_context.rs b/examples/offline_context.rs new file mode 100644 index 00000000..ec3beaf3 --- /dev/null +++ b/examples/offline_context.rs @@ -0,0 +1,33 @@ +extern crate servo_media; + +use servo_media::audio::block::FRAMES_PER_BLOCK_USIZE; +use servo_media::audio::context::{AudioContextOptions, OfflineAudioContextOptions}; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; +use servo_media::ServoMedia; +use std::{thread, time}; +use std::sync::Arc; + +fn run_example(servo_media: Arc) { + let mut options = ::default(); + options.length = 10 * FRAMES_PER_BLOCK_USIZE; + let options = AudioContextOptions::OfflineAudioContext(options); + let context = servo_media.create_audio_context(options); + let osc = context.create_node(AudioNodeInit::OscillatorNode(Default::default())); + let dest = context.dest_node(); + context.connect_ports(osc.output(0), dest.input(0)); + context.message_node( + osc, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + let _ = context.resume(); + thread::sleep(time::Duration::from_millis(3000)); + let _ = context.close(); +} + +fn main() { + if let Ok(servo_media) = ServoMedia::get() { + run_example(servo_media); + } else { + unreachable!() + } +} From 006d5e9e8cde657f665a66b55b3a9a354e89a9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 23 Jul 2018 19:01:07 +0200 Subject: [PATCH 5/8] End of stream callback --- audio/src/offline_context.rs | 55 ++++++++++++++++++++++------ audio/src/render_thread.rs | 11 +++--- audio/src/sink.rs | 2 + backends/gstreamer/src/audio_sink.rs | 2 + examples/offline_context.rs | 1 + 5 files changed, 55 insertions(+), 16 deletions(-) diff --git a/audio/src/offline_context.rs b/audio/src/offline_context.rs index ce7239d2..ab459038 100644 --- a/audio/src/offline_context.rs +++ b/audio/src/offline_context.rs @@ -4,21 +4,32 @@ use sink::AudioSink; use std::cell::{Cell, RefCell}; use std::sync::mpsc::Sender; +pub struct ProcessedAudio(Box<[f32]>); + +impl AsRef<[f32]> for ProcessedAudio { + fn as_ref(&self) -> &[f32] { + &self.0 + } +} + pub struct OfflineAudioContext { - buffers: RefCell>>, + buffer: RefCell>>, + channel_count: usize, has_enough_data: Cell, length: usize, rendered_blocks: Cell, + eos_callback: RefCell>) + Send + Sync + 'static>>>, } impl OfflineAudioContext { - pub fn new(number_of_channels: u8, length: usize) -> Self { - let buffers = vec![Vec::with_capacity(length * FRAMES_PER_BLOCK_USIZE); number_of_channels as usize]; + pub fn new(channel_count: usize, length: usize) -> Self { Self { - buffers: RefCell::new(buffers), + buffer: RefCell::new(None), + channel_count, has_enough_data: Cell::new(false), length, rendered_blocks: Cell::new(0), + eos_callback: RefCell::new(None), } } } @@ -39,18 +50,40 @@ impl AudioSink for OfflineAudioContext { } fn has_enough_data(&self) -> bool { - self.has_enough_data.get() || (self.rendered_blocks.get() >= (self.length / FRAMES_PER_BLOCK_USIZE)) + self.has_enough_data.get() + || (self.rendered_blocks.get() >= (self.length / FRAMES_PER_BLOCK_USIZE)) } fn push_data(&self, chunk: Chunk) -> Result<(), ()> { - let mut buffers = self.buffers.borrow_mut(); - let channel_count = buffers.len(); - for chan in 0..channel_count { - buffers[chan].extend_from_slice(chunk.blocks[0].data_chan(chan as u8)); - } + { + let offset = self.rendered_blocks.get() * FRAMES_PER_BLOCK_USIZE; + let mut buffer = self.buffer.borrow_mut(); + if buffer.is_none() { + *buffer = Some(vec![0.; self.channel_count * self.length]); + } + if let Some(ref mut buffer) = *buffer { + for channel_number in 0..self.channel_count { + let channel_offset = offset + (channel_number * self.length); + let mut channel_data = + &mut buffer[channel_offset..channel_offset + FRAMES_PER_BLOCK_USIZE]; + channel_data.copy_from_slice(chunk.blocks[0].data_chan(channel_number as u8)); + } + }; + self.rendered_blocks.update(|blocks| blocks + 1); + } - self.rendered_blocks.update(|blocks| blocks + 1); + if self.rendered_blocks.get() >= (self.length / FRAMES_PER_BLOCK_USIZE) { + if let Some(callback) = self.eos_callback.borrow_mut().take() { + let processed_audio = + ProcessedAudio(self.buffer.borrow_mut().take().unwrap().into_boxed_slice()); + callback(Box::new(processed_audio)); + } + } Ok(()) } + + fn set_eos_callback(&self, callback: Box>) + Send + Sync + 'static>) { + *self.eos_callback.borrow_mut() = Some(callback); + } } diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index acc1dd0b..d99c6cd7 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -52,12 +52,13 @@ impl AudioRenderThread { options: AudioContextOptions, ) -> Result<(), ()> { let sink: Box = match options { - AudioContextOptions::RealTimeAudioContext(_) => { - Box::new(B::make_sink()?) - }, + AudioContextOptions::RealTimeAudioContext(_) => Box::new(B::make_sink()?), AudioContextOptions::OfflineAudioContext(options) => { - Box::new(OfflineAudioContext::new(options.channels, options.length)) - }, + Box::new(OfflineAudioContext::new( + options.channels as usize, + options.length, + )) + } }; let mut graph = Self { diff --git a/audio/src/sink.rs b/audio/src/sink.rs index 5047294f..03af31aa 100644 --- a/audio/src/sink.rs +++ b/audio/src/sink.rs @@ -12,6 +12,7 @@ pub trait AudioSink { fn stop(&self) -> Result<(), ()>; fn has_enough_data(&self) -> bool; fn push_data(&self, chunk: Chunk) -> Result<(), ()>; + fn set_eos_callback(&self, callback: Box>) + Send + Sync + 'static>); } pub struct DummyAudioSink; @@ -32,4 +33,5 @@ impl AudioSink for DummyAudioSink { fn push_data(&self, _: Chunk) -> Result<(), ()> { Ok(()) } + fn set_eos_callback(&self, _: Box>) + Send + Sync + 'static>) {} } diff --git a/backends/gstreamer/src/audio_sink.rs b/backends/gstreamer/src/audio_sink.rs index 1265322e..a24a5e7a 100644 --- a/backends/gstreamer/src/audio_sink.rs +++ b/backends/gstreamer/src/audio_sink.rs @@ -181,6 +181,8 @@ impl AudioSink for GStreamerAudioSink { .map(|_| ()) .map_err(|_| ()) } + + fn set_eos_callback(&self, _: Box>) + Send + Sync + 'static>) {} } impl Drop for GStreamerAudioSink { diff --git a/examples/offline_context.rs b/examples/offline_context.rs index ec3beaf3..b7738fb4 100644 --- a/examples/offline_context.rs +++ b/examples/offline_context.rs @@ -9,6 +9,7 @@ use std::sync::Arc; fn run_example(servo_media: Arc) { let mut options = ::default(); + options.channels = 2; options.length = 10 * FRAMES_PER_BLOCK_USIZE; let options = AudioContextOptions::OfflineAudioContext(options); let context = servo_media.create_audio_context(options); From 59484d7eaaa02a2eb7c5a67c4017d3aa5f0ccf4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 24 Jul 2018 16:26:01 +0200 Subject: [PATCH 6/8] Eos callback setter and usage example --- audio/src/context.rs | 6 ++++++ audio/src/render_thread.rs | 6 +++++- examples/offline_context.rs | 40 ++++++++++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/audio/src/context.rs b/audio/src/context.rs index 9ab89cf3..6eaa84aa 100644 --- a/audio/src/context.rs +++ b/audio/src/context.rs @@ -245,6 +245,12 @@ impl AudioContext { }) .unwrap(); } + + pub fn set_eos_callback(&self, callback: Box>) + Send + Sync + 'static>) { + let _ = self + .sender + .send(AudioRenderThreadMsg::SetSinkEosCallback(callback)); + } } impl Drop for AudioContext { diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index d99c6cd7..8025a256 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -30,6 +30,8 @@ pub enum AudioRenderThreadMsg { DisconnectTo(NodeId, PortId), DisconnectOutputBetween(PortId, NodeId), DisconnectOutputBetweenTo(PortId, PortId), + + SetSinkEosCallback(Box>) + Send + Sync + 'static>), } pub struct AudioRenderThread { @@ -142,7 +144,6 @@ impl AudioRenderThread { // Do nothing. This will simply unblock the thread so we // can restart the non-blocking event loop. } - AudioRenderThreadMsg::DisconnectAllFrom(id) => { context.graph.disconnect_all_from(id) } @@ -159,6 +160,9 @@ impl AudioRenderThread { AudioRenderThreadMsg::DisconnectOutputBetweenTo(from, to) => { context.graph.disconnect_output_between_to(from, to) } + AudioRenderThreadMsg::SetSinkEosCallback(callback) => { + context.sink.set_eos_callback(callback); + } }; break_loop diff --git a/examples/offline_context.rs b/examples/offline_context.rs index b7738fb4..9a0f6204 100644 --- a/examples/offline_context.rs +++ b/examples/offline_context.rs @@ -1,18 +1,30 @@ extern crate servo_media; use servo_media::audio::block::FRAMES_PER_BLOCK_USIZE; +use servo_media::audio::buffer_source_node::AudioBufferSourceNodeMessage; use servo_media::audio::context::{AudioContextOptions, OfflineAudioContextOptions}; use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage, AudioScheduledSourceNodeMessage}; use servo_media::ServoMedia; use std::{thread, time}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use std::sync::mpsc; fn run_example(servo_media: Arc) { + // Create offline context to process 1024 blocks of a oscillator node produced + // sine wave. let mut options = ::default(); options.channels = 2; - options.length = 10 * FRAMES_PER_BLOCK_USIZE; + options.length = 1024 * FRAMES_PER_BLOCK_USIZE; let options = AudioContextOptions::OfflineAudioContext(options); let context = servo_media.create_audio_context(options); + let processed_audio = Arc::new(Mutex::new(Vec::new())); + let processed_audio_ = processed_audio.clone(); + let (sender, receiver) = mpsc::channel(); + let sender = Mutex::new(sender); + context.set_eos_callback(Box::new(move |buffer| { + processed_audio.lock().unwrap().extend_from_slice((*buffer).as_ref()); + sender.lock().unwrap().send(()).unwrap(); + })); let osc = context.create_node(AudioNodeInit::OscillatorNode(Default::default())); let dest = context.dest_node(); context.connect_ports(osc.output(0), dest.input(0)); @@ -21,8 +33,30 @@ fn run_example(servo_media: Arc) { AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), ); let _ = context.resume(); - thread::sleep(time::Duration::from_millis(3000)); + // Block until we processed the data. + receiver.recv().unwrap(); + // Close offline context. let _ = context.close(); + // Create audio context to play the processed audio. + let context = servo_media.create_audio_context(Default::default()); + let buffer_source = + context.create_node(AudioNodeInit::AudioBufferSourceNode(Default::default())); + let dest = context.dest_node(); + context.connect_ports(buffer_source.output(0), dest.input(0)); + context.message_node( + buffer_source, + AudioNodeMessage::AudioScheduledSourceNode(AudioScheduledSourceNodeMessage::Start(0.)), + ); + context.message_node( + buffer_source, + AudioNodeMessage::AudioBufferSourceNode(AudioBufferSourceNodeMessage::SetBuffer(Some( + processed_audio_.lock().unwrap().to_vec().into(), + ))), + ); + let _ = context.resume(); + thread::sleep(time::Duration::from_millis(5000)); + let _ = context.close(); + } fn main() { From 330964265af98c63cec296ce11651f5f43c527e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 25 Jul 2018 06:53:38 +0200 Subject: [PATCH 7/8] Rename OfflineAudioContext to OfflineAudioSink --- audio/src/lib.rs | 2 +- audio/src/{offline_context.rs => offline_sink.rs} | 6 +++--- audio/src/render_thread.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) rename audio/src/{offline_context.rs => offline_sink.rs} (96%) diff --git a/audio/src/lib.rs b/audio/src/lib.rs index b1797a01..0d3cfb65 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -20,7 +20,7 @@ pub mod destination_node; pub mod gain_node; pub mod graph; pub mod node; -pub mod offline_context; +pub mod offline_sink; pub mod oscillator_node; pub mod param; pub mod render_thread; diff --git a/audio/src/offline_context.rs b/audio/src/offline_sink.rs similarity index 96% rename from audio/src/offline_context.rs rename to audio/src/offline_sink.rs index ab459038..60012047 100644 --- a/audio/src/offline_context.rs +++ b/audio/src/offline_sink.rs @@ -12,7 +12,7 @@ impl AsRef<[f32]> for ProcessedAudio { } } -pub struct OfflineAudioContext { +pub struct OfflineAudioSink { buffer: RefCell>>, channel_count: usize, has_enough_data: Cell, @@ -21,7 +21,7 @@ pub struct OfflineAudioContext { eos_callback: RefCell>) + Send + Sync + 'static>>>, } -impl OfflineAudioContext { +impl OfflineAudioSink { pub fn new(channel_count: usize, length: usize) -> Self { Self { buffer: RefCell::new(None), @@ -34,7 +34,7 @@ impl OfflineAudioContext { } } -impl AudioSink for OfflineAudioContext { +impl AudioSink for OfflineAudioSink { fn init(&self, _: f32, _: Sender) -> Result<(), ()> { Ok(()) } diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index 8025a256..463bedf5 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -7,7 +7,7 @@ use gain_node::GainNode; use graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; use node::BlockInfo; use node::{AudioNodeEngine, AudioNodeInit, AudioNodeMessage}; -use offline_context::OfflineAudioContext; +use offline_sink::OfflineAudioSink; use oscillator_node::OscillatorNode; use sink::AudioSink; use std::marker::PhantomData; @@ -56,7 +56,7 @@ impl AudioRenderThread { let sink: Box = match options { AudioContextOptions::RealTimeAudioContext(_) => Box::new(B::make_sink()?), AudioContextOptions::OfflineAudioContext(options) => { - Box::new(OfflineAudioContext::new( + Box::new(OfflineAudioSink::new( options.channels as usize, options.length, )) From 89d9d04e97b96afbc030138e6cb1ede96efdf804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 25 Jul 2018 07:28:40 +0200 Subject: [PATCH 8/8] Use Sink enum for RT vs Offline --- audio/src/render_thread.rs | 67 +++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index 463bedf5..337b0246 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -10,7 +10,6 @@ use node::{AudioNodeEngine, AudioNodeInit, AudioNodeMessage}; use offline_sink::OfflineAudioSink; use oscillator_node::OscillatorNode; use sink::AudioSink; -use std::marker::PhantomData; use std::sync::mpsc::{Receiver, Sender}; use AudioBackend; @@ -34,14 +33,62 @@ pub enum AudioRenderThreadMsg { SetSinkEosCallback(Box>) + Send + Sync + 'static>), } +pub enum Sink { + RealTime(B::Sink), + Offline(OfflineAudioSink), +} + +impl AudioSink for Sink { + fn init(&self, sample_rate: f32, sender: Sender) -> Result<(), ()> { + match *self { + Sink::RealTime(ref sink) => sink.init(sample_rate, sender), + Sink::Offline(ref sink) => sink.init(sample_rate, sender), + } + } + + fn play(&self) -> Result<(), ()> { + match *self { + Sink::RealTime(ref sink) => sink.play(), + Sink::Offline(ref sink) => sink.play(), + } + } + + fn stop(&self) -> Result<(), ()> { + match *self { + Sink::RealTime(ref sink) => sink.stop(), + Sink::Offline(ref sink) => sink.stop(), + } + } + + fn has_enough_data(&self) -> bool { + match *self { + Sink::RealTime(ref sink) => sink.has_enough_data(), + Sink::Offline(ref sink) => sink.has_enough_data(), + } + } + + fn push_data(&self, chunk: Chunk) -> Result<(), ()> { + match *self { + Sink::RealTime(ref sink) => sink.push_data(chunk), + Sink::Offline(ref sink) => sink.push_data(chunk), + } + } + + fn set_eos_callback(&self, callback: Box>) + Send + Sync + 'static>) { + match *self { + Sink::RealTime(ref sink) => sink.set_eos_callback(callback), + Sink::Offline(ref sink) => sink.set_eos_callback(callback), + } + } +} + pub struct AudioRenderThread { pub graph: AudioGraph, - pub sink: Box, + pub sink: Sink, pub state: ProcessingState, pub sample_rate: f32, pub current_time: f64, pub current_frame: Tick, - pub backend: PhantomData, } impl AudioRenderThread { @@ -53,14 +100,11 @@ impl AudioRenderThread { graph: AudioGraph, options: AudioContextOptions, ) -> Result<(), ()> { - let sink: Box = match options { - AudioContextOptions::RealTimeAudioContext(_) => Box::new(B::make_sink()?), - AudioContextOptions::OfflineAudioContext(options) => { - Box::new(OfflineAudioSink::new( - options.channels as usize, - options.length, - )) - } + let sink = match options { + AudioContextOptions::RealTimeAudioContext(_) => Sink::RealTime(B::make_sink()?), + AudioContextOptions::OfflineAudioContext(options) => Sink::Offline( + OfflineAudioSink::new(options.channels as usize, options.length), + ), }; let mut graph = Self { @@ -70,7 +114,6 @@ impl AudioRenderThread { sample_rate, current_time: 0., current_frame: Tick(0), - backend: PhantomData, }; graph.sink.init(sample_rate, sender)?;