From 677dae2c80c840f2685ead2fd93938f19eea1a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Sun, 30 Sep 2018 12:55:54 +0200 Subject: [PATCH 1/5] Remove duplicated DummyBackend --- audio/src/lib.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/audio/src/lib.rs b/audio/src/lib.rs index 83728e3a..dba031a8 100644 --- a/audio/src/lib.rs +++ b/audio/src/lib.rs @@ -40,21 +40,3 @@ pub trait AudioBackend { fn make_decoder() -> Self::Decoder; fn make_sink() -> Result::Error>; } - -pub struct DummyBackend {} - -impl AudioBackend for DummyBackend { - type Decoder = decoder::DummyAudioDecoder; - type Sink = sink::DummyAudioSink; - fn make_decoder() -> Self::Decoder { - decoder::DummyAudioDecoder - } - - fn make_sink() -> Result { - Ok(sink::DummyAudioSink) - } -} - -impl DummyBackend { - pub fn init() {} -} From b49450b4d7e9dfcfba6a261e7fec60974c7b945d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 1 Oct 2018 10:23:39 +0200 Subject: [PATCH 2/5] Audio decoder error handling --- audio/src/context.rs | 2 +- audio/src/decoder.rs | 37 +++++---- backends/gstreamer/src/audio_decoder.rs | 106 ++++++++++++------------ backends/gstreamer/src/audio_sink.rs | 14 +--- backends/gstreamer/src/lib.rs | 17 ++++ examples/audio_decoder.rs | 4 +- 6 files changed, 93 insertions(+), 87 deletions(-) diff --git a/audio/src/context.rs b/audio/src/context.rs index fe45819f..5cfb9019 100644 --- a/audio/src/context.rs +++ b/audio/src/context.rs @@ -245,7 +245,7 @@ impl AudioContext { /// Asynchronously decodes the audio file data contained in the given /// buffer. - pub fn decode_audio_data(&self, data: Vec, callbacks: AudioDecoderCallbacks) { + pub fn decode_audio_data(&self, data: Vec, callbacks: AudioDecoderCallbacks<::Error>) { let mut options = AudioDecoderOptions::default(); options.sample_rate = self.sample_rate; Builder::new() diff --git a/audio/src/decoder.rs b/audio/src/decoder.rs index aafdc72c..d38b95f3 100644 --- a/audio/src/decoder.rs +++ b/audio/src/decoder.rs @@ -1,15 +1,16 @@ use boxfnonce::SendBoxFnOnce; +use std::fmt::Debug; use std::sync::Mutex; -pub struct AudioDecoderCallbacks { +pub struct AudioDecoderCallbacks { pub eos: Mutex>>, - pub error: Mutex>>, + pub error: Mutex>>, pub progress: Option>, u32) + Send + Sync + 'static>>, pub ready: Mutex>>, } -impl AudioDecoderCallbacks { - pub fn new() -> AudioDecoderCallbacksBuilder { +impl AudioDecoderCallbacks { + pub fn new() -> AudioDecoderCallbacksBuilder { AudioDecoderCallbacksBuilder { eos: None, error: None, @@ -26,11 +27,11 @@ impl AudioDecoderCallbacks { }; } - pub fn error(&self) { - let error = self.error.lock().unwrap().take(); - match error { + pub fn error(&self, error: E) { + let callback = self.error.lock().unwrap().take(); + match callback { None => return, - Some(callback) => callback.call(), + Some(callback) => callback.call(error), }; } @@ -50,17 +51,17 @@ impl AudioDecoderCallbacks { } } -unsafe impl Send for AudioDecoderCallbacks {} -unsafe impl Sync for AudioDecoderCallbacks {} +unsafe impl Send for AudioDecoderCallbacks {} +unsafe impl Sync for AudioDecoderCallbacks {} -pub struct AudioDecoderCallbacksBuilder { +pub struct AudioDecoderCallbacksBuilder { eos: Option>, - error: Option>, + error: Option>, progress: Option>, u32) + Send + Sync + 'static>>, ready: Option>, } -impl AudioDecoderCallbacksBuilder { +impl AudioDecoderCallbacksBuilder { pub fn eos(self, eos: F) -> Self { Self { eos: Some(SendBoxFnOnce::new(eos)), @@ -68,7 +69,7 @@ impl AudioDecoderCallbacksBuilder { } } - pub fn error(self, error: F) -> Self { + pub fn error(self, error: F) -> Self { Self { error: Some(SendBoxFnOnce::new(error)), ..self @@ -92,7 +93,7 @@ impl AudioDecoderCallbacksBuilder { } } - pub fn build(self) -> AudioDecoderCallbacks { + pub fn build(self) -> AudioDecoderCallbacks { AudioDecoderCallbacks { eos: Mutex::new(self.eos), error: Mutex::new(self.error), @@ -115,10 +116,11 @@ impl Default for AudioDecoderOptions { } pub trait AudioDecoder { + type Error: Debug; fn decode( &self, data: Vec, - callbacks: AudioDecoderCallbacks, + callbacks: AudioDecoderCallbacks, options: Option, ); } @@ -126,5 +128,6 @@ pub trait AudioDecoder { pub struct DummyAudioDecoder; impl AudioDecoder for DummyAudioDecoder { - fn decode(&self, _: Vec, _: AudioDecoderCallbacks, _: Option) {} + type Error = (); + fn decode(&self, _: Vec, _: AudioDecoderCallbacks<()>, _: Option) {} } diff --git a/backends/gstreamer/src/audio_decoder.rs b/backends/gstreamer/src/audio_decoder.rs index 03c778ac..3deea427 100644 --- a/backends/gstreamer/src/audio_decoder.rs +++ b/backends/gstreamer/src/audio_decoder.rs @@ -8,6 +8,7 @@ use servo_media_audio::decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecod use std::io::Cursor; use std::io::Read; use std::sync::Arc; +use super::BackendError; pub struct GStreamerAudioDecoderProgress(MappedBuffer); @@ -26,10 +27,11 @@ impl GStreamerAudioDecoder { } impl AudioDecoder for GStreamerAudioDecoder { + type Error = BackendError; fn decode( &self, data: Vec, - callbacks: AudioDecoderCallbacks, + callbacks: AudioDecoderCallbacks, options: Option, ) { let pipeline = gst::Pipeline::new(None); @@ -37,31 +39,26 @@ impl AudioDecoder for GStreamerAudioDecoder { let appsrc = match gst::ElementFactory::make("appsrc", None) { Some(appsrc) => appsrc, - None => return callbacks.error(), + None => return callbacks.error(BackendError::ElementCreationFailed("appsrc")), }; let decodebin = match gst::ElementFactory::make("decodebin", None) { Some(decodebin) => decodebin, - None => return callbacks.error(), + None => return callbacks.error(BackendError::ElementCreationFailed("decodebin")), }; // decodebin uses something called a "sometimes-pad", which is basically // a pad that will show up when a certain condition is met, // in decodebins case that is media being decoded - if pipeline.add_many(&[&appsrc, &decodebin]).is_err() { - return callbacks.error(); + if let Err(e) = pipeline.add_many(&[&appsrc, &decodebin]) { + return callbacks.error(BackendError::PipelineFailed(e.0)); } - if gst::Element::link_many(&[&appsrc, &decodebin]).is_err() { - return callbacks.error(); + if let Err(e) = gst::Element::link_many(&[&appsrc, &decodebin]) { + return callbacks.error(BackendError::PipelineFailed(e.0)); } - let appsrc = match appsrc.downcast::() { - Ok(appsrc) => appsrc, - Err(_) => { - return callbacks.error(); - } - }; + let appsrc = appsrc.downcast::().unwrap(); let options = options.unwrap_or_default(); @@ -89,7 +86,7 @@ impl AudioDecoder for GStreamerAudioDecoder { let callbacks = &callbacks_; let pipeline = match pipeline_.upgrade() { Some(pipeline) => pipeline, - None => return callbacks.error(), + None => return callbacks.error(BackendError::PipelineFailed("upgrade")), }; let (is_audio, caps) = { @@ -103,33 +100,33 @@ impl AudioDecoder for GStreamerAudioDecoder { match media_type { None => { eprintln!("Failed to get media type from pad {}", src_pad.get_name()); - return callbacks.error(); + return callbacks.error(BackendError::Caps("Failed to get media type from pad")); } Some(media_type) => media_type, } }; if !is_audio { - return callbacks.error(); + return callbacks.error(BackendError::InvalidMediaFormat); } let sample_audio_info = match gst_audio::AudioInfo::from_caps(&caps) { Some(sample_audio_info) => sample_audio_info, - None => return callbacks.error(), + None => return callbacks.error(BackendError::AudioInfoFailed), }; let channels = sample_audio_info.channels(); callbacks.ready(channels); - let insert_deinterleave = || -> Result<(), ()> { - let convert = gst::ElementFactory::make("audioconvert", None).ok_or(())?; - let resample = gst::ElementFactory::make("audioresample", None).ok_or(())?; - let filter = gst::ElementFactory::make("capsfilter", None).ok_or(())?; + let insert_deinterleave = || -> Result<(), BackendError> { + let convert = gst::ElementFactory::make("audioconvert", None).ok_or(BackendError::ElementCreationFailed("audioconvert"))?; + let resample = gst::ElementFactory::make("audioresample", None).ok_or(BackendError::ElementCreationFailed("audioresample"))?; + let filter = gst::ElementFactory::make("capsfilter", None).ok_or(BackendError::ElementCreationFailed("capsfilter"))?; let deinterleave = - gst::ElementFactory::make("deinterleave", Some("deinterleave")).ok_or(())?; + gst::ElementFactory::make("deinterleave", Some("deinterleave")).ok_or(BackendError::ElementCreationFailed("deinterleave"))?; deinterleave .set_property("keep-positions", &true.to_value()) - .map_err(|_| ())?; + .map_err(|e| BackendError::SetPropertyFailed(e.0))?; let pipeline_ = pipeline.downgrade(); let callbacks_ = callbacks.clone(); deinterleave.connect_pad_added(move |_, src_pad| { @@ -142,14 +139,14 @@ impl AudioDecoder for GStreamerAudioDecoder { let callbacks = &callbacks_; let pipeline = match pipeline_.upgrade() { Some(pipeline) => pipeline, - None => return callbacks.error(), + None => return callbacks.error(BackendError::PipelineFailed("upgrade")), }; - let insert_sink = || -> Result<(), ()> { - let queue = gst::ElementFactory::make("queue", None).ok_or(())?; - let sink = gst::ElementFactory::make("appsink", None).ok_or(())?; - let appsink = sink.clone().dynamic_cast::().map_err(|_| ())?; + let insert_sink = || -> Result<(), BackendError> { + let queue = gst::ElementFactory::make("queue", None).ok_or(BackendError::ElementCreationFailed("queue"))?; + let sink = gst::ElementFactory::make("appsink", None).ok_or(BackendError::ElementCreationFailed("appsink"))?; + let appsink = sink.clone().dynamic_cast::().unwrap(); sink.set_property("sync", &false.to_value()) - .map_err(|_| ())?; + .map_err(|e| BackendError::SetPropertyFailed(e.0))?; let pipeline_ = pipeline.clone(); let pipeline__ = pipeline.clone(); @@ -168,7 +165,7 @@ impl AudioDecoder for GStreamerAudioDecoder { let buffer = if let Some(buffer) = sample.get_buffer() { buffer } else { - callbacks_.error(); + callbacks_.error(BackendError::InvalidSample); let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; @@ -176,7 +173,7 @@ impl AudioDecoder for GStreamerAudioDecoder { let caps = if let Some(caps) = sample.get_caps() { caps } else { - callbacks_.error(); + callbacks_.error(BackendError::Caps("Could not get caps from sample")); let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; @@ -186,7 +183,7 @@ impl AudioDecoder for GStreamerAudioDecoder { { audio_info } else { - callbacks_.error(); + callbacks_.error(BackendError::AudioInfoFailed); let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; @@ -195,7 +192,7 @@ impl AudioDecoder for GStreamerAudioDecoder { { positions } else { - callbacks_.error(); + callbacks_.error(BackendError::AudioInfoFailed); let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; @@ -206,11 +203,10 @@ impl AudioDecoder for GStreamerAudioDecoder { if let Ok(map) = buffer.into_mapped_buffer_readable() { map } else { - callbacks_.error(); + callbacks_.error(BackendError::BufferReadError); let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; - let progress = Box::new(GStreamerAudioDecoderProgress(map)); let channel = position.to_mask() as u32; callbacks_.progress(progress, channel); @@ -226,23 +222,23 @@ impl AudioDecoder for GStreamerAudioDecoder { ); let elements = &[&queue, &sink]; - pipeline.add_many(elements).map_err(|_| ())?; - gst::Element::link_many(elements).map_err(|_| ())?; + pipeline.add_many(elements).map_err(|e| BackendError::PipelineFailed(e.0))?; + gst::Element::link_many(elements).map_err(|e| BackendError::PipelineFailed(e.0))?; for e in elements { - e.sync_state_with_parent().map_err(|_| ())?; + e.sync_state_with_parent().map_err(|e| BackendError::PipelineFailed(e.0))?; } - let sink_pad = queue.get_static_pad("sink").ok_or(())?; + let sink_pad = queue.get_static_pad("sink").ok_or(BackendError::GetStaticPadFailed("sink"))?; src_pad .link(&sink_pad) .into_result() .map(|_| ()) - .map_err(|_| ()) + .map_err(|_| BackendError::PadLinkFailed) }; - if insert_sink().is_err() { - callbacks.error(); + if let Err(e) = insert_sink() { + callbacks.error(e); } }); @@ -250,38 +246,40 @@ impl AudioDecoder for GStreamerAudioDecoder { gst_audio::AUDIO_FORMAT_F32, options.sample_rate as u32, channels, - ).build() - .ok_or(())?; - let caps = audio_info.to_caps().ok_or(())?; + ).build().ok_or(BackendError::AudioInfoFailed)?; + let caps = audio_info.to_caps().ok_or(BackendError::AudioInfoFailed)?; filter .set_property("caps", &caps.to_value()) - .map_err(|_| ())?; + .map_err(|_| BackendError::SetPropertyFailed("caps"))?; let elements = &[&convert, &resample, &filter, &deinterleave]; - pipeline.add_many(elements).map_err(|_| ())?; - gst::Element::link_many(elements).map_err(|_| ())?; + pipeline.add_many(elements).map_err(|e| BackendError::PipelineFailed(e.0))?; + gst::Element::link_many(elements).map_err(|e| BackendError::PipelineFailed(e.0))?; for e in elements { - e.sync_state_with_parent().map_err(|_| ())?; + e.sync_state_with_parent().map_err(|e| BackendError::PipelineFailed(e.0))?; } - let sink_pad = convert.get_static_pad("sink").ok_or(())?; + let sink_pad = convert.get_static_pad("sink").ok_or(BackendError::GetStaticPadFailed("sink"))?; src_pad .link(&sink_pad) .into_result() .map(|_| ()) - .map_err(|_| ()) + .map_err(|_| BackendError::PadLinkFailed) }; - if insert_deinterleave().is_err() { - callbacks.error(); + if let Err(e) = insert_deinterleave() { + callbacks.error(e); } }); appsrc.set_property_format(gst::Format::Bytes); appsrc.set_property_block(true); - let _ = pipeline.set_state(gst::State::Playing); + if pipeline.set_state(gst::State::Playing).into_result().is_err() { + callbacks.error(BackendError::StateChangeFailed); + return; + } let max_bytes = appsrc.get_max_bytes() as usize; let data_len = data.len(); diff --git a/backends/gstreamer/src/audio_sink.rs b/backends/gstreamer/src/audio_sink.rs index 5095960e..2e9e3cc7 100644 --- a/backends/gstreamer/src/audio_sink.rs +++ b/backends/gstreamer/src/audio_sink.rs @@ -10,8 +10,7 @@ use std::cell::{Cell, RefCell}; use std::sync::mpsc::Sender; use std::sync::Arc; use std::thread::Builder; - -// XXX Define own error type. +use super::BackendError; const DEFAULT_SAMPLE_RATE: f32 = 44100.; @@ -23,17 +22,6 @@ pub struct GStreamerAudioSink { sample_offset: Cell, } -#[derive(Debug)] -pub enum BackendError { - Gstreamer(gst::Error), - Flow(gst::FlowError), - ElementCreationFailed(&'static str), - AudioInfoFailed, - PipelineFailed(&'static str), - StateChangeFailed, -} - - impl GStreamerAudioSink { pub fn new() -> Result { if let Some(category) = gst::DebugCategory::get("openslessink") { diff --git a/backends/gstreamer/src/lib.rs b/backends/gstreamer/src/lib.rs index 0662aec4..7b6ebb5a 100644 --- a/backends/gstreamer/src/lib.rs +++ b/backends/gstreamer/src/lib.rs @@ -18,6 +18,23 @@ pub mod audio_decoder; pub mod audio_sink; pub mod player; +#[derive(Debug)] +pub enum BackendError { + AudioInfoFailed, + BufferReadError, + Caps(&'static str), + ElementCreationFailed(&'static str), + Flow(gst::FlowError), + GetStaticPadFailed(&'static str), + Gstreamer(gst::Error), + InvalidMediaFormat, + InvalidSample, + PadLinkFailed, + PipelineFailed(&'static str), + SetPropertyFailed(&'static str), + StateChangeFailed, +} + pub struct GStreamerBackend; impl AudioBackend for GStreamerBackend { diff --git a/examples/audio_decoder.rs b/examples/audio_decoder.rs index 67769463..d7601434 100644 --- a/examples/audio_decoder.rs +++ b/examples/audio_decoder.rs @@ -34,8 +34,8 @@ fn run_example(servo_media: Arc) { .eos(move || { sender.send(()).unwrap(); }) - .error(|| { - eprintln!("Error decoding audio"); + .error(|e| { + eprintln!("Error decoding audio {:?}", e); }) .progress(move |buffer, channel| { let mut decoded_audio = decoded_audio_.lock().unwrap(); From 1fd8d2f2fc284a9166306deacc014f0836a6ce88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 1 Oct 2018 10:51:28 +0200 Subject: [PATCH 3/5] rustfmt audio/ backends/ --- audio/src/analyser_node.rs | 27 ++++++---- audio/src/biquad_filter_node.rs | 70 +++++++++++++------------ audio/src/block.rs | 12 +++-- audio/src/buffer_source_node.rs | 2 +- audio/src/channel_node.rs | 4 +- audio/src/context.rs | 23 +++++--- audio/src/graph.rs | 31 +++++------ audio/src/node.rs | 2 +- audio/src/offline_sink.rs | 9 ++-- audio/src/panner_node.rs | 52 +++++++++--------- audio/src/param.rs | 16 +++--- audio/src/render_thread.rs | 53 +++++++++++-------- backends/gstreamer/src/audio_decoder.rs | 62 +++++++++++++++------- backends/gstreamer/src/audio_sink.rs | 4 +- backends/gstreamer/src/lib.rs | 2 +- 15 files changed, 210 insertions(+), 159 deletions(-) diff --git a/audio/src/analyser_node.rs b/audio/src/analyser_node.rs index 45322ecc..a6a119d0 100644 --- a/audio/src/analyser_node.rs +++ b/audio/src/analyser_node.rs @@ -2,8 +2,8 @@ use block::{Block, Chunk, FRAMES_PER_BLOCK_USIZE}; use node::AudioNodeEngine; use node::BlockInfo; use node::{AudioNodeType, ChannelInfo, ChannelInterpretation}; -use std::f32::consts::PI; use std::cmp; +use std::f32::consts::PI; #[derive(AudioNodeCommon)] pub(crate) struct AnalyserNode { @@ -13,9 +13,11 @@ pub(crate) struct AnalyserNode { impl AnalyserNode { pub fn new(callback: Box, channel_info: ChannelInfo) -> Self { - Self { callback, channel_info } + Self { + callback, + channel_info, + } } - } impl AudioNodeEngine for AnalyserNode { @@ -50,7 +52,7 @@ pub struct AnalysisEngine { min_decibels: f64, max_decibels: f64, /// This is a ring buffer containing the last MAX_FFT_SIZE - /// sample-frames + /// sample-frames data: Box<[f32; MAX_FFT_SIZE]>, /// The index of the current block current_block: usize, @@ -68,8 +70,12 @@ pub struct AnalysisEngine { } impl AnalysisEngine { - pub fn new(fft_size: usize, smoothing_constant: f64, - min_decibels: f64, max_decibels: f64) -> Self { + pub fn new( + fft_size: usize, + smoothing_constant: f64, + min_decibels: f64, + max_decibels: f64, + ) -> Self { debug_assert!(fft_size >= 32 && fft_size <= 32768); // must be a power of two debug_assert!(fft_size & fft_size - 1 == 0); @@ -184,8 +190,8 @@ impl AnalysisEngine { self.blackman_windows.resize(self.fft_size, 0.); let coeff = PI * 2. / self.fft_size as f32; for n in 0..self.fft_size { - self.blackman_windows[n] = ALPHA_0 - ALPHA_1 * (coeff * n as f32).cos() - + ALPHA_2 * (2. * coeff * n as f32).cos(); + self.blackman_windows[n] = ALPHA_0 - ALPHA_1 * (coeff * n as f32).cos() + + ALPHA_2 * (2. * coeff * n as f32).cos(); } } @@ -212,7 +218,7 @@ impl AnalysisEngine { for k in 0..(self.fft_size / 2) { let mut sum_real = 0.; let mut sum_imaginary = 0.; - let factor = - 2. * PI * k as f32 / self.fft_size as f32; + let factor = -2. * PI * k as f32 / self.fft_size as f32; for n in 0..(self.fft_size) { sum_real += self.windowed[n] * (factor * n as f32).cos(); sum_imaginary += self.windowed[n] * (factor * n as f32).sin(); @@ -221,7 +227,8 @@ impl AnalysisEngine { let sum_imaginary = sum_imaginary / self.fft_size as f32; let magnitude = (sum_real * sum_real + sum_imaginary * sum_imaginary).sqrt(); self.smoothed_fft_data[k] = (self.smoothing_constant * self.smoothed_fft_data[k] as f64 - + (1. - self.smoothing_constant) * magnitude as f64) as f32; + + (1. - self.smoothing_constant) * magnitude as f64) + as f32; self.computed_fft_data[k] = 20. * self.smoothed_fft_data[k].log(10.); } } diff --git a/audio/src/biquad_filter_node.rs b/audio/src/biquad_filter_node.rs index c5cd46ab..2cbd38b8 100644 --- a/audio/src/biquad_filter_node.rs +++ b/audio/src/biquad_filter_node.rs @@ -5,7 +5,7 @@ use node::BlockInfo; use node::{AudioNodeMessage, AudioNodeType, ChannelInfo}; use param::{Param, ParamType}; use smallvec::SmallVec; -use std::f32::consts::{PI, SQRT_2}; +use std::f32::consts::{SQRT_2, PI}; #[derive(Copy, Clone, Debug)] pub struct BiquadFilterNodeOptions { @@ -25,7 +25,7 @@ pub enum FilterType { HighShelf, Peaking, Notch, - AllPass + AllPass, } impl Default for BiquadFilterNodeOptions { @@ -42,7 +42,7 @@ impl Default for BiquadFilterNodeOptions { #[derive(Copy, Clone, Debug)] pub enum BiquadFilterNodeMessage { - SetFilterType(FilterType) + SetFilterType(FilterType), } /// The last two input and output values, per-channel @@ -105,9 +105,11 @@ pub(crate) struct BiquadFilterNode { } impl BiquadFilterNode { - pub fn new(options: BiquadFilterNodeOptions, - channel_info: ChannelInfo, - sample_rate: f32) -> Self { + pub fn new( + options: BiquadFilterNodeOptions, + channel_info: ChannelInfo, + sample_rate: f32, + ) -> Self { let mut ret = Self { channel_info, filter: options.filter, @@ -115,7 +117,11 @@ impl BiquadFilterNode { gain: Param::new(options.gain), q: Param::new(options.q), detune: Param::new(options.detune), - b0: 0., b1: 0., b2: 0., a1: 0., a2: 0., + b0: 0., + b1: 0., + b2: 0., + a1: 0., + a2: 0., state: SmallVec::new(), }; ret.update_coefficients(sample_rate); @@ -174,8 +180,8 @@ impl BiquadFilterNode { } FilterType::HighPass => { self.b0 = (1. + cos_omega) / 2.; - self.b1 = - (1. + cos_omega); - self.b2 = - self.b1 / 2.; + self.b1 = -(1. + cos_omega); + self.b2 = -self.b1 / 2.; a0 = 1. + alpha_q_db; self.a1 = -2. * cos_omega; self.a2 = 1. - alpha_q_db; @@ -185,7 +191,7 @@ impl BiquadFilterNode { self.b1 = 0.; self.b2 = -alpha_q; a0 = 1. + alpha_q; - self.a1 = - 2. * cos_omega; + self.a1 = -2. * cos_omega; self.a2 = 1. - alpha_q; } FilterType::Notch => { @@ -214,21 +220,21 @@ impl BiquadFilterNode { } FilterType::LowShelf => { let alpha_rt_a = 2. * alpha_s * a.sqrt(); - self.b0 = a * ((a + 1.) - (a - 1.) * cos_omega + alpha_rt_a); - self.b1 = 2. * a * ((a - 1.) - (a + 1.) * cos_omega); - self.b2 = a * ((a + 1.) - (a - 1.) * cos_omega - alpha_rt_a); - a0 = (a + 1.) + (a - 1.) * cos_omega + alpha_rt_a; - self.a1 = -2. * ((a - 1.) + (a + 1.) * cos_omega); - self.a2 = (a + 1.) + (a - 1.) * cos_omega - alpha_rt_a; + self.b0 = a * ((a + 1.) - (a - 1.) * cos_omega + alpha_rt_a); + self.b1 = 2. * a * ((a - 1.) - (a + 1.) * cos_omega); + self.b2 = a * ((a + 1.) - (a - 1.) * cos_omega - alpha_rt_a); + a0 = (a + 1.) + (a - 1.) * cos_omega + alpha_rt_a; + self.a1 = -2. * ((a - 1.) + (a + 1.) * cos_omega); + self.a2 = (a + 1.) + (a - 1.) * cos_omega - alpha_rt_a; } FilterType::HighShelf => { let alpha_rt_a = 2. * alpha_s * a.sqrt(); - self.b0 = a * ((a + 1.) + (a - 1.) * cos_omega + alpha_rt_a); + self.b0 = a * ((a + 1.) + (a - 1.) * cos_omega + alpha_rt_a); self.b1 = -2. * a * ((a - 1.) + (a + 1.) * cos_omega); - self.b2 = a * ((a + 1.) + (a - 1.) * cos_omega - alpha_rt_a); - a0 = (a + 1.) - (a - 1.) * cos_omega + alpha_rt_a; - self.a1 = 2. * ((a - 1.) - (a + 1.) * cos_omega); - self.a2 = (a + 1.) - (a - 1.) * cos_omega - alpha_rt_a; + self.b2 = a * ((a + 1.) + (a - 1.) * cos_omega - alpha_rt_a); + a0 = (a + 1.) - (a - 1.) * cos_omega + alpha_rt_a; + self.a1 = 2. * ((a - 1.) - (a + 1.) * cos_omega); + self.a2 = (a + 1.) - (a - 1.) * cos_omega - alpha_rt_a; } } self.b0 = self.b0 / a0; @@ -246,7 +252,8 @@ impl AudioNodeEngine for BiquadFilterNode { fn process(&mut self, mut inputs: Chunk, info: &BlockInfo) -> Chunk { debug_assert!(inputs.len() == 1); - self.state.resize(inputs.blocks[0].chan_count() as usize, Default::default()); + self.state + .resize(inputs.blocks[0].chan_count() as usize, Default::default()); self.update_parameters(info, info.frame); // XXXManishearth this node has tail time, so even if the block is silence @@ -257,7 +264,6 @@ impl AudioNodeEngine for BiquadFilterNode { let repeat_or_silence = inputs.blocks[0].is_silence() || inputs.blocks[0].is_repeat(); - if repeat_or_silence && !self.state.iter().all(|s| *s == self.state[0]) { // In case our input is repeat/silence but our states are not identical, we must // explicitly duplicate, since mutate_with will otherwise only operate @@ -277,11 +283,11 @@ impl AudioNodeEngine for BiquadFilterNode { let state = &mut self.state[chan as usize]; let x0 = *sample; *sample = self.b0 * x0 + self.b1 * state.x1 + self.b2 * state.x2 - - self.a1 * state.y1 - self.a2 * state.y2; + - self.a1 * state.y1 + - self.a2 * state.y2; state.update(x0, *sample); }); } - } if inputs.blocks[0].is_repeat() { @@ -304,15 +310,13 @@ impl AudioNodeEngine for BiquadFilterNode { fn message_specific(&mut self, message: AudioNodeMessage, sample_rate: f32) { match message { - AudioNodeMessage::BiquadFilterNode(m) => { - match m { - BiquadFilterNodeMessage::SetFilterType(f) => { - self.filter = f; - self.update_coefficients(sample_rate); - } + AudioNodeMessage::BiquadFilterNode(m) => match m { + BiquadFilterNodeMessage::SetFilterType(f) => { + self.filter = f; + self.update_coefficients(sample_rate); } - } - _ => () + }, + _ => (), } } } diff --git a/audio/src/block.rs b/audio/src/block.rs index 6a0adb3d..715a1161 100644 --- a/audio/src/block.rs +++ b/audio/src/block.rs @@ -1,5 +1,5 @@ -use euclid::Vector3D; use byte_slice_cast::*; +use euclid::Vector3D; use graph::{PortIndex, PortKind}; use node::ChannelInterpretation; use smallvec::SmallVec; @@ -87,7 +87,7 @@ impl Block { Block { channels, repeat: false, - buffer: vec![0.; FRAMES_PER_BLOCK_USIZE * channels as usize] + buffer: vec![0.; FRAMES_PER_BLOCK_USIZE * channels as usize], } } @@ -539,9 +539,11 @@ impl<'a> FrameRef<'a> { f(&mut self.block.buffer[self.frame.0 as usize], 0) } else { for chan in 0..self.block.channels { - f(&mut self.block.buffer - [chan as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize], - chan) + f( + &mut self.block.buffer + [chan as usize * FRAMES_PER_BLOCK_USIZE + self.frame.0 as usize], + chan, + ) } } } diff --git a/audio/src/buffer_source_node.rs b/audio/src/buffer_source_node.rs index 1d4915f6..78475811 100644 --- a/audio/src/buffer_source_node.rs +++ b/audio/src/buffer_source_node.rs @@ -141,7 +141,7 @@ impl AudioNodeEngine for AudioBufferSourceNode { } else { len - self.playback_offset } - }, + } }; let next_offset = self.playback_offset + samples_to_copy; diff --git a/audio/src/channel_node.rs b/audio/src/channel_node.rs index 34021f42..d8cf811f 100644 --- a/audio/src/channel_node.rs +++ b/audio/src/channel_node.rs @@ -69,9 +69,7 @@ pub(crate) struct ChannelSplitterNode { impl ChannelSplitterNode { pub fn new(channel_info: ChannelInfo) -> Self { - ChannelSplitterNode { - channel_info, - } + ChannelSplitterNode { channel_info } } } diff --git a/audio/src/context.rs b/audio/src/context.rs index 5cfb9019..f126a916 100644 --- a/audio/src/context.rs +++ b/audio/src/context.rs @@ -1,4 +1,3 @@ -use AudioBackend; use decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecoderOptions}; use graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; use node::{AudioNodeInit, AudioNodeMessage, ChannelInfo}; @@ -8,6 +7,7 @@ use std::cell::Cell; use std::marker::PhantomData; use std::sync::mpsc::{self, Sender}; use std::thread::Builder; +use AudioBackend; /// Describes the state of the audio context on the control thread. #[derive(Clone, Copy, Debug, PartialEq)] @@ -121,7 +121,9 @@ impl AudioContext { pub fn new(options: AudioContextOptions) -> Self { let (sample_rate, channels) = match options { AudioContextOptions::RealTimeAudioContext(ref options) => (options.sample_rate, 2), - AudioContextOptions::OfflineAudioContext(ref options) => (options.sample_rate, options.channels) + AudioContextOptions::OfflineAudioContext(ref options) => { + (options.sample_rate, options.channels) + } }; let (sender, receiver) = mpsc::channel(); @@ -132,9 +134,14 @@ impl AudioContext { Builder::new() .name("AudioRenderThread".to_owned()) .spawn(move || { - AudioRenderThread::::start(|| B::make_sink(), - receiver, sender_, sample_rate, - graph, options); + AudioRenderThread::::start( + || B::make_sink(), + receiver, + sender_, + sample_rate, + graph, + options, + ); }) .unwrap(); Self { @@ -245,7 +252,11 @@ impl AudioContext { /// Asynchronously decodes the audio file data contained in the given /// buffer. - pub fn decode_audio_data(&self, data: Vec, callbacks: AudioDecoderCallbacks<::Error>) { + pub fn decode_audio_data( + &self, + data: Vec, + callbacks: AudioDecoderCallbacks<::Error>, + ) { let mut options = AudioDecoderOptions::default(); options.sample_rate = self.sample_rate; Builder::new() diff --git a/audio/src/graph.rs b/audio/src/graph.rs index c7a8567f..d22bba85 100644 --- a/audio/src/graph.rs +++ b/audio/src/graph.rs @@ -1,8 +1,8 @@ -use param::ParamType; use block::{Block, Chunk}; use destination_node::DestinationNode; use listener::AudioListenerNode; use node::{AudioNodeEngine, BlockInfo, ChannelCountMode, ChannelInterpretation}; +use param::ParamType; use petgraph::graph::DefaultIx; use petgraph::stable_graph::NodeIndex; use petgraph::stable_graph::StableGraph; @@ -47,7 +47,7 @@ pub enum PortIndex { Param(Kind::ParamId), /// special variant only used for the implicit connection /// from listeners to params - Listener(Kind::Listener) + Listener(Kind::Listener), } impl PortId { @@ -57,10 +57,8 @@ impl PortId { } pub trait PortKind { - type ParamId: Copy + Eq + PartialEq + Ord - + PartialOrd + hash::Hash + fmt::Debug; - type Listener: Copy + Eq + PartialEq + Ord - + PartialOrd + hash::Hash + fmt::Debug; + type ParamId: Copy + Eq + PartialEq + Ord + PartialOrd + hash::Hash + fmt::Debug; + type Listener: Copy + Eq + PartialEq + Ord + PartialOrd + hash::Hash + fmt::Debug; } /// An identifier for a port. @@ -93,7 +91,6 @@ impl PortKind for OutputPort { type Listener = Void; } - pub struct AudioGraph { graph: StableGraph, dest_id: NodeId, @@ -162,9 +159,14 @@ struct Connection { impl AudioGraph { pub fn new(channel_count: u8) -> Self { let mut graph = StableGraph::new(); - let dest_id = NodeId(graph.add_node(Node::new(Box::new(DestinationNode::new(channel_count))))); + let dest_id = + NodeId(graph.add_node(Node::new(Box::new(DestinationNode::new(channel_count))))); let listener_id = NodeId(graph.add_node(Node::new(Box::new(AudioListenerNode::new())))); - AudioGraph { graph, dest_id, listener_id } + AudioGraph { + graph, + dest_id, + listener_id, + } } /// Create a node, obtain its id @@ -269,11 +271,7 @@ impl AudioGraph { /// Only used in WebAudio for disconnecting audio params /// /// https://webaudio.github.io/web-audio-api/#dom-audionode-disconnect-destinationparam - pub fn disconnect_to( - &mut self, - node: NodeId, - inp: PortId, - ) { + pub fn disconnect_to(&mut self, node: NodeId, inp: PortId) { let edge = self .graph .edges(node.0) @@ -335,7 +333,6 @@ impl AudioGraph { self.listener_id } - /// For a given block, process all the data on this graph pub fn process(&mut self, info: &BlockInfo) -> Chunk { // DFS post order: Children are processed before their parent, @@ -388,9 +385,7 @@ impl AudioGraph { block.mix(1, ChannelInterpretation::Speakers); curr.get_param(param).add_block(block) } - PortIndex::Listener(_) => { - curr.set_listenerdata(block) - } + PortIndex::Listener(_) => curr.set_listenerdata(block), } } } diff --git a/audio/src/node.rs b/audio/src/node.rs index aa33d737..0b5c16a2 100644 --- a/audio/src/node.rs +++ b/audio/src/node.rs @@ -1,6 +1,6 @@ use biquad_filter_node::{BiquadFilterNodeMessage, BiquadFilterNodeOptions}; -use boxfnonce::SendBoxFnOnce; use block::{Block, Chunk, Tick}; +use boxfnonce::SendBoxFnOnce; use buffer_source_node::{AudioBufferSourceNodeMessage, AudioBufferSourceNodeOptions}; use channel_node::ChannelNodeOptions; use gain_node::GainNodeOptions; diff --git a/audio/src/offline_sink.rs b/audio/src/offline_sink.rs index 0cda9879..0781552e 100644 --- a/audio/src/offline_sink.rs +++ b/audio/src/offline_sink.rs @@ -79,17 +79,16 @@ impl AudioSink for OfflineAudioSink { if let Some(ref mut buffer) = *buffer { for channel_number in 0..self.channel_count { let channel_offset = offset + (channel_number * self.length); - let mut channel_data = - &mut buffer[channel_offset..channel_offset + copy_len]; - channel_data.copy_from_slice(&chunk.blocks[0].data_chan(channel_number as u8)[0..copy_len]); + let mut channel_data = &mut buffer[channel_offset..channel_offset + copy_len]; + channel_data + .copy_from_slice(&chunk.blocks[0].data_chan(channel_number as u8)[0..copy_len]); } }; self.rendered_blocks.set(self.rendered_blocks.get() + 1); if last { if let Some(callback) = self.eos_callback.borrow_mut().take() { - let processed_audio = - ProcessedAudio(buffer.take().unwrap().into_boxed_slice()); + let processed_audio = ProcessedAudio(buffer.take().unwrap().into_boxed_slice()); callback(Box::new(processed_audio)); } } diff --git a/audio/src/panner_node.rs b/audio/src/panner_node.rs index 75d95e45..4a654782 100644 --- a/audio/src/panner_node.rs +++ b/audio/src/panner_node.rs @@ -1,5 +1,5 @@ +use block::{Block, Chunk, Tick, FRAMES_PER_BLOCK}; use euclid::Vector3D; -use block::{Block, Chunk, FRAMES_PER_BLOCK, Tick}; use node::{AudioNodeEngine, AudioNodeMessage, BlockInfo}; use node::{AudioNodeType, ChannelInfo}; use param::{Param, ParamDir, ParamType}; @@ -18,14 +18,14 @@ fn normalize_zero(v: Vector3D) -> Vector3D { #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum PanningModel { EqualPower, - HRTF + HRTF, } #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum DistanceModel { Linear, Inverse, - Exponential + Exponential, } #[derive(Copy, Clone, Debug)] @@ -136,19 +136,20 @@ impl PannerNode { /// /// https://webaudio.github.io/web-audio-api/#azimuth-elevation /// https://webaudio.github.io/web-audio-api/#Spatialization-distance-effects - fn azimuth_elevation_distance(&self, - listener: (Vector3D, Vector3D, Vector3D)) - -> (f32, f32, f64) { + fn azimuth_elevation_distance( + &self, + listener: (Vector3D, Vector3D, Vector3D), + ) -> (f32, f32, f64) { let (listener_position, listener_forward, listener_up) = listener; let source_position = Vector3D::new( self.position_x.value(), self.position_y.value(), - self.position_z.value() + self.position_z.value(), ); // degenerate case if source_position == listener_position { - return (0., 0., 0.) + return (0., 0., 0.); } let diff = source_position - listener_position; @@ -186,24 +187,23 @@ impl PannerNode { } /// https://webaudio.github.io/web-audio-api/#Spatialization-sound-cones - fn cone_gain(&self, - listener: (Vector3D, Vector3D, Vector3D)) - -> f64 { + fn cone_gain(&self, listener: (Vector3D, Vector3D, Vector3D)) -> f64 { let (listener_position, _, _) = listener; let source_position = Vector3D::new( self.position_x.value(), self.position_y.value(), - self.position_z.value() + self.position_z.value(), ); let source_orientation = Vector3D::new( self.orientation_x.value(), self.orientation_y.value(), - self.orientation_z.value() + self.orientation_z.value(), ); - if source_orientation == Vector3D::zero() || - (self.cone_inner_angle == 360. && self.cone_outer_angle == 360.) { - return 0. + if source_orientation == Vector3D::zero() + || (self.cone_inner_angle == 360. && self.cone_outer_angle == 360.) + { + return 0.; } let normalized_source_orientation = normalize_zero(source_orientation); @@ -278,16 +278,16 @@ impl AudioNodeEngine for PannerNode { let listener_data = if let Some(listener_data) = self.listener_data.take() { listener_data } else { - return inputs + return inputs; }; // We clamp this early - let rolloff_factor = if self.distance_model == DistanceModel::Linear && - self.rolloff_factor > 1. { - 1. - } else { - self.rolloff_factor - }; + let rolloff_factor = + if self.distance_model == DistanceModel::Linear && self.rolloff_factor > 1. { + 1. + } else { + self.rolloff_factor + }; { let block = &mut inputs.blocks[0]; @@ -342,7 +342,7 @@ impl AudioNodeEngine for PannerNode { let mut gain_l = x.cos(); let mut gain_r = x.sin(); // 9. * PI / 2 is often slightly negative, clamp - if gain_l <= 0. {; + if gain_l <= 0. { gain_l = 0. } if gain_r <= 0. { @@ -401,8 +401,8 @@ impl AudioNodeEngine for PannerNode { PannerNodeMessage::SetConeInner(val) => self.cone_inner_angle = val, PannerNodeMessage::SetConeOuter(val) => self.cone_outer_angle = val, PannerNodeMessage::SetConeGain(val) => self.cone_outer_gain = val, - } - _ => () + }, + _ => (), } } } diff --git a/audio/src/param.rs b/audio/src/param.rs index 5806abba..cf7effb0 100644 --- a/audio/src/param.rs +++ b/audio/src/param.rs @@ -1,6 +1,6 @@ -use block::FRAMES_PER_BLOCK_USIZE; use block::Block; use block::Tick; +use block::FRAMES_PER_BLOCK_USIZE; use node::BlockInfo; #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] @@ -18,7 +18,9 @@ pub enum ParamType { #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] pub enum ParamDir { - X, Y, Z + X, + Y, + Z, } /// An AudioParam. @@ -74,16 +76,16 @@ impl Param { if let Some(first) = self.blocks.pop() { // first sum them together // https://webaudio.github.io/web-audio-api/#dom-audionode-connect-destinationparam-output - let block = self.blocks.drain(..) - .fold(first, |acc, block| acc.sum(block)); + let block = self + .blocks + .drain(..) + .fold(first, |acc, block| acc.sum(block)); self.blocks.push(block); - } } else if self.kind == ParamRate::KRate { return false; } - // Even if the timeline does nothing, it's still possible // that there were connected inputs, so we should not // directly return `false` after this point, instead returning @@ -232,7 +234,7 @@ impl Param { block[tick] = self.val; } } - // if the value is zero, our buffer is already zeroed + // if the value is zero, our buffer is already zeroed } else { for tick in 0..(FRAMES_PER_BLOCK_USIZE) { self.update(info, Tick(tick as u64)); diff --git a/audio/src/render_thread.rs b/audio/src/render_thread.rs index a1b5d875..f5575959 100644 --- a/audio/src/render_thread.rs +++ b/audio/src/render_thread.rs @@ -6,8 +6,8 @@ use channel_node::{ChannelMergerNode, ChannelSplitterNode}; use context::{AudioContextOptions, ProcessingState, StateChangeResult}; use gain_node::GainNode; use graph::{AudioGraph, InputPort, NodeId, OutputPort, PortId}; -use node::{BlockInfo, ChannelInfo}; use node::{AudioNodeEngine, AudioNodeInit, AudioNodeMessage}; +use node::{BlockInfo, ChannelInfo}; use offline_sink::OfflineAudioSink; use oscillator_node::OscillatorNode; use panner_node::PannerNode; @@ -42,7 +42,11 @@ pub enum Sink { impl AudioSink for Sink { type Error = S::Error; - fn init(&self, sample_rate: f32, sender: Sender) -> Result<(), Self::Error> { + fn init( + &self, + sample_rate: f32, + sender: Sender, + ) -> Result<(), Self::Error> { match *self { Sink::RealTime(ref sink) => sink.init(sample_rate, sender), Sink::Offline(ref sink) => Ok(sink.init(sample_rate, sender).unwrap()), @@ -85,7 +89,6 @@ impl AudioSink for Sink { } } - pub struct AudioRenderThread { pub graph: AudioGraph, pub sink: Sink, @@ -105,26 +108,26 @@ impl AudioRenderThread { sample_rate: f32, graph: AudioGraph, options: AudioContextOptions, - ) -> Result - where F: FnOnce() -> Result + ) -> Result + where + F: FnOnce() -> Result, { let sink = match options { AudioContextOptions::RealTimeAudioContext(_) => { let sink = match make_sink() { Ok(s) => s, - Err(e) => return Err((graph, e)) + Err(e) => return Err((graph, e)), }; Sink::RealTime(sink) - }, + } AudioContextOptions::OfflineAudioContext(options) => Sink::Offline( OfflineAudioSink::new(options.channels as usize, options.length), ), }; - if let Err(e) = sink.init(sample_rate, sender) { - return Err((graph, e)) + return Err((graph, e)); } Ok(Self { @@ -147,21 +150,27 @@ impl AudioRenderThread { sample_rate: f32, graph: AudioGraph, options: AudioContextOptions, - ) - where F: FnOnce() -> Result + ) where + F: FnOnce() -> Result, { let thread = Self::prepare_thread(make_sink, sender.clone(), sample_rate, graph, options); match thread { Ok(mut thread) => thread.event_loop(event_queue), Err((graph, e)) => { - error!("Could not start audio render thread due to error `{:?}`, \ - falling back to dummy backend", e); - let mut thread = AudioRenderThread:: - ::prepare_thread(|| Ok(DummyAudioSink), - sender, sample_rate, graph, options) - .map_err(|_| ()).unwrap(); + error!( + "Could not start audio render thread due to error `{:?}`, \ + falling back to dummy backend", + e + ); + let mut thread = AudioRenderThread::::prepare_thread( + || Ok(DummyAudioSink), + sender, + sample_rate, + graph, + options, + ).map_err(|_| ()) + .unwrap(); thread.event_loop(event_queue) - } } } @@ -184,12 +193,12 @@ impl AudioRenderThread { AudioNodeInit::PannerNode(options) => { needs_listener = true; Box::new(PannerNode::new(options, ch)) - }, + } AudioNodeInit::OscillatorNode(options) => Box::new(OscillatorNode::new(options, ch)), - AudioNodeInit::ChannelMergerNode(options) => Box::new(ChannelMergerNode::new(options, ch)), - AudioNodeInit::ChannelSplitterNode => { - Box::new(ChannelSplitterNode::new(ch)) + AudioNodeInit::ChannelMergerNode(options) => { + Box::new(ChannelMergerNode::new(options, ch)) } + AudioNodeInit::ChannelSplitterNode => Box::new(ChannelSplitterNode::new(ch)), _ => unimplemented!(), }; let id = self.graph.add_node(node); diff --git a/backends/gstreamer/src/audio_decoder.rs b/backends/gstreamer/src/audio_decoder.rs index 3deea427..acea73fb 100644 --- a/backends/gstreamer/src/audio_decoder.rs +++ b/backends/gstreamer/src/audio_decoder.rs @@ -1,5 +1,6 @@ use super::gst_app::{AppSink, AppSinkCallbacks, AppSrc}; use super::gst_audio; +use super::BackendError; use byte_slice_cast::*; use gst; use gst::buffer::{MappedBuffer, Readable}; @@ -8,7 +9,6 @@ use servo_media_audio::decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecod use std::io::Cursor; use std::io::Read; use std::sync::Arc; -use super::BackendError; pub struct GStreamerAudioDecoderProgress(MappedBuffer); @@ -100,7 +100,8 @@ impl AudioDecoder for GStreamerAudioDecoder { match media_type { None => { eprintln!("Failed to get media type from pad {}", src_pad.get_name()); - return callbacks.error(BackendError::Caps("Failed to get media type from pad")); + return callbacks + .error(BackendError::Caps("Failed to get media type from pad")); } Some(media_type) => media_type, } @@ -118,11 +119,14 @@ impl AudioDecoder for GStreamerAudioDecoder { callbacks.ready(channels); let insert_deinterleave = || -> Result<(), BackendError> { - let convert = gst::ElementFactory::make("audioconvert", None).ok_or(BackendError::ElementCreationFailed("audioconvert"))?; - let resample = gst::ElementFactory::make("audioresample", None).ok_or(BackendError::ElementCreationFailed("audioresample"))?; - let filter = gst::ElementFactory::make("capsfilter", None).ok_or(BackendError::ElementCreationFailed("capsfilter"))?; - let deinterleave = - gst::ElementFactory::make("deinterleave", Some("deinterleave")).ok_or(BackendError::ElementCreationFailed("deinterleave"))?; + let convert = gst::ElementFactory::make("audioconvert", None) + .ok_or(BackendError::ElementCreationFailed("audioconvert"))?; + let resample = gst::ElementFactory::make("audioresample", None) + .ok_or(BackendError::ElementCreationFailed("audioresample"))?; + let filter = gst::ElementFactory::make("capsfilter", None) + .ok_or(BackendError::ElementCreationFailed("capsfilter"))?; + let deinterleave = gst::ElementFactory::make("deinterleave", Some("deinterleave")) + .ok_or(BackendError::ElementCreationFailed("deinterleave"))?; deinterleave .set_property("keep-positions", &true.to_value()) @@ -142,8 +146,10 @@ impl AudioDecoder for GStreamerAudioDecoder { None => return callbacks.error(BackendError::PipelineFailed("upgrade")), }; let insert_sink = || -> Result<(), BackendError> { - let queue = gst::ElementFactory::make("queue", None).ok_or(BackendError::ElementCreationFailed("queue"))?; - let sink = gst::ElementFactory::make("appsink", None).ok_or(BackendError::ElementCreationFailed("appsink"))?; + let queue = gst::ElementFactory::make("queue", None) + .ok_or(BackendError::ElementCreationFailed("queue"))?; + let sink = gst::ElementFactory::make("appsink", None) + .ok_or(BackendError::ElementCreationFailed("appsink"))?; let appsink = sink.clone().dynamic_cast::().unwrap(); sink.set_property("sync", &false.to_value()) .map_err(|e| BackendError::SetPropertyFailed(e.0))?; @@ -173,7 +179,9 @@ impl AudioDecoder for GStreamerAudioDecoder { let caps = if let Some(caps) = sample.get_caps() { caps } else { - callbacks_.error(BackendError::Caps("Could not get caps from sample")); + callbacks_.error(BackendError::Caps( + "Could not get caps from sample", + )); let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; @@ -222,14 +230,20 @@ impl AudioDecoder for GStreamerAudioDecoder { ); let elements = &[&queue, &sink]; - pipeline.add_many(elements).map_err(|e| BackendError::PipelineFailed(e.0))?; - gst::Element::link_many(elements).map_err(|e| BackendError::PipelineFailed(e.0))?; + pipeline + .add_many(elements) + .map_err(|e| BackendError::PipelineFailed(e.0))?; + gst::Element::link_many(elements) + .map_err(|e| BackendError::PipelineFailed(e.0))?; for e in elements { - e.sync_state_with_parent().map_err(|e| BackendError::PipelineFailed(e.0))?; + e.sync_state_with_parent() + .map_err(|e| BackendError::PipelineFailed(e.0))?; } - let sink_pad = queue.get_static_pad("sink").ok_or(BackendError::GetStaticPadFailed("sink"))?; + let sink_pad = queue + .get_static_pad("sink") + .ok_or(BackendError::GetStaticPadFailed("sink"))?; src_pad .link(&sink_pad) .into_result() @@ -246,21 +260,27 @@ impl AudioDecoder for GStreamerAudioDecoder { gst_audio::AUDIO_FORMAT_F32, options.sample_rate as u32, channels, - ).build().ok_or(BackendError::AudioInfoFailed)?; + ).build() + .ok_or(BackendError::AudioInfoFailed)?; let caps = audio_info.to_caps().ok_or(BackendError::AudioInfoFailed)?; filter .set_property("caps", &caps.to_value()) .map_err(|_| BackendError::SetPropertyFailed("caps"))?; let elements = &[&convert, &resample, &filter, &deinterleave]; - pipeline.add_many(elements).map_err(|e| BackendError::PipelineFailed(e.0))?; + pipeline + .add_many(elements) + .map_err(|e| BackendError::PipelineFailed(e.0))?; gst::Element::link_many(elements).map_err(|e| BackendError::PipelineFailed(e.0))?; for e in elements { - e.sync_state_with_parent().map_err(|e| BackendError::PipelineFailed(e.0))?; + e.sync_state_with_parent() + .map_err(|e| BackendError::PipelineFailed(e.0))?; } - let sink_pad = convert.get_static_pad("sink").ok_or(BackendError::GetStaticPadFailed("sink"))?; + let sink_pad = convert + .get_static_pad("sink") + .ok_or(BackendError::GetStaticPadFailed("sink"))?; src_pad .link(&sink_pad) .into_result() @@ -276,7 +296,11 @@ impl AudioDecoder for GStreamerAudioDecoder { appsrc.set_property_format(gst::Format::Bytes); appsrc.set_property_block(true); - if pipeline.set_state(gst::State::Playing).into_result().is_err() { + if pipeline + .set_state(gst::State::Playing) + .into_result() + .is_err() + { callbacks.error(BackendError::StateChangeFailed); return; } diff --git a/backends/gstreamer/src/audio_sink.rs b/backends/gstreamer/src/audio_sink.rs index 2e9e3cc7..ad847d0a 100644 --- a/backends/gstreamer/src/audio_sink.rs +++ b/backends/gstreamer/src/audio_sink.rs @@ -1,3 +1,4 @@ +use super::BackendError; use byte_slice_cast::*; use gst; use gst::prelude::*; @@ -10,7 +11,6 @@ use std::cell::{Cell, RefCell}; use std::sync::mpsc::Sender; use std::sync::Arc; use std::thread::Builder; -use super::BackendError; const DEFAULT_SAMPLE_RATE: f32 = 44100.; @@ -30,7 +30,7 @@ impl GStreamerAudioSink { gst::init().map_err(BackendError::Gstreamer)?; let appsrc = gst::ElementFactory::make("appsrc", None) - .ok_or(BackendError::ElementCreationFailed("appsrc"))?; + .ok_or(BackendError::ElementCreationFailed("appsrc"))?; let appsrc = appsrc.downcast::().unwrap(); Ok(Self { pipeline: gst::Pipeline::new(None), diff --git a/backends/gstreamer/src/lib.rs b/backends/gstreamer/src/lib.rs index 7b6ebb5a..0c81c895 100644 --- a/backends/gstreamer/src/lib.rs +++ b/backends/gstreamer/src/lib.rs @@ -10,8 +10,8 @@ extern crate ipc_channel; extern crate servo_media_audio; extern crate servo_media_player; -use servo_media_audio::AudioBackend; use servo_media_audio::sink::AudioSink; +use servo_media_audio::AudioBackend; use servo_media_player::PlayerBackend; pub mod audio_decoder; From 9b2cff9b574dff239f32b431898ccbf8cee81ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 1 Oct 2018 11:55:42 +0200 Subject: [PATCH 4/5] Handle pipeline bus errors (including missing decoders and stuff) --- backends/gstreamer/src/audio_decoder.rs | 33 ++++++++++++++++++++++++- backends/gstreamer/src/lib.rs | 1 + 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/backends/gstreamer/src/audio_decoder.rs b/backends/gstreamer/src/audio_decoder.rs index acea73fb..346d7458 100644 --- a/backends/gstreamer/src/audio_decoder.rs +++ b/backends/gstreamer/src/audio_decoder.rs @@ -2,13 +2,14 @@ use super::gst_app::{AppSink, AppSinkCallbacks, AppSrc}; use super::gst_audio; use super::BackendError; use byte_slice_cast::*; -use gst; use gst::buffer::{MappedBuffer, Readable}; use gst::prelude::*; +use gst::{self, MessageView}; use servo_media_audio::decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecoderOptions}; use std::io::Cursor; use std::io::Read; use std::sync::Arc; +use std::thread::Builder; pub struct GStreamerAudioDecoderProgress(MappedBuffer); @@ -296,6 +297,36 @@ impl AudioDecoder for GStreamerAudioDecoder { appsrc.set_property_format(gst::Format::Bytes); appsrc.set_property_block(true); + let pipeline_ = pipeline.downgrade(); + let callbacks_ = callbacks.clone(); + + Builder::new() + .name("GStreamer pipeline bus".to_owned()) + .spawn(move || { + let callbacks = &callbacks_; + let pipeline = match pipeline_.upgrade() { + Some(pipeline) => pipeline, + None => return callbacks.error(BackendError::PipelineFailed("upgrade")), + }; + let bus = pipeline + .get_bus() + .expect("Pipeline without bus. Shouldn't happen!"); + + while let Some(msg) = bus.timed_pop(gst::CLOCK_TIME_NONE) { + match msg.view() { + MessageView::Eos(..) => break, + MessageView::Error(e) => { + callbacks.error(BackendError::PipelineBusError( + e.get_debug().unwrap_or("Unknown".to_owned()), + )); + break; + } + _ => (), + } + } + }) + .unwrap(); + if pipeline .set_state(gst::State::Playing) .into_result() diff --git a/backends/gstreamer/src/lib.rs b/backends/gstreamer/src/lib.rs index 0c81c895..1a01c0a9 100644 --- a/backends/gstreamer/src/lib.rs +++ b/backends/gstreamer/src/lib.rs @@ -30,6 +30,7 @@ pub enum BackendError { InvalidMediaFormat, InvalidSample, PadLinkFailed, + PipelineBusError(String), PipelineFailed(&'static str), SetPropertyFailed(&'static str), StateChangeFailed, From 45aceb0fe56793abc49a3c67ecb73046c50b1d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 1 Oct 2018 23:46:53 +0200 Subject: [PATCH 5/5] Use a sync handler to get pipeline bus messages --- backends/gstreamer/src/audio_decoder.rs | 92 +++++++++++++------------ 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/backends/gstreamer/src/audio_decoder.rs b/backends/gstreamer/src/audio_decoder.rs index 346d7458..c002fdc0 100644 --- a/backends/gstreamer/src/audio_decoder.rs +++ b/backends/gstreamer/src/audio_decoder.rs @@ -8,8 +8,7 @@ use gst::{self, MessageView}; use servo_media_audio::decoder::{AudioDecoder, AudioDecoderCallbacks, AudioDecoderOptions}; use std::io::Cursor; use std::io::Read; -use std::sync::Arc; -use std::thread::Builder; +use std::sync::{mpsc, Arc, Mutex}; pub struct GStreamerAudioDecoderProgress(MappedBuffer); @@ -63,8 +62,12 @@ impl AudioDecoder for GStreamerAudioDecoder { let options = options.unwrap_or_default(); + let (sender, receiver) = mpsc::channel(); + let sender = Arc::new(Mutex::new(sender)); + let pipeline_ = pipeline.downgrade(); let callbacks_ = callbacks.clone(); + let sender_ = sender.clone(); // Initial pipeline looks like // // appsrc ! decodebin2! ... @@ -85,9 +88,14 @@ impl AudioDecoder for GStreamerAudioDecoder { // each channel. let callbacks = &callbacks_; + let sender = &sender_; let pipeline = match pipeline_.upgrade() { Some(pipeline) => pipeline, - None => return callbacks.error(BackendError::PipelineFailed("upgrade")), + None => { + callbacks.error(BackendError::PipelineFailed("upgrade")); + let _ = sender.lock().unwrap().send(()); + return; + } }; let (is_audio, caps) = { @@ -100,21 +108,27 @@ impl AudioDecoder for GStreamerAudioDecoder { match media_type { None => { - eprintln!("Failed to get media type from pad {}", src_pad.get_name()); - return callbacks - .error(BackendError::Caps("Failed to get media type from pad")); + callbacks.error(BackendError::Caps("Failed to get media type from pad")); + let _ = sender.lock().unwrap().send(()); + return; } Some(media_type) => media_type, } }; if !is_audio { - return callbacks.error(BackendError::InvalidMediaFormat); + callbacks.error(BackendError::InvalidMediaFormat); + let _ = sender.lock().unwrap().send(()); + return; } let sample_audio_info = match gst_audio::AudioInfo::from_caps(&caps) { Some(sample_audio_info) => sample_audio_info, - None => return callbacks.error(BackendError::AudioInfoFailed), + None => { + callbacks.error(BackendError::AudioInfoFailed); + let _ = sender.lock().unwrap().send(()); + return; + } }; let channels = sample_audio_info.channels(); callbacks.ready(channels); @@ -155,8 +169,6 @@ impl AudioDecoder for GStreamerAudioDecoder { sink.set_property("sync", &false.to_value()) .map_err(|e| BackendError::SetPropertyFailed(e.0))?; - let pipeline_ = pipeline.clone(); - let pipeline__ = pipeline.clone(); let callbacks_ = callbacks.clone(); let callbacks__ = callbacks.clone(); appsink.set_callbacks( @@ -173,7 +185,6 @@ impl AudioDecoder for GStreamerAudioDecoder { buffer } else { callbacks_.error(BackendError::InvalidSample); - let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; @@ -183,7 +194,6 @@ impl AudioDecoder for GStreamerAudioDecoder { callbacks_.error(BackendError::Caps( "Could not get caps from sample", )); - let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; @@ -193,7 +203,6 @@ impl AudioDecoder for GStreamerAudioDecoder { audio_info } else { callbacks_.error(BackendError::AudioInfoFailed); - let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; assert_eq!(audio_info.channels(), 1); @@ -202,7 +211,6 @@ impl AudioDecoder for GStreamerAudioDecoder { positions } else { callbacks_.error(BackendError::AudioInfoFailed); - let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; @@ -213,7 +221,6 @@ impl AudioDecoder for GStreamerAudioDecoder { map } else { callbacks_.error(BackendError::BufferReadError); - let _ = pipeline_.set_state(gst::State::Null); return gst::FlowReturn::Error; }; let progress = Box::new(GStreamerAudioDecoderProgress(map)); @@ -225,7 +232,6 @@ impl AudioDecoder for GStreamerAudioDecoder { }) .eos(move |_| { callbacks__.eos(); - let _ = pipeline__.set_state(gst::State::Null); }) .build(), ); @@ -291,41 +297,37 @@ impl AudioDecoder for GStreamerAudioDecoder { if let Err(e) = insert_deinterleave() { callbacks.error(e); + let _ = sender.lock().unwrap().send(()); } }); appsrc.set_property_format(gst::Format::Bytes); appsrc.set_property_block(true); - let pipeline_ = pipeline.downgrade(); - let callbacks_ = callbacks.clone(); + let bus = match pipeline.get_bus() { + Some(bus) => bus, + None => { + callbacks.error(BackendError::PipelineFailed( + "Pipeline without bus. Shouldn't happen!", + )); + let _ = sender.lock().unwrap().send(()); + return; + } + }; - Builder::new() - .name("GStreamer pipeline bus".to_owned()) - .spawn(move || { - let callbacks = &callbacks_; - let pipeline = match pipeline_.upgrade() { - Some(pipeline) => pipeline, - None => return callbacks.error(BackendError::PipelineFailed("upgrade")), - }; - let bus = pipeline - .get_bus() - .expect("Pipeline without bus. Shouldn't happen!"); - - while let Some(msg) = bus.timed_pop(gst::CLOCK_TIME_NONE) { - match msg.view() { - MessageView::Eos(..) => break, - MessageView::Error(e) => { - callbacks.error(BackendError::PipelineBusError( - e.get_debug().unwrap_or("Unknown".to_owned()), - )); - break; - } - _ => (), - } + let callbacks_ = callbacks.clone(); + bus.set_sync_handler(move |_, msg| { + match msg.view() { + MessageView::Error(e) => { + callbacks_.error(BackendError::PipelineBusError( + e.get_debug().unwrap_or("Unknown".to_owned()), + )); + let _ = sender.lock().unwrap().send(()); } - }) - .unwrap(); + _ => (), + } + gst::BusSyncReply::Drop + }); if pipeline .set_state(gst::State::Playing) @@ -356,5 +358,9 @@ impl AudioDecoder for GStreamerAudioDecoder { let _ = appsrc.push_buffer(buffer); } let _ = appsrc.end_of_stream(); + + // Wait until we get an error or EOS. + receiver.recv().unwrap(); + let _ = pipeline.set_state(gst::State::Null); } }