From 69e68634d76a3f62db59b33d402d75b1dfa16507 Mon Sep 17 00:00:00 2001 From: ms2300 Date: Sun, 23 Sep 2018 21:12:51 -0700 Subject: [PATCH 1/2] Initial implementation of asynchronous blob url fetching --- components/net/blob_loader.rs | 72 +++---------- components/net/fetch/methods.rs | 19 ++-- components/net/filemanager_thread.rs | 154 ++++++++++++++++++++++++++- 3 files changed, 172 insertions(+), 73 deletions(-) diff --git a/components/net/blob_loader.rs b/components/net/blob_loader.rs index 1fe05cc1c20b..bf4c40187dad 100644 --- a/components/net/blob_loader.rs +++ b/components/net/blob_loader.rs @@ -2,25 +2,24 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +use fetch::methods::DoneChannel; use filemanager_thread::FileManager; -use hyper::header::{Charset, ContentLength, ContentType, Headers}; -use hyper::header::{ContentDisposition, DispositionParam, DispositionType}; -use ipc_channel::ipc; -use mime::{Attr, Mime}; use net_traits::NetworkError; use net_traits::blob_url_store::parse_blob_url; -use net_traits::filemanager_thread::ReadFileProgress; +use net_traits::response::{Response, ResponseBody}; use servo_url::ServoUrl; +use std::sync::mpsc::channel; // TODO: Check on GET // https://w3c.github.io/FileAPI/#requestResponseModel /// https://fetch.spec.whatwg.org/#concept-basic-fetch (partial) -// TODO: make async. -pub fn load_blob_sync +pub fn load_blob_async (url: ServoUrl, - filemanager: FileManager) - -> Result<(Headers, Vec), NetworkError> { + filemanager: FileManager, + response: &Response, + done_chan: &mut DoneChannel) + -> Result<(), NetworkError> { let (id, origin) = match parse_blob_url(&url) { Ok((id, origin)) => (id, origin), Err(()) => { @@ -29,56 +28,11 @@ pub fn load_blob_sync } }; - let (sender, receiver) = ipc::channel().unwrap(); + let (sender, receiver) = channel(); + *done_chan = Some((sender.clone(), receiver)); + *response.body.lock().unwrap() = ResponseBody::Receiving(vec![]); let check_url_validity = true; - filemanager.read_file(sender, id, check_url_validity, origin); + filemanager.fetch_file(sender, id, check_url_validity, origin, response); - let blob_buf = match receiver.recv().unwrap() { - Ok(ReadFileProgress::Meta(blob_buf)) => blob_buf, - Ok(_) => { - return Err(NetworkError::Internal("Invalid filemanager reply".to_string())); - } - Err(e) => { - return Err(NetworkError::Internal(format!("{:?}", e))); - } - }; - - let content_type: Mime = blob_buf.type_string.parse().unwrap_or(mime!(Text / Plain)); - let charset = content_type.get_param(Attr::Charset); - - let mut headers = Headers::new(); - - if let Some(name) = blob_buf.filename { - let charset = charset.and_then(|c| c.as_str().parse().ok()); - headers.set(ContentDisposition { - disposition: DispositionType::Inline, - parameters: vec![ - DispositionParam::Filename(charset.unwrap_or(Charset::Us_Ascii), - None, name.as_bytes().to_vec()) - ] - }); - } - - // Basic fetch, Step 4. - headers.set(ContentLength(blob_buf.size as u64)); - // Basic fetch, Step 5. - headers.set(ContentType(content_type.clone())); - - let mut bytes = blob_buf.bytes; - loop { - match receiver.recv().unwrap() { - Ok(ReadFileProgress::Partial(ref mut new_bytes)) => { - bytes.append(new_bytes); - } - Ok(ReadFileProgress::EOF) => { - return Ok((headers, bytes)); - } - Ok(_) => { - return Err(NetworkError::Internal("Invalid filemanager reply".to_string())); - } - Err(e) => { - return Err(NetworkError::Internal(format!("{:?}", e))); - } - } - } + Ok(()) } diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 931a6e778e7e..f7171f85fb7b 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use blob_loader::load_blob_sync; +use blob_loader::load_blob_async; use data_loader::decode; use devtools_traits::DevtoolsControlMsg; use fetch::cors_cache::CorsCache; @@ -553,18 +553,11 @@ fn scheme_fetch(request: &mut Request, return Response::network_error(NetworkError::Internal("Unexpected method for blob".into())); } - match load_blob_sync(url.clone(), context.filemanager.clone()) { - Ok((headers, bytes)) => { - let mut response = Response::new(url); - response.headers = headers; - *response.body.lock().unwrap() = ResponseBody::Done(bytes); - response - }, - Err(e) => { - debug!("Failed to load {}: {:?}", url, e); - Response::network_error(e) - }, - } + let mut response = Response::new(url); + + load_blob_async(url.clone(), context.filemanager.clone(), &response, done_chan); + + response }, "ftp" => { diff --git a/components/net/filemanager_thread.rs b/components/net/filemanager_thread.rs index 3dda46a979d5..d018ca8699cc 100644 --- a/components/net/filemanager_thread.rs +++ b/components/net/filemanager_thread.rs @@ -4,17 +4,23 @@ use embedder_traits::{EmbedderMsg, EmbedderProxy, FilterPattern}; use ipc_channel::ipc::{self, IpcSender}; +use fetch::methods::Data; +use hyper::header::{Charset, ContentLength, ContentType, Headers}; +use hyper::header::{ContentDisposition, DispositionParam, DispositionType}; +use mime::{Attr, Mime}; use mime_guess::guess_mime_type_opt; +use net_traits::NetworkError; use net_traits::blob_url_store::{BlobBuf, BlobURLStoreError}; use net_traits::filemanager_thread::{FileManagerResult, FileManagerThreadMsg, FileOrigin}; use net_traits::filemanager_thread::{FileManagerThreadError, ReadFileProgress, RelativePos, SelectedFile}; +use net_traits::response::{Response, ResponseBody}; use servo_config::prefs::PREFS; use std::collections::HashMap; use std::fs::File; use std::io::{Read, Seek, SeekFrom}; use std::ops::Index; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, RwLock, Mutex, mpsc}; use std::sync::atomic::{self, AtomicBool, AtomicUsize, Ordering}; use std::thread; use url::Url; @@ -85,6 +91,19 @@ impl FileManager { }).expect("Thread spawning failed"); } + pub fn fetch_file(&self, + sender: mpsc::Sender, + id: Uuid, + check_url_validity: bool, + origin: FileOrigin, + response: &Response) { + let store = self.store.clone(); + let mut res_body = response.body.clone(); + thread::Builder::new().name("read file".to_owned()).spawn(move || { + store.try_fetch_file(&sender, id, check_url_validity, origin, response, res_body) + }).expect("Thread spawning failed"); + } + pub fn promote_memory(&self, blob_buf: BlobBuf, set_valid: bool, @@ -414,6 +433,94 @@ impl FileManagerStore { self.get_blob_buf(sender, &id, &origin_in, RelativePos::full_range(), check_url_validity) } + fn fetch_blob_buf(&self, sender: &mpsc::Sender, + id: &Uuid, origin_in: &FileOrigin, rel_pos: RelativePos, + check_url_validity: bool, response: &Response, res_body: Arc>) -> Result<(), BlobURLStoreError> { + let mut bytes = vec![]; + let file_impl = self.get_impl(id, origin_in, check_url_validity)?; + match file_impl { + FileImpl::Memory(buf) => { + let range = rel_pos.to_abs_range(buf.size as usize); + let blob_buf = BlobBuf { + filename: None, + type_string: buf.type_string, + size: range.len() as u64, + bytes: buf.bytes.index(range).to_vec(), + }; + + let content_type: Mime = blob_buf.type_string.parse().unwrap_or(mime!(Text / Plain)); + let charset = content_type.get_param(Attr::Charset); + let mut headers = Headers::new(); + + if let Some(name) = blob_buf.filename { + let charset = charset.and_then(|c| c.as_str().parse().ok()); + headers.set(ContentDisposition { + disposition: DispositionType::Inline, + parameters: vec![ + DispositionParam::Filename(charset.unwrap_or(Charset::Us_Ascii), + None, name.as_bytes().to_vec()) + ] + }); + } + + headers.set(ContentLength(blob_buf.size as u64)); + headers.set(ContentType(content_type.clone())); + + bytes.extend_from_slice(&blob_buf.bytes); + + response.headers = headers; + *res_body.lock().unwrap() = ResponseBody::Done(bytes); + let _ = sender.send(Data::Done); + Ok(()) + } + FileImpl::MetaDataOnly(metadata) => { + /* XXX: Snapshot state check (optional) https://w3c.github.io/FileAPI/#snapshot-state. + Concretely, here we create another file, and this file might not + has the same underlying file state (meta-info plus content) as the time + create_entry is called. + */ + + let opt_filename = metadata.path.file_name() + .and_then(|osstr| osstr.to_str()) + .map(|s| s.to_string()); + + let mime = guess_mime_type_opt(metadata.path.clone()); + let range = rel_pos.to_abs_range(metadata.size as usize); + + let mut file = File::open(&metadata.path) + .map_err(|e| BlobURLStoreError::External(e.to_string()))?; + let seeked_start = file.seek(SeekFrom::Start(range.start as u64)) + .map_err(|e| BlobURLStoreError::External(e.to_string()))?; + + if seeked_start == (range.start as u64) { + let type_string = match mime { + Some(x) => format!("{}", x), + None => "".to_string(), + }; + + chunked_fetch(sender, &mut file, range.len(), opt_filename, + type_string, response, res_body, &mut bytes); + Ok(()) + } else { + Err(BlobURLStoreError::InvalidEntry) + } + } + FileImpl::Sliced(parent_id, inner_rel_pos) => { + // Next time we don't need to check validity since + // we have already done that for requesting URL if necessary + self.fetch_blob_buf(sender, &parent_id, origin_in, + rel_pos.slice_inner(&inner_rel_pos), false, response, res_body) + } + } + } + + fn try_fetch_file(&self, sender: &mpsc::Sender, id: Uuid, check_url_validity: bool, + origin_in: FileOrigin, response: &Response, res_body: Arc>) + -> Result<(), BlobURLStoreError> { + self.fetch_blob_buf(sender, &id, &origin_in, RelativePos::full_range(), check_url_validity, + response, res_body) + } + fn dec_ref(&self, id: &Uuid, origin_in: &FileOrigin) -> Result<(), BlobURLStoreError> { let (do_remove, opt_parent_id) = match self.entries.read().unwrap().get(id) { Some(entry) => { @@ -562,3 +669,48 @@ fn chunked_read(sender: &IpcSender>, } } } + +fn chunked_fetch(sender: &mpsc::Sender, + file: &mut File, size: usize, opt_filename: Option, + type_string: String, response: &Response, res_body: Arc>, bytes: &mut Vec) { + // First chunk + let mut buf = vec![0; CHUNK_SIZE]; + match file.read(&mut buf) { + Ok(n) => { + buf.truncate(n); + let blob_buf = BlobBuf { + filename: opt_filename, + type_string: type_string, + size: size as u64, + bytes: buf, + }; + bytes.extend_from_slice(&blob_buf.bytes); + let _ = sender.send(Data::Payload(blob_buf.bytes)); + } + Err(_) => { + *response = Response::network_error(NetworkError::Internal("Opening file failed".into())); + return; + } + } + + // Send the remaining chunks + loop { + let mut buf = vec![0; CHUNK_SIZE]; + match file.read(&mut buf) { + Ok(0) => { + *res_body.lock().unwrap() = ResponseBody::Done(bytes.to_vec()); + let _ = sender.send(Data::Done); + return; + } + Ok(n) => { + buf.truncate(n); + bytes.extend_from_slice(&buf); + let _ = sender.send(Data::Payload(buf)); + } + Err(e) => { + *response = Response::network_error(NetworkError::Internal("Opening file failed".into())); + return; + } + } + } +} From afd7f6e17471d1098be1b539dcccde8fd3c20a08 Mon Sep 17 00:00:00 2001 From: ms2300 Date: Tue, 25 Sep 2018 14:24:43 -0700 Subject: [PATCH 2/2] Blob url's changes now build and test --- components/net/blob_loader.rs | 13 +++++----- components/net/fetch/methods.rs | 8 ++---- components/net/filemanager_thread.rs | 38 +++++++++++++++------------- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/components/net/blob_loader.rs b/components/net/blob_loader.rs index bf4c40187dad..788b1332756e 100644 --- a/components/net/blob_loader.rs +++ b/components/net/blob_loader.rs @@ -8,7 +8,7 @@ use net_traits::NetworkError; use net_traits::blob_url_store::parse_blob_url; use net_traits::response::{Response, ResponseBody}; use servo_url::ServoUrl; -use std::sync::mpsc::channel; +use servo_channel::channel; // TODO: Check on GET // https://w3c.github.io/FileAPI/#requestResponseModel @@ -17,22 +17,21 @@ use std::sync::mpsc::channel; pub fn load_blob_async (url: ServoUrl, filemanager: FileManager, - response: &Response, done_chan: &mut DoneChannel) - -> Result<(), NetworkError> { + -> Response { let (id, origin) = match parse_blob_url(&url) { Ok((id, origin)) => (id, origin), Err(()) => { - let e = format!("Invalid blob URL format {:?}", url); - return Err(NetworkError::Internal(e)); + return Response::network_error(NetworkError::Internal("Invalid blob url".into())); } }; + let mut response = Response::new(url); let (sender, receiver) = channel(); *done_chan = Some((sender.clone(), receiver)); *response.body.lock().unwrap() = ResponseBody::Receiving(vec![]); let check_url_validity = true; - filemanager.fetch_file(sender, id, check_url_validity, origin, response); + filemanager.fetch_file(sender, id, check_url_validity, origin, &mut response); - Ok(()) + response } diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index f7171f85fb7b..3e3459307f9e 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -552,12 +552,8 @@ fn scheme_fetch(request: &mut Request, if request.method != Method::Get { return Response::network_error(NetworkError::Internal("Unexpected method for blob".into())); } - - let mut response = Response::new(url); - - load_blob_async(url.clone(), context.filemanager.clone(), &response, done_chan); - - response + + load_blob_async(url.clone(), context.filemanager.clone(), done_chan) }, "ftp" => { diff --git a/components/net/filemanager_thread.rs b/components/net/filemanager_thread.rs index d018ca8699cc..28857e25a503 100644 --- a/components/net/filemanager_thread.rs +++ b/components/net/filemanager_thread.rs @@ -14,13 +14,14 @@ use net_traits::blob_url_store::{BlobBuf, BlobURLStoreError}; use net_traits::filemanager_thread::{FileManagerResult, FileManagerThreadMsg, FileOrigin}; use net_traits::filemanager_thread::{FileManagerThreadError, ReadFileProgress, RelativePos, SelectedFile}; use net_traits::response::{Response, ResponseBody}; +use servo_channel; use servo_config::prefs::PREFS; use std::collections::HashMap; use std::fs::File; use std::io::{Read, Seek, SeekFrom}; use std::ops::Index; use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock, Mutex, mpsc}; +use std::sync::{Arc, RwLock}; use std::sync::atomic::{self, AtomicBool, AtomicUsize, Ordering}; use std::thread; use url::Url; @@ -92,15 +93,15 @@ impl FileManager { } pub fn fetch_file(&self, - sender: mpsc::Sender, + sender: servo_channel::Sender, id: Uuid, check_url_validity: bool, origin: FileOrigin, - response: &Response) { + response: &mut Response) { let store = self.store.clone(); - let mut res_body = response.body.clone(); + let mut r2 = response.clone(); thread::Builder::new().name("read file".to_owned()).spawn(move || { - store.try_fetch_file(&sender, id, check_url_validity, origin, response, res_body) + store.try_fetch_file(&sender, id, check_url_validity, origin, &mut r2) }).expect("Thread spawning failed"); } @@ -433,9 +434,9 @@ impl FileManagerStore { self.get_blob_buf(sender, &id, &origin_in, RelativePos::full_range(), check_url_validity) } - fn fetch_blob_buf(&self, sender: &mpsc::Sender, + fn fetch_blob_buf(&self, sender: &servo_channel::Sender, id: &Uuid, origin_in: &FileOrigin, rel_pos: RelativePos, - check_url_validity: bool, response: &Response, res_body: Arc>) -> Result<(), BlobURLStoreError> { + check_url_validity: bool, response: &mut Response) -> Result<(), BlobURLStoreError> { let mut bytes = vec![]; let file_impl = self.get_impl(id, origin_in, check_url_validity)?; match file_impl { @@ -469,7 +470,7 @@ impl FileManagerStore { bytes.extend_from_slice(&blob_buf.bytes); response.headers = headers; - *res_body.lock().unwrap() = ResponseBody::Done(bytes); + *response.body.lock().unwrap() = ResponseBody::Done(bytes); let _ = sender.send(Data::Done); Ok(()) } @@ -499,7 +500,7 @@ impl FileManagerStore { }; chunked_fetch(sender, &mut file, range.len(), opt_filename, - type_string, response, res_body, &mut bytes); + type_string, response, &mut bytes); Ok(()) } else { Err(BlobURLStoreError::InvalidEntry) @@ -509,16 +510,15 @@ impl FileManagerStore { // Next time we don't need to check validity since // we have already done that for requesting URL if necessary self.fetch_blob_buf(sender, &parent_id, origin_in, - rel_pos.slice_inner(&inner_rel_pos), false, response, res_body) + rel_pos.slice_inner(&inner_rel_pos), false, response) } } } - fn try_fetch_file(&self, sender: &mpsc::Sender, id: Uuid, check_url_validity: bool, - origin_in: FileOrigin, response: &Response, res_body: Arc>) + fn try_fetch_file(&self, sender: &servo_channel::Sender, id: Uuid, check_url_validity: bool, + origin_in: FileOrigin, response: &mut Response) -> Result<(), BlobURLStoreError> { - self.fetch_blob_buf(sender, &id, &origin_in, RelativePos::full_range(), check_url_validity, - response, res_body) + self.fetch_blob_buf(sender, &id, &origin_in, RelativePos::full_range(), check_url_validity, response) } fn dec_ref(&self, id: &Uuid, origin_in: &FileOrigin) -> Result<(), BlobURLStoreError> { @@ -670,9 +670,9 @@ fn chunked_read(sender: &IpcSender>, } } -fn chunked_fetch(sender: &mpsc::Sender, +fn chunked_fetch(sender: &servo_channel::Sender, file: &mut File, size: usize, opt_filename: Option, - type_string: String, response: &Response, res_body: Arc>, bytes: &mut Vec) { + type_string: String, response: &mut Response, bytes: &mut Vec) { // First chunk let mut buf = vec![0; CHUNK_SIZE]; match file.read(&mut buf) { @@ -684,7 +684,9 @@ fn chunked_fetch(sender: &mpsc::Sender, size: size as u64, bytes: buf, }; + bytes.extend_from_slice(&blob_buf.bytes); + let _ = sender.send(Data::Payload(blob_buf.bytes)); } Err(_) => { @@ -698,7 +700,7 @@ fn chunked_fetch(sender: &mpsc::Sender, let mut buf = vec![0; CHUNK_SIZE]; match file.read(&mut buf) { Ok(0) => { - *res_body.lock().unwrap() = ResponseBody::Done(bytes.to_vec()); + *response.body.lock().unwrap() = ResponseBody::Done(bytes.to_vec()); let _ = sender.send(Data::Done); return; } @@ -707,7 +709,7 @@ fn chunked_fetch(sender: &mpsc::Sender, bytes.extend_from_slice(&buf); let _ = sender.send(Data::Payload(buf)); } - Err(e) => { + Err(_) => { *response = Response::network_error(NetworkError::Internal("Opening file failed".into())); return; }