diff --git a/Cargo.lock b/Cargo.lock index 7758842697..0492d948e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,6 +197,15 @@ dependencies = [ "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "chrono" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "clap" version = "2.31.2" @@ -972,6 +981,16 @@ dependencies = [ "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "num" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", + "num-iter 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num-integer" version = "0.1.38" @@ -1957,6 +1976,7 @@ dependencies = [ "base64 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "core-graphics 0.17.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2070,6 +2090,7 @@ dependencies = [ "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" "checksum cgl 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "55e7ec0b74fe5897894cbc207092c577e87c52f8a59e8ca8d97ef37551f60a49" +"checksum chrono 0.2.25 (registry+https://github.com/rust-lang/crates.io-index)" = "9213f7cd7c27e95c2b57c49f0e69b1ea65b27138da84a170133fd21b07659c00" "checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cmake 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "95470235c31c726d72bf2e1f421adc1e65b9d561bf5529612cbe1a72da1467b3" @@ -2155,6 +2176,7 @@ dependencies = [ "checksum nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" "checksum nom 5.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b471253da97532da4b61552249c521e01e736071f71c1a4f7ebbfbf0a06aad6" +"checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" "checksum num-integer 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)" = "6ac0ea58d64a89d9d6b7688031b3be9358d6c919badcf7fbb0527ccfd891ee45" "checksum num-iter 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "af3fdbbc3291a5464dc57b03860ec37ca6bf915ed6ee385e7c6c052c422b2124" "checksum num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4e96f040177bb3da242b5b1ecf3f54b5d5af3efbbfb18608977a5d2767b22f10" diff --git a/webrender/src/renderer.rs b/webrender/src/renderer.rs index 6e482d7fc8..ca4cc4714c 100644 --- a/webrender/src/renderer.rs +++ b/webrender/src/renderer.rs @@ -2434,6 +2434,7 @@ impl Renderer { } else if device.supports_extension("GL_EXT_debug_marker") { GpuDebugMethod::MarkerEXT } else { + println!("Warning: asking to enable_gpu_markers but no supporting extension was found"); GpuDebugMethod::None }; diff --git a/wrench/Cargo.toml b/wrench/Cargo.toml index 5f16b129d0..933c14b17f 100644 --- a/wrench/Cargo.toml +++ b/wrench/Cargo.toml @@ -21,6 +21,7 @@ yaml-rust = "0.4" serde_json = "1.0" ron = "0.1.5" time = "0.1" +chrono = "0.2" crossbeam = "0.2" osmesa-sys = { version = "0.1.2", optional = true } osmesa-src = { git = "https://github.com/servo/osmesa-src", optional = true } diff --git a/wrench/src/args.yaml b/wrench/src/args.yaml index 7080b1bbdb..b96b853b30 100644 --- a/wrench/src/args.yaml +++ b/wrench/src/args.yaml @@ -175,6 +175,27 @@ subcommands: help: name of the file to save benchmarks to required: true index: 1 + - benchmark: + help: benchmark list filename (default is benchmarks/benchmarks.list) + required: false + index: 2 + - auto-filename: + long: auto-filename + help: generate output filename from date and time (user provided filename is the directory prefix) + required: false + - csv: + long: csv + help: save benchmark results as .csv (default is json) + required: false + - warmup_frames: + long: warmup_frames + takes_value: true + help: number of frames to skip before recording timings + required: false + - sample_count: + long: sample_count + takes_value: true + help: number of samples to capture - compare_perf: about: compare two benchmark files args: diff --git a/wrench/src/main.rs b/wrench/src/main.rs index 4ffd0eddc2..8150f54248 100644 --- a/wrench/src/main.rs +++ b/wrench/src/main.rs @@ -710,10 +710,35 @@ fn main() { // Perf mode wants to benchmark the total cost of drawing // a new displaty list each frame. wrench.rebuild_display_lists = true; - let harness = PerfHarness::new(&mut wrench, &mut window, rx.unwrap()); - let base_manifest = Path::new("benchmarks/benchmarks.list"); - let filename = subargs.value_of("filename").unwrap(); - harness.run(base_manifest, filename); + + let as_csv = subargs.is_present("csv"); + let auto_filename = subargs.is_present("auto-filename"); + + let warmup_frames = subargs.value_of("warmup_frames").map(|s| s.parse().unwrap()); + let sample_count = subargs.value_of("sample_count").map(|s| s.parse().unwrap()); + + let harness = PerfHarness::new(&mut wrench, + &mut window, + rx.unwrap(), + warmup_frames, + sample_count); + + let benchmark = match subargs.value_of("benchmark") { + Some(path) => path, + None => "benchmarks/benchmarks.list" + }; + println!("Benchmark: {}", benchmark); + let base_manifest = Path::new(benchmark); + + let mut filename = subargs.value_of("filename").unwrap().to_string(); + if auto_filename { + let timestamp = chrono::Local::now().format("%Y-%m-%d-%H-%M-%S"); + filename.push_str( + &format!("/wrench-perf-{}.{}", + timestamp, + if as_csv { "csv" } else { "json" })); + } + harness.run(base_manifest, &filename, as_csv); return; } else if let Some(subargs) = args.subcommand_matches("compare_perf") { let first_filename = subargs.value_of("first_filename").unwrap(); diff --git a/wrench/src/perf.rs b/wrench/src/perf.rs index 045d85ba84..bd41ba6016 100644 --- a/wrench/src/perf.rs +++ b/wrench/src/perf.rs @@ -13,6 +13,8 @@ use std::path::{Path, PathBuf}; use std::sync::mpsc::Receiver; use crate::wrench::{Wrench, WrenchThing}; use crate::yaml_frame_reader::YamlFrameReader; +use webrender::DebugFlags; +use webrender::api::DebugCommand; const COLOR_DEFAULT: &str = "\x1b[0m"; const COLOR_RED: &str = "\x1b[31m"; @@ -71,15 +73,45 @@ impl BenchmarkManifest { } } +#[derive(Clone, Serialize, Deserialize)] +struct TestProfileRange { + min: u64, + avg: u64, + max: u64, +} + #[derive(Clone, Serialize, Deserialize)] struct TestProfile { name: String, - backend_time_ns: u64, - composite_time_ns: u64, - paint_time_ns: u64, + backend_time_ns: TestProfileRange, + composite_time_ns: TestProfileRange, + paint_time_ns: TestProfileRange, draw_calls: usize, } +impl TestProfile { + fn csv_header() -> String { + "name,\ + backend_time_ns min, avg, max,\ + composite_time_ns min, avg, max,\ + paint_time_ns min, avg, max,\ + draw_calls\n".to_string() + } + + fn convert_to_csv(&self) -> String { + format!("{},\ + {},{},{},\ + {},{},{},\ + {},{},{},\ + {}\n", + self.name, + self.backend_time_ns.min, self.backend_time_ns.avg, self.backend_time_ns.max, + self.composite_time_ns.min, self.composite_time_ns.avg, self.composite_time_ns.max, + self.paint_time_ns.min, self.paint_time_ns.avg, self.paint_time_ns.max, + self.draw_calls) + } +} + #[derive(Serialize, Deserialize)] struct Profile { tests: Vec, @@ -94,11 +126,18 @@ impl Profile { self.tests.push(profile); } - fn save(&self, filename: &str) { + fn save(&self, filename: &str, as_csv: bool) { let mut file = File::create(&filename).unwrap(); - let s = serde_json::to_string_pretty(self).unwrap(); - file.write_all(&s.into_bytes()).unwrap(); - file.write_all(b"\n").unwrap(); + if as_csv { + file.write_all(&TestProfile::csv_header().into_bytes()).unwrap(); + for test in &self.tests { + file.write_all(&test.convert_to_csv().into_bytes()).unwrap(); + } + } else { + let s = serde_json::to_string_pretty(self).unwrap(); + file.write_all(&s.into_bytes()).unwrap(); + file.write_all(b"\n").unwrap(); + } } fn load(filename: &str) -> Profile { @@ -125,14 +164,26 @@ pub struct PerfHarness<'a> { wrench: &'a mut Wrench, window: &'a mut WindowWrapper, rx: Receiver, + warmup_frames: usize, + sample_count: usize, } impl<'a> PerfHarness<'a> { - pub fn new(wrench: &'a mut Wrench, window: &'a mut WindowWrapper, rx: Receiver) -> Self { - PerfHarness { wrench, window, rx } + pub fn new(wrench: &'a mut Wrench, + window: &'a mut WindowWrapper, + rx: Receiver, + warmup_frames: Option, + sample_count: Option) -> Self { + PerfHarness { + wrench, + window, + rx, + warmup_frames: warmup_frames.unwrap_or(0usize), + sample_count: sample_count.unwrap_or(MIN_SAMPLE_COUNT), + } } - pub fn run(mut self, base_manifest: &Path, filename: &str) { + pub fn run(mut self, base_manifest: &Path, filename: &str, as_csv: bool) { let manifest = BenchmarkManifest::new(base_manifest); let mut profile = Profile::new(); @@ -142,7 +193,7 @@ impl<'a> PerfHarness<'a> { profile.add(stats); } - profile.save(filename); + profile.save(filename, as_csv); } fn render_yaml(&mut self, filename: &Path) -> TestProfile { @@ -153,25 +204,40 @@ impl<'a> PerfHarness<'a> { let mut cpu_frame_profiles = Vec::new(); let mut gpu_frame_profiles = Vec::new(); - while cpu_frame_profiles.len() < MIN_SAMPLE_COUNT || - gpu_frame_profiles.len() < MIN_SAMPLE_COUNT + let mut debug_flags = DebugFlags::empty(); + debug_flags.set(DebugFlags::GPU_TIME_QUERIES | DebugFlags::GPU_SAMPLE_QUERIES, true); + self.wrench.api.send_debug_cmd(DebugCommand::SetFlags(debug_flags)); + + let mut frame_count = 0; + + while cpu_frame_profiles.len() < self.sample_count || + gpu_frame_profiles.len() < self.sample_count { reader.do_frame(self.wrench); self.rx.recv().unwrap(); self.wrench.render(); self.window.swap_buffers(); let (cpu_profiles, gpu_profiles) = self.wrench.get_frame_profiles(); - cpu_frame_profiles.extend(cpu_profiles); - gpu_frame_profiles.extend(gpu_profiles); + if frame_count >= self.warmup_frames { + cpu_frame_profiles.extend(cpu_profiles); + gpu_frame_profiles.extend(gpu_profiles); + } + frame_count = frame_count + 1; } // Ensure the draw calls match in every sample. let draw_calls = cpu_frame_profiles[0].draw_calls; - assert!( + let draw_calls_same = cpu_frame_profiles .iter() - .all(|s| s.draw_calls == draw_calls) - ); + .all(|s| s.draw_calls == draw_calls); + + // this can be normal in cases where some elements are cached (eg. linear + // gradients), but print a warning in case it's not (which could make the + // benchmark produce unexpected results). + if !draw_calls_same { + println!("Warning: not every frame has the same number of draw calls"); + } let composite_time_ns = extract_sample(&mut cpu_frame_profiles, |a| a.composite_time_ns); let paint_time_ns = extract_sample(&mut gpu_frame_profiles, |a| a.paint_time_ns); @@ -187,7 +253,9 @@ impl<'a> PerfHarness<'a> { } } -fn extract_sample(profiles: &mut [T], f: F) -> u64 +// returns min, average, max, after removing the lowest and highest SAMPLE_EXCLUDE_COUNT +// samples (each). +fn extract_sample(profiles: &mut [T], f: F) -> TestProfileRange where F: Fn(&T) -> u64, { @@ -195,7 +263,11 @@ where samples.sort(); let useful_samples = &samples[SAMPLE_EXCLUDE_COUNT .. samples.len() - SAMPLE_EXCLUDE_COUNT]; let total_time: u64 = useful_samples.iter().sum(); - total_time / useful_samples.len() as u64 + TestProfileRange { + min: useful_samples[0], + avg: total_time / useful_samples.len() as u64, + max: useful_samples[useful_samples.len()-1] + } } fn select_color(base: f32, value: f32) -> &'static str { @@ -239,11 +311,11 @@ pub fn compare(first_filename: &str, second_filename: &str) { let test0 = &map0[test_name]; let test1 = &map1[test_name]; - let composite_time0 = test0.composite_time_ns as f32 / 1000000.0; - let composite_time1 = test1.composite_time_ns as f32 / 1000000.0; + let composite_time0 = test0.composite_time_ns.avg as f32 / 1000000.0; + let composite_time1 = test1.composite_time_ns.avg as f32 / 1000000.0; - let paint_time0 = test0.paint_time_ns as f32 / 1000000.0; - let paint_time1 = test1.paint_time_ns as f32 / 1000000.0; + let paint_time0 = test0.paint_time_ns.avg as f32 / 1000000.0; + let paint_time1 = test1.paint_time_ns.avg as f32 / 1000000.0; let draw_calls_color = if test0.draw_calls == test1.draw_calls { COLOR_DEFAULT