From 33d88e496c4913036abe5d7b7a057b77d165d5c6 Mon Sep 17 00:00:00 2001 From: o0Ignition0o Date: Sat, 3 Feb 2018 23:13:01 +0100 Subject: [PATCH 1/3] First tryouts on a bisect command. --- python/mach_bootstrap.py | 6 +++ python/servo/bisect_commands.py | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 python/servo/bisect_commands.py diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 9a2a79c6d6ae..6e7183a396c9 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -24,6 +24,7 @@ os.path.join('python', 'servo', 'post_build_commands.py'), os.path.join('python', 'servo', 'package_commands.py'), os.path.join('python', 'servo', 'devenv_commands.py'), + os.path.join('python', 'servo', 'bisect_commands.py'), ] CATEGORIES = { @@ -67,6 +68,11 @@ 'long': 'Potent potables and assorted snacks.', 'priority': 10, }, + 'bisect': { + 'short': 'Bisect Commands', + 'long': 'Bisect with the nightly versions', + 'priority': 5, + }, 'disabled': { 'short': 'Disabled', 'long': 'The disabled commands are hidden by default. Use -v to display them. These commands are unavailable ' diff --git a/python/servo/bisect_commands.py b/python/servo/bisect_commands.py new file mode 100644 index 000000000000..0ded8c60dbc4 --- /dev/null +++ b/python/servo/bisect_commands.py @@ -0,0 +1,81 @@ +# Copyright 2013 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +from __future__ import print_function, unicode_literals + +import os +import os.path as path +import subprocess +import time +from shutil import copytree, rmtree, copy2 + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) + +from servo.command_base import ( + CommandBase, + check_call, check_output, BIN_SUFFIX, + is_linux, is_windows, is_macosx, set_osmesa_env, + get_browserhtml_path, +) + + +def read_file(filename, if_exists=False): + if if_exists and not path.exists(filename): + return None + with open(filename) as f: + return f.read() + + +@CommandProvider +class BisectCommands(CommandBase): + @Command('bisect', + description='Run Servo using downloaded nightlies', + category='bisect') + @CommandArgument('--release', '-r', action='store_true', + help='Run the release build') + @CommandArgument('--dev', '-d', action='store_true', + help='Run the dev build') + @CommandArgument('--android', action='store_true', default=None, + help='Run on an Android device through `adb shell`') + @CommandArgument('--debug', action='store_true', + help='Enable the debugger. Not specifying a ' + '--debugger option will result in the default ' + 'debugger being used. The following arguments ' + 'have no effect without this.') + @CommandArgument('--debugger', default=None, type=str, + help='Name of debugger to use.') + @CommandArgument('--browserhtml', '-b', action='store_true', + help='Launch with Browser.html') + @CommandArgument('--headless', '-z', action='store_true', + help='Launch in headless mode') + @CommandArgument('--software', '-s', action='store_true', + help='Launch with software rendering') + @CommandArgument( + 'params', nargs='...', + help="Command-line arguments to be passed through to Servo") + def bisect(self, params, release=False, dev=False, android=None, debug=False, debugger=None, browserhtml=False, + headless=False, software=False): + # Run the command + SCRIPT_PATH = os.path.split(__file__)[0] + command_to_run = [os.path.abspath(os.path.join(SCRIPT_PATH, "..", "..", "mach"))] + command_to_run.extend(params) + print("Wrapping bisect around command {}".format(command_to_run)) + start = time.time() + proc = subprocess.Popen(command_to_run) + output = proc.communicate() + if proc.returncode: + print("Bisection on nightly {} was bad, trying older version".format("FOOBAR")) + else: + print("Bisection on nightly {} was good, trying newer version".format("FOOBAR11234")) + end = time.time() + print("Bisect completed in {} seconds.".format(round(end-start,2))) From 60395ef749f84051d589b9d116f211c76a9021e0 Mon Sep 17 00:00:00 2001 From: o0Ignition0o Date: Sun, 4 Feb 2018 00:20:49 +0100 Subject: [PATCH 2/3] WIP : Rough first tries to download and cache nightly versions. Lots of things are hardcoded for now in order to first make sure this works. --- python/servo/bisect_commands.py | 89 +++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/python/servo/bisect_commands.py b/python/servo/bisect_commands.py index 0ded8c60dbc4..7cca1a0c206d 100644 --- a/python/servo/bisect_commands.py +++ b/python/servo/bisect_commands.py @@ -13,7 +13,10 @@ import os.path as path import subprocess import time +import tarfile + from shutil import copytree, rmtree, copy2 +from servo.util import extract, download_file from mach.decorators import ( CommandArgument, @@ -21,12 +24,7 @@ Command, ) -from servo.command_base import ( - CommandBase, - check_call, check_output, BIN_SUFFIX, - is_linux, is_windows, is_macosx, set_osmesa_env, - get_browserhtml_path, -) +from servo.command_base import CommandBase def read_file(filename, if_exists=False): @@ -41,41 +39,66 @@ class BisectCommands(CommandBase): @Command('bisect', description='Run Servo using downloaded nightlies', category='bisect') - @CommandArgument('--release', '-r', action='store_true', - help='Run the release build') - @CommandArgument('--dev', '-d', action='store_true', - help='Run the dev build') - @CommandArgument('--android', action='store_true', default=None, - help='Run on an Android device through `adb shell`') - @CommandArgument('--debug', action='store_true', - help='Enable the debugger. Not specifying a ' - '--debugger option will result in the default ' - 'debugger being used. The following arguments ' - 'have no effect without this.') - @CommandArgument('--debugger', default=None, type=str, - help='Name of debugger to use.') - @CommandArgument('--browserhtml', '-b', action='store_true', - help='Launch with Browser.html') - @CommandArgument('--headless', '-z', action='store_true', - help='Launch in headless mode') - @CommandArgument('--software', '-s', action='store_true', - help='Launch with software rendering') @CommandArgument( 'params', nargs='...', help="Command-line arguments to be passed through to Servo") - def bisect(self, params, release=False, dev=False, android=None, debug=False, debugger=None, browserhtml=False, - headless=False, software=False): + def bisect(self, params): # Run the command SCRIPT_PATH = os.path.split(__file__)[0] - command_to_run = [os.path.abspath(os.path.join(SCRIPT_PATH, "..", "..", "mach"))] + command_to_run = [os.path.abspath( + os.path.join(SCRIPT_PATH, "..", "..", "mach"))] command_to_run.extend(params) - print("Wrapping bisect around command {}".format(command_to_run)) + + print("Wrapping bisect around command {}".format( + " ".join(command_to_run))) + bisect_start = time.time() + self.run_once_with_version(command_to_run, "2017-01-01T01-21-52Z") + bisect_end = time.time() + print("Bisect completed in {} seconds.".format( + round(bisect_end - bisect_start, 2))) + + def run_once_with_version(self, command, version): + fetch_version(version) + print("Running command {} with version {} ".format(command, version)) start = time.time() - proc = subprocess.Popen(command_to_run) + proc = subprocess.Popen(command) output = proc.communicate() if proc.returncode: - print("Bisection on nightly {} was bad, trying older version".format("FOOBAR")) + print( + "Bisection on nightly {} was bad, trying older version".format(version)) else: - print("Bisection on nightly {} was good, trying newer version".format("FOOBAR11234")) + print("Bisection on nightly {} was good, trying newer version".format( + "FOOBAR11234")) end = time.time() - print("Bisect completed in {} seconds.".format(round(end-start,2))) + print("Command {} with servo version {} completed in {} seconds.".format( + command, version, round(end - start, 2))) + + +def get_all_versions(os): + url = "https://servo-builds.s3.amazonaws.com/?list-type=2&prefix=nightly/linux" + + +def fetch_version(version): + # Download from + cache_folder = "./nightlies/" + build_name = "-servo-tech-demo" + extension = ".tar.gz" + file_name = version + build_name + destination = cache_folder + file_name + extension + version_folder = cache_folder + file_name + source = "https://servo-builds.s3.amazonaws.com/nightly/linux/" + file_name + extension + + if not os.path.exists(cache_folder): + os.mkdir(cache_folder) + + if os.path.isfile(destination): + print("The nightly version {} has already been downloaded.".format(version)) + else: + download_file(file_name,source, destination) + + if os.path.isdir(version_folder): + print("The version {} has already been extracted.".format(version)) + else: + print("Extracting to {}...".format(version_folder), end='') + with tarfile.open(destination, "r") as tar: + tar.extractall(version_folder) From 023520be9c8d6134878dd51fa7cf23e8865a7e8b Mon Sep 17 00:00:00 2001 From: o0Ignition0o Date: Sat, 8 Dec 2018 15:29:57 +0100 Subject: [PATCH 3/3] ./mach run -n on mac os. --- python/servo/command_base.py | 72 +++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 277f1d3aac12..6c193191fb0d 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -9,6 +9,7 @@ from errno import ENOENT as NO_SUCH_FILE_OR_DIRECTORY from glob import glob +import shutil import gzip import itertools import locale @@ -403,6 +404,55 @@ def get_binary_path(self, release, dev, android=False, magicleap=False): " --release" if release else "")) sys.exit() + def detach_volume(self, mounted_volume): + print("Detaching volume {}".format(mounted_volume)) + try: + subprocess.check_call(['hdiutil', 'detach', mounted_volume]) + except subprocess.CalledProcessError as e: + print("Could not detach volume {} : {}".format(mounted_volume, e.returncode)) + sys.exit(1) + + def detach_volume_if_attached(self, mounted_volume): + if os.path.exists(mounted_volume): + self.detach_volume(mounted_volume) + + def mount_dmg(self, dmg_path): + print("Mounting dmg {}".format(dmg_path)) + try: + subprocess.check_call(['hdiutil', 'attach', dmg_path]) + except subprocess.CalledProcessError as e: + print("Could not mount Servo dmg : {}".format(e.returncode)) + sys.exit(1) + + def extract_nightly(self, nightlies_folder, destination_folder, destination_file): + print("Extracting to {} ...".format(destination_folder)) + if is_macosx(): + mounted_volume = path.join(path.sep, "Volumes", "Servo") + self.detach_volume_if_attached(mounted_volume) + self.mount_dmg(destination_file) + # Servo folder is always this one + servo_directory = path.join(path.sep, "Volumes", "Servo", "Servo.app", "Contents", "MacOS") + print("Copying files from {} to {}".format(servo_directory, destination_folder)) + shutil.copytree(servo_directory, destination_folder) + self.detach_volume(mounted_volume) + else: + if is_windows(): + command = 'msiexec /a {} /qn TARGETDIR={}'.format( + os.path.join(nightlies_folder, destination_file), destination_folder) + if subprocess.call(command, stdout=PIPE, stderr=PIPE) != 0: + print("Could not extract the nightly executable from the msi package.") + sys.exit(1) + else: + with tarfile.open(os.path.join(nightlies_folder, destination_file), "r") as tar: + tar.extractall(destination_folder) + + def get_executable(self, destination_folder): + if is_windows(): + return path.join(destination_folder, "PFiles", "Mozilla research", "Servo Tech Demo") + if is_linux: + return path.join(destination_folder, "servo", "servo") + return path.join(destination_folder, "servo") + def get_nightly_binary_path(self, nightly_date): if nightly_date is None: return @@ -415,8 +465,7 @@ def get_nightly_binary_path(self, nightly_date): if is_windows(): os_prefix = "windows-msvc" if is_macosx(): - print("The nightly flag is not supported on mac yet.") - sys.exit(1) + os_prefix = "mac" nightly_date = nightly_date.strip() # Fetch the filename to download from the build list repository_index = NIGHTLY_REPOSITORY_URL + "?list-type=2&prefix=nightly" @@ -467,23 +516,12 @@ def get_nightly_binary_path(self, nightly_date): # Extract the downloaded nightly version if os.path.isdir(destination_folder): - print("The nightly file {} has already been extracted.".format( + print("The nightly folder {} has already been extracted.".format( destination_folder)) else: - print("Extracting to {} ...".format(destination_folder)) - if is_windows(): - command = 'msiexec /a {} /qn TARGETDIR={}'.format( - os.path.join(nightlies_folder, destination_file), destination_folder) - if subprocess.call(command, stdout=PIPE, stderr=PIPE) != 0: - print("Could not extract the nightly executable from the msi package.") - sys.exit(1) - else: - with tarfile.open(os.path.join(nightlies_folder, destination_file), "r") as tar: - tar.extractall(destination_folder) - bin_folder = path.join(destination_folder, "servo") - if is_windows(): - bin_folder = path.join(destination_folder, "PFiles", "Mozilla research", "Servo Tech Demo") - return path.join(bin_folder, "servo{}".format(BIN_SUFFIX)) + self.extract_nightly(nightlies_folder, destination_folder, destination_file) + + return self.get_executable(destination_folder) def needs_gstreamer_env(self, target): try: