From 5b8d783f4cb622fa68c5270b27ebad1e838ee7cb Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 13 Jan 2017 16:40:38 -0500 Subject: [PATCH 1/5] Remove unusable --android flag for `mach bootstrap` `mach` can't do any bootstrapping for Android, so the flag is useless. --- python/servo/bootstrap_commands.py | 7 ++----- python/servo/bootstrapper/base.py | 8 -------- python/servo/bootstrapper/bootstrap.py | 6 ++---- python/servo/bootstrapper/windows_gnu.py | 3 --- python/servo/bootstrapper/windows_msvc.py | 3 --- 5 files changed, 4 insertions(+), 23 deletions(-) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 9bc7c60fd4ae..49dd40648b16 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -136,17 +136,14 @@ def env(self): @CommandArgument('--interactive', "-i", action='store_true', help='Need to answer any (Y/n) interactive prompts.') - @CommandArgument('--android', - action='store_true', - help='Install required packages for Android') @CommandArgument('--force', '-f', action='store_true', help='Force reinstall packages') - def bootstrap(self, android=False, interactive=False, force=False): + def bootstrap(self, interactive=False, force=False): from servo.bootstrapper.bootstrap import Bootstrapper bootstrapper = Bootstrapper(self.context) - bootstrapper.bootstrap(android=android, interactive=interactive, force=force) + bootstrapper.bootstrap(interactive=interactive, force=force) @Command('bootstrap-rust', description='Download the Rust compiler', diff --git a/python/servo/bootstrapper/base.py b/python/servo/bootstrapper/base.py index 40f3d1f5ebc7..b988e06c53af 100644 --- a/python/servo/bootstrapper/base.py +++ b/python/servo/bootstrapper/base.py @@ -29,14 +29,6 @@ def install_system_packages(self): raise NotImplementedError('%s must implement install_system_packages()' % __name__) - def install_mobile_android_packages(self): - ''' - Install packages required to build Servo for Android. - ''' - raise NotImplementedError('Cannot bootstrap Servo for Android: ' - '%s does not yet implement install_mobile_android_packages()' - % __name__) - def which(self, name): """Python implementation of which. diff --git a/python/servo/bootstrapper/bootstrap.py b/python/servo/bootstrapper/bootstrap.py index d07dc12fc3ca..4cd1fc7bc5c7 100644 --- a/python/servo/bootstrapper/bootstrap.py +++ b/python/servo/bootstrapper/bootstrap.py @@ -30,13 +30,11 @@ def __init__(self, context): self.instance = cls(**args) self.instance.context = context - def bootstrap(self, android=False, interactive=False, force=False): + def bootstrap(self, interactive=False, force=False): self.instance.interactive = interactive self.instance.force = force - if android: - self.instance.install_mobile_android_packages() - elif force: + if force: self.instance.install_system_packages() else: self.instance.ensure_system_packages() diff --git a/python/servo/bootstrapper/windows_gnu.py b/python/servo/bootstrapper/windows_gnu.py index 794f1790cd12..4980baa3bb19 100644 --- a/python/servo/bootstrapper/windows_gnu.py +++ b/python/servo/bootstrapper/windows_gnu.py @@ -32,9 +32,6 @@ def install_system_packages(self, packages=deps): self._ensure_package_manager_updated() self.pacman_install(*packages) - def install_mobile_android_packages(self): - sys.exit('We do not support building Android on Windows. Sorry!') - def _update_package_manager(self): self.pacman_update() diff --git a/python/servo/bootstrapper/windows_msvc.py b/python/servo/bootstrapper/windows_msvc.py index 11ff765c99d1..8ca11682d798 100644 --- a/python/servo/bootstrapper/windows_msvc.py +++ b/python/servo/bootstrapper/windows_msvc.py @@ -81,6 +81,3 @@ def install_system_packages(self, packages=deps): with open(installed_deps_file, 'w') as installed_file: for line in packages: installed_file.write(line + "\n") - - def install_mobile_android_packages(self): - sys.exit('We do not support building Android on Windows. Sorry!') From 02b054ec9e799705296d5545a744bbf766ddbae6 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 13 Jan 2017 21:55:33 -0500 Subject: [PATCH 2/5] Create a util.py Python module for common functions Extracting these functions helps avoid circular dependencies, and make them easier to find/reuse. --- python/servo/bootstrap_commands.py | 93 +------------ python/servo/bootstrapper/windows_msvc.py | 3 +- python/servo/build_commands.py | 3 +- python/servo/command_base.py | 48 +------ python/servo/testing_commands.py | 3 +- python/servo/util.py | 151 ++++++++++++++++++++++ 6 files changed, 160 insertions(+), 141 deletions(-) create mode 100644 python/servo/util.py diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 49dd40648b16..9128c1dccffa 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -8,7 +8,6 @@ # except according to those terms. from __future__ import print_function, unicode_literals -from socket import error as socket_error import base64 import json @@ -17,9 +16,6 @@ import re import shutil import sys -import StringIO -import tarfile -import zipfile import urllib2 from mach.decorators import ( @@ -28,93 +24,8 @@ Command, ) -from servo.command_base import CommandBase, host_triple, BIN_SUFFIX - - -def download(desc, src, writer, start_byte=0): - if start_byte: - print("Resuming download of %s..." % desc) - else: - print("Downloading %s..." % desc) - dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty()) - - try: - req = urllib2.Request(src) - if start_byte: - req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)}) - resp = urllib2.urlopen(req) - - fsize = None - if resp.info().getheader('Content-Length'): - fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte - - recved = start_byte - chunk_size = 8192 - - while True: - chunk = resp.read(chunk_size) - if not chunk: - break - recved += len(chunk) - if not dumb: - if fsize is not None: - pct = recved * 100.0 / fsize - print("\rDownloading %s: %5.1f%%" % (desc, pct), end="") - - sys.stdout.flush() - writer.write(chunk) - - if not dumb: - print() - except urllib2.HTTPError, e: - print("Download failed (%d): %s - %s" % (e.code, e.reason, src)) - if e.code == 403: - print("No Rust compiler binary available for this platform. " - "Please see https://github.com/servo/servo/#prerequisites") - sys.exit(1) - except urllib2.URLError, e: - print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src)) - sys.exit(1) - except socket_error, e: - print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e)) - sys.exit(1) - except KeyboardInterrupt: - writer.flush() - raise - - -def download_file(desc, src, dst): - tmp_path = dst + ".part" - try: - start_byte = os.path.getsize(tmp_path) - with open(tmp_path, 'ab') as fd: - download(desc, src, fd, start_byte=start_byte) - except os.error: - with open(tmp_path, 'wb') as fd: - download(desc, src, fd) - os.rename(tmp_path, dst) - - -def download_bytes(desc, src): - content_writer = StringIO.StringIO() - download(desc, src, content_writer) - return content_writer.getvalue() - - -def extract(src, dst, movedir=None): - if src.endswith(".zip"): - zipfile.ZipFile(src).extractall(dst) - else: - tarfile.open(src).extractall(dst) - - if movedir: - for f in os.listdir(movedir): - frm = path.join(movedir, f) - to = path.join(dst, f) - os.rename(frm, to) - os.rmdir(movedir) - - os.remove(src) +from servo.command_base import CommandBase, BIN_SUFFIX +from servo.util import download_bytes, download_file, extract, host_triple @CommandProvider diff --git a/python/servo/bootstrapper/windows_msvc.py b/python/servo/bootstrapper/windows_msvc.py index 8ca11682d798..0545928f4533 100644 --- a/python/servo/bootstrapper/windows_msvc.py +++ b/python/servo/bootstrapper/windows_msvc.py @@ -3,10 +3,10 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os -import sys import shutil from distutils import spawn +from servo.util import extract, download_file from base import BaseBootstrapper from packages import WINDOWS_MSVC as deps @@ -21,7 +21,6 @@ def ensure_system_packages(self): self.install_system_packages() def install_system_packages(self, packages=deps): - from servo.bootstrap_commands import extract, download_file deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies") deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/" diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index 6e7ec8d24c50..5fbb7ba164d3 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -24,7 +24,8 @@ Command, ) -from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, host_triple, find_dep_path_newest +from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, find_dep_path_newest +from servo.util import host_triple def format_duration(seconds): diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 80fe675da1c8..00bc38de070a 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -18,11 +18,12 @@ from subprocess import PIPE import sys import tarfile -import platform import toml from mach.registrar import Registrar +from servo.util import host_triple + BIN_SUFFIX = ".exe" if sys.platform == "win32" else "" @@ -107,51 +108,6 @@ def reset(tarinfo): os.rename(temp_file, dest_archive) -def host_triple(): - os_type = platform.system().lower() - if os_type == "linux": - os_type = "unknown-linux-gnu" - elif os_type == "darwin": - os_type = "apple-darwin" - elif os_type == "android": - os_type = "linux-androideabi" - elif os_type == "windows": - # If we are in a Visual Studio environment, use msvc - if os.getenv("PLATFORM") is not None: - os_type = "pc-windows-msvc" - elif os.getenv("MSYSTEM") is not None: - os_type = "pc-windows-gnu" - else: - os_type = "unknown" - elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"): - os_type = "pc-windows-gnu" - elif os_type == "freebsd": - os_type = "unknown-freebsd" - else: - os_type = "unknown" - - cpu_type = platform.machine().lower() - if os_type.endswith("-msvc"): - # vcvars*.bat should set it properly - platform_env = os.environ.get("PLATFORM") - if platform_env == "X86": - cpu_type = "i686" - elif platform_env == "X64": - cpu_type = "x86_64" - else: - cpu_type = "unknown" - elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]: - cpu_type = "i686" - elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: - cpu_type = "x86_64" - elif cpu_type == "arm": - cpu_type = "arm" - else: - cpu_type = "unknown" - - return "%s-%s" % (cpu_type, os_type) - - def normalize_env(env): # There is a bug in subprocess where it doesn't like unicode types in # environment variables. Here, ensure all unicode are converted to diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 04b0542a0799..185e7420a47d 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -30,8 +30,9 @@ from servo.command_base import ( BuildNotFound, CommandBase, - call, cd, check_call, host_triple, set_osmesa_env, + call, cd, check_call, set_osmesa_env, ) +from servo.util import host_triple from wptrunner import wptcommandline from update import updatecommandline diff --git a/python/servo/util.py b/python/servo/util.py new file mode 100644 index 000000000000..24d3f4117369 --- /dev/null +++ b/python/servo/util.py @@ -0,0 +1,151 @@ +# Copyright 2013 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import os.path as path +import platform +import sys +from socket import error as socket_error +import StringIO +import tarfile +import zipfile +import urllib2 + + +def host_triple(): + os_type = platform.system().lower() + if os_type == "linux": + os_type = "unknown-linux-gnu" + elif os_type == "darwin": + os_type = "apple-darwin" + elif os_type == "android": + os_type = "linux-androideabi" + elif os_type == "windows": + # If we are in a Visual Studio environment, use msvc + if os.getenv("PLATFORM") is not None: + os_type = "pc-windows-msvc" + elif os.getenv("MSYSTEM") is not None: + os_type = "pc-windows-gnu" + else: + os_type = "unknown" + elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"): + os_type = "pc-windows-gnu" + elif os_type == "freebsd": + os_type = "unknown-freebsd" + else: + os_type = "unknown" + + cpu_type = platform.machine().lower() + if os_type.endswith("-msvc"): + # vcvars*.bat should set it properly + platform_env = os.environ.get("PLATFORM") + if platform_env == "X86": + cpu_type = "i686" + elif platform_env == "X64": + cpu_type = "x86_64" + else: + cpu_type = "unknown" + elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]: + cpu_type = "i686" + elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: + cpu_type = "x86_64" + elif cpu_type == "arm": + cpu_type = "arm" + else: + cpu_type = "unknown" + + return "{}-{}".format(cpu_type, os_type) + + +def download(desc, src, writer, start_byte=0): + if start_byte: + print("Resuming download of %s..." % desc) + else: + print("Downloading %s..." % desc) + dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty()) + + try: + req = urllib2.Request(src) + if start_byte: + req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)}) + resp = urllib2.urlopen(req) + + fsize = None + if resp.info().getheader('Content-Length'): + fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte + + recved = start_byte + chunk_size = 8192 + + while True: + chunk = resp.read(chunk_size) + if not chunk: + break + recved += len(chunk) + if not dumb: + if fsize is not None: + pct = recved * 100.0 / fsize + print("\rDownloading %s: %5.1f%%" % (desc, pct), end="") + + sys.stdout.flush() + writer.write(chunk) + + if not dumb: + print() + except urllib2.HTTPError, e: + print("Download failed (%d): %s - %s" % (e.code, e.reason, src)) + if e.code == 403: + print("No Rust compiler binary available for this platform. " + "Please see https://github.com/servo/servo/#prerequisites") + sys.exit(1) + except urllib2.URLError, e: + print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src)) + sys.exit(1) + except socket_error, e: + print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e)) + sys.exit(1) + except KeyboardInterrupt: + writer.flush() + raise + + +def download_bytes(desc, src): + content_writer = StringIO.StringIO() + download(desc, src, content_writer) + return content_writer.getvalue() + + +def download_file(desc, src, dst): + tmp_path = dst + ".part" + try: + start_byte = path.getsize(tmp_path) + with open(tmp_path, 'ab') as fd: + download(desc, src, fd, start_byte=start_byte) + except os.error: + with open(tmp_path, 'wb') as fd: + download(desc, src, fd) + os.rename(tmp_path, dst) + + +def extract(src, dst, movedir=None): + if src.endswith(".zip"): + zipfile.ZipFile(src).extractall(dst) + else: + tarfile.open(src).extractall(dst) + + if movedir: + for f in os.listdir(movedir): + frm = path.join(movedir, f) + to = path.join(dst, f) + os.rename(frm, to) + os.rmdir(movedir) + + os.remove(src) From ef900cbdcb0e544639ae10b390a68da2afd8bcce Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sun, 15 Jan 2017 15:19:13 -0500 Subject: [PATCH 3/5] Avoid hardcoded references to rustc in download() Use the `desc` parameter instead to make the error messages customized for the actual download. Also use new-style format strings. --- python/servo/util.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/servo/util.py b/python/servo/util.py index 24d3f4117369..03f993b3bf0b 100644 --- a/python/servo/util.py +++ b/python/servo/util.py @@ -67,9 +67,9 @@ def host_triple(): def download(desc, src, writer, start_byte=0): if start_byte: - print("Resuming download of %s..." % desc) + print("Resuming download of {}...".format(desc)) else: - print("Downloading %s..." % desc) + print("Downloading {}...".format(desc)) dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty()) try: @@ -101,16 +101,16 @@ def download(desc, src, writer, start_byte=0): if not dumb: print() except urllib2.HTTPError, e: - print("Download failed (%d): %s - %s" % (e.code, e.reason, src)) + print("Download failed ({}): {} - {}".format(e.code, e.reason, src)) if e.code == 403: print("No Rust compiler binary available for this platform. " "Please see https://github.com/servo/servo/#prerequisites") sys.exit(1) except urllib2.URLError, e: - print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src)) + print("Error downloading {}: {}. The failing URL was: {}".format(desc, e.reason, src)) sys.exit(1) except socket_error, e: - print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e)) + print("Looks like there's a connectivity issue, check your Internet connection. {}".format(e)) sys.exit(1) except KeyboardInterrupt: writer.flush() From 60a1503b2997b05e3f36f4ce92d688df44fdeae7 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 13 Jan 2017 23:11:34 -0500 Subject: [PATCH 4/5] Clean up and simplify existing `mach bootstrap` - Default to interactive mode and remove the `--interactive` flag - Use `--force` to skip interactivity - Change MSVC dependency storage organization on disk: put each version into its own folder and directly refer to the versioned folders, providing immutability and making the installation list redundant - Reuse `host_triple()` function to fix broken bootstrapper dispatching - Simplify code: - Remove or inline many unused and redudant functions and variables - Prefer plain functions to classes - Consolidate into fewer files, remove unnecessary bootstrapper/ dir - Improve Python style - Sort dependency list --- python/servo/bootstrap.py | 110 ++++++++++++++++++++ python/servo/bootstrap_commands.py | 15 +-- python/servo/bootstrapper/__init__.py | 3 - python/servo/bootstrapper/base.py | 54 ---------- python/servo/bootstrapper/bootstrap.py | 40 ------- python/servo/bootstrapper/windows_gnu.py | 72 ------------- python/servo/bootstrapper/windows_msvc.py | 82 --------------- python/servo/command_base.py | 18 ++-- python/servo/{bootstrapper => }/packages.py | 24 ++--- 9 files changed, 137 insertions(+), 281 deletions(-) create mode 100644 python/servo/bootstrap.py delete mode 100644 python/servo/bootstrapper/__init__.py delete mode 100644 python/servo/bootstrapper/base.py delete mode 100644 python/servo/bootstrapper/bootstrap.py delete mode 100644 python/servo/bootstrapper/windows_gnu.py delete mode 100644 python/servo/bootstrapper/windows_msvc.py rename python/servo/{bootstrapper => }/packages.py (73%) diff --git a/python/servo/bootstrap.py b/python/servo/bootstrap.py new file mode 100644 index 000000000000..ec8a0a6c4496 --- /dev/null +++ b/python/servo/bootstrap.py @@ -0,0 +1,110 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function + +from distutils.spawn import find_executable +import os +import subprocess +import sys + +import servo.packages as packages +from servo.util import extract, download_file, host_triple + + +def windows_gnu(context, force=False): + '''Bootstrapper for msys2 based environments for building in Windows.''' + + if not find_executable('pacman'): + print( + 'The Windows GNU bootstrapper only works with msys2 with pacman. ' + 'Get msys2 at http://msys2.github.io/' + ) + return 1 + + # Ensure repositories are up to date + command = ['pacman', '--sync', '--refresh'] + subprocess.check_call(command) + + # Install packages + command = ['pacman', '--sync', '--needed'] + if force: + command.append('--noconfirm') + subprocess.check_call(command + list(packages.WINDOWS_GNU)) + + # Downgrade GCC to 5.4.0-1 + gcc_pkgs = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"] + gcc_version = "5.4.0-1" + mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz" + gcc_list = [mingw_url.format(gcc, gcc_version) for gcc in gcc_pkgs] + + # Note: `--upgrade` also does downgrades + downgrade_command = ['pacman', '--upgrade'] + if force: + downgrade_command.append('--noconfirm') + subprocess.check_call(downgrade_command + gcc_list) + + +def windows_msvc(context, force=False): + '''Bootstrapper for MSVC building on Windows.''' + + deps_dir = os.path.join(context.sharedir, "msvc-dependencies") + deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/" + + def version(package): + return packages.WINDOWS_MSVC[package] + + def package_dir(package): + return os.path.join(deps_dir, package, version(package)) + + to_install = {} + for package in packages.WINDOWS_MSVC: + # Don't install CMake if it already exists in PATH + if package == "cmake" and find_executable(package): + continue + + if not os.path.isdir(package_dir(package)): + to_install[package] = version(package) + + if not to_install: + return 0 + + print("Installing missing MSVC dependencies...") + for package in to_install: + full_spec = '{}-{}'.format(package, version(package)) + + parent_dir = os.path.dirname(package_dir(package)) + if not os.path.isdir(parent_dir): + os.makedirs(parent_dir) + + zip_path = package_dir(package) + ".zip" + if not os.path.isfile(zip_path): + zip_url = "{}{}.zip".format(deps_url, full_spec) + download_file(full_spec, zip_url, zip_path) + + print("Extracting {}...".format(full_spec), end='') + extract(zip_path, deps_dir) + print("done") + + extracted_path = os.path.join(deps_dir, full_spec) + os.rename(extracted_path, package_dir(package)) + + return 0 + + +def bootstrap(context, force=False): + '''Dispatches to the right bootstrapping function for the OS.''' + + bootstrapper = None + + if "windows-gnu" in host_triple(): + bootstrapper = windows_gnu + elif "windows-msvc" in host_triple(): + bootstrapper = windows_msvc + + if bootstrapper is None: + print('Bootstrap support is not yet available for your OS.') + return 1 + + return bootstrapper(context, force=force) diff --git a/python/servo/bootstrap_commands.py b/python/servo/bootstrap_commands.py index 9128c1dccffa..8975f174d840 100644 --- a/python/servo/bootstrap_commands.py +++ b/python/servo/bootstrap_commands.py @@ -7,7 +7,7 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -from __future__ import print_function, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import base64 import json @@ -24,6 +24,7 @@ Command, ) +import servo.bootstrap as bootstrap from servo.command_base import CommandBase, BIN_SUFFIX from servo.util import download_bytes, download_file, extract, host_triple @@ -44,17 +45,11 @@ def env(self): @Command('bootstrap', description='Install required packages for building.', category='bootstrap') - @CommandArgument('--interactive', "-i", - action='store_true', - help='Need to answer any (Y/n) interactive prompts.') @CommandArgument('--force', '-f', action='store_true', - help='Force reinstall packages') - def bootstrap(self, interactive=False, force=False): - from servo.bootstrapper.bootstrap import Bootstrapper - - bootstrapper = Bootstrapper(self.context) - bootstrapper.bootstrap(interactive=interactive, force=force) + help='Boostrap without confirmation') + def bootstrap(self, force=False): + return bootstrap.bootstrap(self.context, force=force) @Command('bootstrap-rust', description='Download the Rust compiler', diff --git a/python/servo/bootstrapper/__init__.py b/python/servo/bootstrapper/__init__.py deleted file mode 100644 index 6fbe8159b2db..000000000000 --- a/python/servo/bootstrapper/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/python/servo/bootstrapper/base.py b/python/servo/bootstrapper/base.py deleted file mode 100644 index b988e06c53af..000000000000 --- a/python/servo/bootstrapper/base.py +++ /dev/null @@ -1,54 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import print_function, unicode_literals - -import distutils -import subprocess - - -class BaseBootstrapper(object): - """Base class for system bootstrappers.""" - - def __init__(self, interactive=False): - self.package_manager_updated = False - self.interactive = interactive - - def ensure_system_packages(self): - ''' - Check for missing packages. - ''' - raise NotImplementedError('%s must implement ensure_system_packages()' % - __name__) - - def install_system_packages(self): - ''' - Install packages required to build Servo. - ''' - raise NotImplementedError('%s must implement install_system_packages()' % - __name__) - - def which(self, name): - """Python implementation of which. - - It returns the path of an executable or None if it couldn't be found. - """ - return distutils.spawn.find_executable(name) - - def check_output(self, *args, **kwargs): - """Run subprocess.check_output.""" - return subprocess.check_output(*args, **kwargs) - - def _ensure_package_manager_updated(self): - if self.package_manager_updated: - return - - self._update_package_manager() - self.package_manager_updated = True - - def _update_package_manager(self): - """Updates the package manager's manifests/package list. - - This should be defined in child classes. - """ diff --git a/python/servo/bootstrapper/bootstrap.py b/python/servo/bootstrapper/bootstrap.py deleted file mode 100644 index 4cd1fc7bc5c7..000000000000 --- a/python/servo/bootstrapper/bootstrap.py +++ /dev/null @@ -1,40 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import print_function - -import sys - -from windows_gnu import WindowsGnuBootstrapper -from windows_msvc import WindowsMsvcBootstrapper - - -class Bootstrapper(object): - """Main class that performs system bootstrap.""" - - def __init__(self, context): - self.instance = None - cls = None - args = {} - - if sys.platform.startswith('msys'): - cls = WindowsGnuBootstrapper - - elif sys.platform.startswith('win32'): - cls = WindowsMsvcBootstrapper - - if cls is None: - sys.exit('Bootstrap support is not yet available for your OS.') - - self.instance = cls(**args) - self.instance.context = context - - def bootstrap(self, interactive=False, force=False): - self.instance.interactive = interactive - self.instance.force = force - - if force: - self.instance.install_system_packages() - else: - self.instance.ensure_system_packages() diff --git a/python/servo/bootstrapper/windows_gnu.py b/python/servo/bootstrapper/windows_gnu.py deleted file mode 100644 index 4980baa3bb19..000000000000 --- a/python/servo/bootstrapper/windows_gnu.py +++ /dev/null @@ -1,72 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import sys -import subprocess - -from base import BaseBootstrapper -from packages import WINDOWS_GNU as deps - - -class WindowsGnuBootstrapper(BaseBootstrapper): - '''Bootstrapper for msys2 based environments for building in Windows.''' - - def __init__(self, **kwargs): - BaseBootstrapper.__init__(self, **kwargs) - - if not self.which('pacman'): - raise NotImplementedError('The Windows bootstrapper only works with msys2 with pacman. Get msys2 at ' - 'http://msys2.github.io/') - - def ensure_system_packages(self): - install_packages = [] - for p in deps: - command = ['pacman', '-Qs', p] - if self.run_check(command): - install_packages += [p] - if install_packages: - install_packages(install_packages) - - def install_system_packages(self, packages=deps): - self._ensure_package_manager_updated() - self.pacman_install(*packages) - - def _update_package_manager(self): - self.pacman_update() - - def run(self, command): - subprocess.check_call(command, stdin=sys.stdin) - - def run_check(self, command): - return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - def pacman_update(self): - command = ['pacman', '--sync', '--refresh'] - self.run(command) - - def pacman_upgrade(self): - command = ['pacman', '--sync', '--refresh', '--sysupgrade'] - self.run(command) - - def pacman_install(self, *packages): - command = ['pacman', '--sync'] - if not self.force: - command.append('--needed') - if not self.interactive: - command.append('--noconfirm') - command.extend(packages) - self.run(command) - - # downgrade GCC to 5.4.0-1 - gcc_type = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"] - gcc_version = "5.4.0-1" - mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz" - gcc_list = [] - for gcc in gcc_type: - gcc_list += [mingw_url.format(gcc, gcc_version)] - downgrade_command = ['pacman', '-U'] - if not self.interactive: - downgrade_command.append('--noconfirm') - downgrade_command.extend(gcc_list) - self.run(downgrade_command) diff --git a/python/servo/bootstrapper/windows_msvc.py b/python/servo/bootstrapper/windows_msvc.py deleted file mode 100644 index 0545928f4533..000000000000 --- a/python/servo/bootstrapper/windows_msvc.py +++ /dev/null @@ -1,82 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import shutil -from distutils import spawn - -from servo.util import extract, download_file -from base import BaseBootstrapper -from packages import WINDOWS_MSVC as deps - - -class WindowsMsvcBootstrapper(BaseBootstrapper): - '''Bootstrapper for MSVC building on Windows.''' - - def __init__(self, **kwargs): - BaseBootstrapper.__init__(self, **kwargs) - - def ensure_system_packages(self): - self.install_system_packages() - - def install_system_packages(self, packages=deps): - - deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies") - deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/" - first_run = True - - if self.force: - if os.path.isdir(deps_dir): - shutil.rmtree(deps_dir) - - if not os.path.isdir(deps_dir): - os.makedirs(deps_dir) - - # Read file with installed dependencies, if exist - installed_deps_file = os.path.join(deps_dir, "installed-dependencies.txt") - if os.path.exists(installed_deps_file): - installed_deps = [l.strip() for l in open(installed_deps_file)] - else: - installed_deps = [] - - # list of dependencies that need to be updated - update_deps = list(set(packages) - set(installed_deps)) - - for dep in packages: - dep_name = dep.split("-")[0] - - # Don't download CMake if already exists in PATH - if dep_name == "cmake": - if spawn.find_executable(dep_name): - continue - - dep_dir = os.path.join(deps_dir, dep_name) - # if not installed or need to be updated - if not os.path.exists(dep_dir) or dep in update_deps: - if first_run: - print "Installing missing MSVC dependencies..." - first_run = False - - dep_version_dir = os.path.join(deps_dir, dep) - - if os.path.exists(dep_version_dir): - shutil.rmtree(dep_version_dir) - - dep_zip = dep_version_dir + ".zip" - if not os.path.isfile(dep_zip): - download_file(dep, "%s%s.zip" % (deps_url, dep), dep_zip) - - print "Extracting %s..." % dep, - extract(dep_zip, deps_dir) - print "done" - - # Delete directory if exist - if os.path.exists(dep_dir): - shutil.rmtree(dep_dir) - os.rename(dep_version_dir, dep_dir) - - # Write in installed-dependencies.txt file - with open(installed_deps_file, 'w') as installed_file: - for line in packages: - installed_file.write(line + "\n") diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 00bc38de070a..e80c9cb018bb 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -19,12 +19,12 @@ import sys import tarfile +from mach.registrar import Registrar import toml -from mach.registrar import Registrar +from servo.packages import WINDOWS_MSVC as msvc_deps from servo.util import host_triple - BIN_SUFFIX = ".exe" if sys.platform == "win32" else "" @@ -384,14 +384,18 @@ def build_env(self, hosts_file_path=None, target=None, is_build=False): if "msvc" in (target or host_triple()): msvc_x64 = "64" if "x86_64" in (target or host_triple()) else "" msvc_deps_dir = path.join(self.context.sharedir, "msvc-dependencies") - extra_path += [path.join(msvc_deps_dir, "cmake", "bin")] - extra_path += [path.join(msvc_deps_dir, "ninja", "bin")] + + def package_dir(package): + return path.join(msvc_deps_dir, package, msvc_deps[package]) + + extra_path += [path.join(package_dir("cmake"), "bin")] + extra_path += [path.join(package_dir("ninja"), "bin")] # Link openssl - env["OPENSSL_INCLUDE_DIR"] = path.join(msvc_deps_dir, "openssl", "include") - env["OPENSSL_LIB_DIR"] = path.join(msvc_deps_dir, "openssl", "lib" + msvc_x64) + env["OPENSSL_INCLUDE_DIR"] = path.join(package_dir("openssl"), "include") + env["OPENSSL_LIB_DIR"] = path.join(package_dir("openssl"), "lib" + msvc_x64) env["OPENSSL_LIBS"] = "ssleay32MD:libeay32MD" # Link moztools - env["MOZTOOLS_PATH"] = path.join(msvc_deps_dir, "moztools", "bin") + env["MOZTOOLS_PATH"] = path.join(package_dir("moztools"), "bin") if is_windows(): if not os.environ.get("NATIVE_WIN32_PYTHON"): diff --git a/python/servo/bootstrapper/packages.py b/python/servo/packages.py similarity index 73% rename from python/servo/bootstrapper/packages.py rename to python/servo/packages.py index 7e9a7cd9c0e5..9e4906dc5cf4 100644 --- a/python/servo/bootstrapper/packages.py +++ b/python/servo/packages.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -# Listed all packages for different platforms in one file - -WINDOWS_GNU = [ +WINDOWS_GNU = set([ + "diffutils", + "make", "mingw-w64-x86_64-toolchain", "mingw-w64-x86_64-freetype", "mingw-w64-x86_64-icu", @@ -12,17 +12,15 @@ "mingw-w64-x86_64-ca-certificates", "mingw-w64-x86_64-expat", "mingw-w64-x86_64-cmake", - "tar", - "diffutils", "patch", "patchutils", - "make", "python2-setuptools", -] + "tar", +]) -WINDOWS_MSVC = [ - "cmake-3.6.1", - "ninja-1.7.1", - "openssl-1.0.1t-vs2015", - "moztools-0.0.1-5", -] +WINDOWS_MSVC = { + "cmake": "3.6.1", + "moztools": "0.0.1-5", + "ninja": "1.7.1", + "openssl": "1.0.1t-vs2015", +} From b4f508630510c1fc96f6f3245bfb29dd0e16dcf4 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 11 Jan 2017 16:43:54 -0500 Subject: [PATCH 5/5] Add a Salt bootstrapper The Salt bootstrapper invokes Salt during `./mach bootstrap` to install Servo's build dependencies. It uses salt-call pinned to the same version of Salt as used in saltfs. Currently, the implementation uses gitfs and reads directly from the master branch of the saltfs repo; in the future this should be changed when the relevant Salt states are moved in-tree as part of Dockerization for TaskCluster. We have not Salted our Windows machines, so the existing Windows bootstrappers are retained. Currently this is only tested on Ubuntu Trusty. Salt uses various system python libraries, including `python-apt` on Debian-based OSes to interact with apt. `python-apt` does not seem to be installable via a requirements.txt file, and the versions available on PyPI are far behind the versions installed on actual Ubuntu machines. Additionally, adding `python-apt` as an unconditional python dependency would add bloat for users of other OSes, and lead to more churn as additional OSes are supported. However, as `python-apt` is already installed via apt on these machines, we can allow Salt to instead use the module by using `--system-site-packages` for the python virtualenv. We also add the `-I` flag to `pip install` to ensure we have a local, untouched copy of any other python packages used. However, because this prints system-level Python packages in scope, it slightly breaks isolation, so it is important to always pin all dependencies in the requirements files. --- python/mach_bootstrap.py | 10 ++- python/requirements-salt.txt | 6 ++ python/requirements.txt | 3 + python/servo/bootstrap.py | 121 ++++++++++++++++++++++++++++++++++- 4 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 python/requirements-salt.txt diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index d41bb1f50993..fa612ddd868e 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -118,7 +118,11 @@ def _activate_virtualenv(topdir): if not virtualenv: sys.exit("Python virtualenv is not installed. Please install it prior to running mach.") - process = Popen([virtualenv, "-p", python, virtualenv_path], stdout=PIPE, stderr=PIPE) + process = Popen( + [virtualenv, "-p", python, "--system-site-packages", virtualenv_path], + stdout=PIPE, + stderr=PIPE + ) process.wait() if process.returncode: out, err = process.communicate() @@ -153,7 +157,7 @@ def _activate_virtualenv(topdir): if not pip: sys.exit("Python pip is either not installed or not found in virtualenv.") - process = Popen([pip, "install", "-q", "-U", "pip"], stdout=PIPE, stderr=PIPE) + process = Popen([pip, "install", "-q", "-I", "-U", "pip"], stdout=PIPE, stderr=PIPE) process.wait() if process.returncode: out, err = process.communicate() @@ -175,7 +179,7 @@ def _activate_virtualenv(topdir): if not pip: sys.exit("Python pip is either not installed or not found in virtualenv.") - process = Popen([pip, "install", "-q", "-r", req_path], stdout=PIPE, stderr=PIPE) + process = Popen([pip, "install", "-q", "-I", "-r", req_path], stdout=PIPE, stderr=PIPE) process.wait() if process.returncode: out, err = process.communicate() diff --git a/python/requirements-salt.txt b/python/requirements-salt.txt new file mode 100644 index 000000000000..1486b5900531 --- /dev/null +++ b/python/requirements-salt.txt @@ -0,0 +1,6 @@ +# Ensure all versions are pinned for repeatability, +# since `--system-site-packages` is enabled + +# For boostrapping, make sure versions match those in saltfs +salt == 2016.3.4 +GitPython == 0.3.2 diff --git a/python/requirements.txt b/python/requirements.txt index 8248a21e9303..41b25ccdb5a1 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,3 +1,6 @@ +# Ensure all versions are pinned for repeatability, +# since `--system-site-packages` is enabled + blessings == 1.6 mach == 0.6.0 mozdebug == 0.1 diff --git a/python/servo/bootstrap.py b/python/servo/bootstrap.py index ec8a0a6c4496..54862a885bdd 100644 --- a/python/servo/bootstrap.py +++ b/python/servo/bootstrap.py @@ -5,14 +5,129 @@ from __future__ import absolute_import, print_function from distutils.spawn import find_executable +import json import os +import platform +import shutil import subprocess -import sys import servo.packages as packages from servo.util import extract, download_file, host_triple +def salt(context, force=False): + # Ensure Salt is installed in the virtualenv + # It's not instaled globally because it's a large, non-required dependency, + # and the installation fails on Windows + print("Checking Salt installation...", end='') + reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt') + process = subprocess.Popen( + ["pip", "install", "-q", "-I", "-r", reqs_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + process.wait() + if process.returncode: + out, err = process.communicate() + print('failed to install Salt via pip:') + print('Output: {}\nError: {}'.format(out, err)) + return 1 + print("done") + + salt_root = os.path.join(context.sharedir, 'salt') + config_dir = os.path.join(salt_root, 'etc', 'salt') + pillar_dir = os.path.join(config_dir, 'pillars') + + # In order to allow `mach bootstrap` to work from any CWD, + # the `root_dir` must be an absolute path. + # We place it under `context.sharedir` because + # Salt caches data (e.g. gitfs files) in its `var` subdirectory. + # Hence, dynamically generate the config with an appropriate `root_dir` + # and serialize it as JSON (which is valid YAML). + config = { + 'fileserver_backend': ['git'], + 'gitfs_env_whitelist': 'base', + 'gitfs_provider': 'gitpython', + 'gitfs_remotes': [ + 'https://github.com/servo/saltfs.git', + ], + 'hash_type': 'sha384', + 'master': 'localhost', + 'root_dir': salt_root, + 'state_output': 'changes', + 'state_tabular': True, + } + + if not os.path.exists(config_dir): + os.makedirs(config_dir, mode=0o700) + with open(os.path.join(config_dir, 'minion'), 'w') as config_file: + config_file.write(json.dumps(config) + '\n') + + # Similarly, the pillar data is created dynamically + # and temporarily serialized to disk. + # This dynamism is not yet used, but will be in the future + # to enable Android bootstrapping by using + # context.sharedir as a location for Android packages. + pillar = { + 'top.sls': { + 'base': { + '*': ['bootstrap'], + }, + }, + 'bootstrap.sls': { + 'fully_managed': False, + }, + } + if os.path.exists(pillar_dir): + shutil.rmtree(pillar_dir) + os.makedirs(pillar_dir, mode=0o700) + for filename in pillar: + with open(os.path.join(pillar_dir, filename), 'w') as pillar_file: + pillar_file.write(json.dumps(pillar[filename]) + '\n') + + cmd = [ + 'sudo', + # sudo escapes from the venv, need to use full path + find_executable('salt-call'), + '--local', + '--config-dir={}'.format(config_dir), + '--pillar-root={}'.format(pillar_dir), + 'state.apply', + 'servo-build-dependencies', + ] + + if not force: + print('Running bootstrap in dry-run mode to show changes') + # Because `test=True` mode runs each state individually without + # considering how required/previous states affect the system, + # it will often report states with requisites as failing due + # to the requisites not actually being run, + # even though these are spurious and will succeed during + # the actual highstate. + # Hence `--retcode-passthrough` is not helpful in dry-run mode, + # so only detect failures of the actual salt-call binary itself. + retcode = subprocess.call(cmd + ['test=True']) + if retcode != 0: + print('Something went wrong while bootstrapping') + return retcode + + proceed = raw_input( + 'Proposed changes are above, proceed with bootstrap? [y/N]: ' + ) + if proceed.lower() not in ['y', 'yes']: + return 0 + + print('') + + print('Running Salt bootstrap') + retcode = subprocess.call(cmd + ['--retcode-passthrough']) + if retcode == 0: + print('Salt bootstrapping complete') + else: + print('Salt bootstrapping encountered errors') + return retcode + + def windows_gnu(context, force=False): '''Bootstrapper for msys2 based environments for building in Windows.''' @@ -102,6 +217,10 @@ def bootstrap(context, force=False): bootstrapper = windows_gnu elif "windows-msvc" in host_triple(): bootstrapper = windows_msvc + elif "linux-gnu" in host_triple(): + distro, version, _ = platform.linux_distribution() + if distro == 'Ubuntu' and version == '14.04': + bootstrapper = salt if bootstrapper is None: print('Bootstrap support is not yet available for your OS.')