From c8689204ddef07f7632f4eeeb916300062221084 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 12 Apr 2017 11:13:35 -0400 Subject: [PATCH 1/5] Use double bracket bash conditional per style guide --- .travis/dispatch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/dispatch.sh b/.travis/dispatch.sh index c82923f2..74e0ee80 100755 --- a/.travis/dispatch.sh +++ b/.travis/dispatch.sh @@ -86,7 +86,7 @@ elif [[ -n "${SALT_DOCKER_IMAGE:-}" ]]; then # macOS bash is too old for `-v` run_inside_docker "$@" else - if [ "${SALT_FROM_SCRATCH}" = "true" ]; then + if [[ "${SALT_FROM_SCRATCH}" = "true" ]]; then run_salt 'scratch' else git fetch origin master:master From b87e0562e6d6d384945a915f63132b220bd54b6c Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 12 Apr 2017 15:33:18 -0400 Subject: [PATCH 2/5] Support Debian for `./mach bootstrap` The Travis builders are detected as Debian for some reason by bootstrap. --- python/init.sls | 4 ++-- servo-build-dependencies/init.sls | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/init.sls b/python/init.sls index d0ee3b07..e242cbc1 100644 --- a/python/init.sls +++ b/python/init.sls @@ -18,7 +18,7 @@ python3: {% endif %} {% endif %} -{% if grains['os'] == 'Ubuntu' %} +{% if grains['os_family'] == 'Debian' %} python2-dev: pkg.installed: - pkgs: @@ -28,7 +28,7 @@ python2-dev: pip: pkg.installed: - pkgs: - {% if grains['os'] in ['CentOS', 'Fedora', 'Ubuntu'] %} + {% if grains['os'] in ['CentOS', 'Fedora', 'Ubuntu', 'Debian'] %} - python-pip {% elif grains['os'] == 'MacOS' %} - python # pip is included with python in homebrew diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index ef70633a..24e264ac 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -16,7 +16,7 @@ servo-dependencies: - openssl - pkg-config - yasm - {% elif grains['os'] == 'Ubuntu' %} + {% elif grains['os_family'] == 'Debian' %} - autoconf2.13 - curl - freeglut3-dev From d06e5e12a779a1c224ba2b503b77ec2e109e5332 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 12 Apr 2017 11:13:09 -0400 Subject: [PATCH 3/5] Add test for mach bootstrap This will ensure that Salt changes or refactors don't break the important `./mach bootstrap` use case. Make sure to test that bootstrap actually uses the local saltfs tree, and doesn't just pull down saltfs via gitfs. Use a new Travis node instead of the existing test node to avoid any side effects caused by the bootstrapper, and for added parallelism. --- .travis.yml | 6 ++++++ .travis/dispatch.sh | 15 +++++++++++++++ Vagrantfile | 2 +- tests/servo/__init__.py | 0 tests/servo/bootstrap.py | 13 +++++++++++++ 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/servo/__init__.py create mode 100644 tests/servo/bootstrap.py diff --git a/.travis.yml b/.travis.yml index 2c1b1147..7b33aaa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,6 +77,12 @@ matrix: os: linux sudo: required dist: trusty + # Not a Salt node, checks that mach bootstrap works + - env: + - SALT_NODE_ID=mach-bootstrap + os: linux + sudo: required + dist: trusty script: .travis/dispatch.sh diff --git a/.travis/dispatch.sh b/.travis/dispatch.sh index 74e0ee80..87cc6a52 100755 --- a/.travis/dispatch.sh +++ b/.travis/dispatch.sh @@ -81,6 +81,21 @@ if [[ "${SALT_NODE_ID}" == "test" ]]; then # Run test suite separately for parallelism setup_venv ./test.py +elif [[ "${SALT_NODE_ID}" == "mach-bootstrap" ]]; then + # Use system Python 2, not one of the supplemental ones Travis installs, + # so that `python-apt` is available + export PATH="/usr/bin:${PATH}" + + # Run mach bootstrap test separately from `test` because it installs things + git clone --depth 1 https://github.com/servo/servo.git ../servo + + # mach is intolerant of being run from other directories + cd ../servo + SERVO_SALTFS_ROOT="../saltfs" ./mach bootstrap --force + cd ../saltfs + + # Check that the local saltfs tree was used + ./test.py servo.bootstrap elif [[ -n "${SALT_DOCKER_IMAGE:-}" ]]; then # macOS bash is too old for `-v` printf "Using %s\n" "$(docker -v)" diff --git a/Vagrantfile b/Vagrantfile index 65d2917d..d6138a95 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -46,7 +46,7 @@ Vagrant.configure(2) do |config| os_and_version = node['os'] + version puts "OS #{os_and_version} is not yet supported" end - elsif node_config[:id] != 'test' + elsif node_config[:id].start_with? 'servo-' node_config end end.compact.uniq.each do |node| diff --git a/tests/servo/__init__.py b/tests/servo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/servo/bootstrap.py b/tests/servo/bootstrap.py new file mode 100644 index 00000000..daccef01 --- /dev/null +++ b/tests/servo/bootstrap.py @@ -0,0 +1,13 @@ +import json + +from tests.util import Failure, Success + + +def run(): + with open('.servo/salt/etc/salt/minion') as config_file: + config = json.load(config_file) + ok = config['fileserver_backend'] == ['roots'] + if ok: + return Success("mach bootstrap ran with the local saltfs tree") + else: + return Failure("mach bootstrap created bad config:", config) From 92b49d6119b79fcd6caf4c4d47c9703ae010403a Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 5 May 2017 23:03:56 -0400 Subject: [PATCH 4/5] WIP: Test mach bootstrap inside docker --- .travis.yml | 11 ++++++++++- .travis/dispatch.sh | 26 ++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b33aaa9..c92a777f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,9 +77,18 @@ matrix: os: linux sudo: required dist: trusty - # Not a Salt node, checks that mach bootstrap works + # Check that mach bootstrap works - env: - SALT_NODE_ID=mach-bootstrap + # ubuntu/14.04 + - SALT_DOCKER_IMAGE=ubuntu@sha256:edf05697d8ea17028a69726b4b450ad48da8b29884cd640fec950c904bfb50ce + os: linux + sudo: required + dist: trusty + - env: + - SALT_NODE_ID=mach-bootstrap + # ubuntu/16.04 + - SALT_DOCKER_IMAGE=ubuntu@sha256:f3a61450ae43896c4332bda5e78b453f4a93179045f20c8181043b26b5e79028 os: linux sudo: required dist: trusty diff --git a/.travis/dispatch.sh b/.travis/dispatch.sh index 87cc6a52..dc29d81e 100755 --- a/.travis/dispatch.sh +++ b/.travis/dispatch.sh @@ -46,9 +46,19 @@ run_inside_docker() { # (without exporting the `SALT_DOCKER_IMAGE` environment variable # to prevent recursion) local -r DOCKER_SALT_ROOT="/tmp/salt" + + # Use an env file for variables which may or may not be present + local -r DOCKER_ENV_FILE="/tmp/docker-env-file" + printf '' > "${DOCKER_ENV_FILE}" + # macOS bash is too old for `-v`, so use `-n` and default empty instead + if [[ -n "${SALT_FROM_SCRATCH:-}" ]]; then + printf "%s\n" >> "${DOCKER_ENV_FILE}" \ + "SALT_FROM_SCRATCH=${SALT_FROM_SCRATCH}" + fi + docker run \ + --env-file="${DOCKER_ENV_FILE}" \ --env="SALT_NODE_ID=${SALT_NODE_ID}" \ - --env="SALT_FROM_SCRATCH=${SALT_FROM_SCRATCH}" \ --env="TRAVIS_COMMIT=${TRAVIS_COMMIT}" \ --env="TRAVIS_OS_NAME=${TRAVIS_OS_NAME}" \ --volume="$(pwd):${DOCKER_SALT_ROOT}" \ @@ -77,7 +87,11 @@ if (( EUID != 0 )); then fi -if [[ "${SALT_NODE_ID}" == "test" ]]; then +if [[ -n "${SALT_DOCKER_IMAGE:-}" ]]; then # macOS bash is too old for `-v` + printf "Using %s\n" "$(docker -v)" + + run_inside_docker "$@" +elif [[ "${SALT_NODE_ID}" == "test" ]]; then # Run test suite separately for parallelism setup_venv ./test.py @@ -86,6 +100,10 @@ elif [[ "${SALT_NODE_ID}" == "mach-bootstrap" ]]; then # so that `python-apt` is available export PATH="/usr/bin:${PATH}" + # Install git (not present by default in Docker) + sudo apt-get update + sudo apt-get -y install git + # Run mach bootstrap test separately from `test` because it installs things git clone --depth 1 https://github.com/servo/servo.git ../servo @@ -96,10 +114,6 @@ elif [[ "${SALT_NODE_ID}" == "mach-bootstrap" ]]; then # Check that the local saltfs tree was used ./test.py servo.bootstrap -elif [[ -n "${SALT_DOCKER_IMAGE:-}" ]]; then # macOS bash is too old for `-v` - printf "Using %s\n" "$(docker -v)" - - run_inside_docker "$@" else if [[ "${SALT_FROM_SCRATCH}" = "true" ]]; then run_salt 'scratch' From fc0591fb0f94d04f634423303073ffd25983d3d7 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 5 May 2017 23:33:43 -0400 Subject: [PATCH 5/5] debug: only run mach bootstrap on travis --- .travis.yml | 142 ++++++++++++++++++++++---------------------- .travis/dispatch.sh | 8 +-- 2 files changed, 75 insertions(+), 75 deletions(-) diff --git a/.travis.yml b/.travis.yml index c92a777f..800af27a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,77 +6,77 @@ language: cpp # sudo: required and dist: trusty enable usage of Trusty matrix: include: - # Salt from scratch - - env: - - SALT_NODE_ID=servo-linux-cross1 - - SALT_FROM_SCRATCH=true - os: linux - sudo: required - dist: trusty - - env: - - SALT_NODE_ID=servo-mac1 - - SALT_FROM_SCRATCH=true - os: osx - osx_image: xcode7.3 - - env: - - SALT_NODE_ID=servo-linux1 - - SALT_FROM_SCRATCH=true - os: linux - sudo: required - dist: trusty - - env: - - SALT_NODE_ID=servo-master1 - - SALT_FROM_SCRATCH=true - os: linux - sudo: required - dist: trusty - # Salt from previous configuration - - env: - - SALT_NODE_ID=servo-linux-cross1 - - SALT_FROM_SCRATCH=false - os: linux - sudo: required - dist: trusty - - env: - - SALT_NODE_ID=servo-mac1 - - SALT_FROM_SCRATCH=false - os: osx - osx_image: xcode7.3 - - env: - - SALT_NODE_ID=servo-linux1 - - SALT_FROM_SCRATCH=false - os: linux - sudo: required - dist: trusty - - env: - - SALT_NODE_ID=servo-master1 - - SALT_FROM_SCRATCH=false - os: linux - sudo: required - dist: trusty - # Salt inside Docker - - env: - - SALT_NODE_ID=servo-linux1 - - SALT_FROM_SCRATCH=true - # ubuntu/14.04 - - SALT_DOCKER_IMAGE=ubuntu@sha256:edf05697d8ea17028a69726b4b450ad48da8b29884cd640fec950c904bfb50ce - os: linux - sudo: required - dist: trusty - - env: - - SALT_NODE_ID=servo-linux1 - - SALT_FROM_SCRATCH=true - # ubuntu/16.04 - - SALT_DOCKER_IMAGE=ubuntu@sha256:f3a61450ae43896c4332bda5e78b453f4a93179045f20c8181043b26b5e79028 - os: linux - sudo: required - dist: trusty - # Not a Salt node, runs test suite instead - - env: - - SALT_NODE_ID=test - os: linux - sudo: required - dist: trusty + # # Salt from scratch + # - env: + # - SALT_NODE_ID=servo-linux-cross1 + # - SALT_FROM_SCRATCH=true + # os: linux + # sudo: required + # dist: trusty + # - env: + # - SALT_NODE_ID=servo-mac1 + # - SALT_FROM_SCRATCH=true + # os: osx + # osx_image: xcode7.3 + # - env: + # - SALT_NODE_ID=servo-linux1 + # - SALT_FROM_SCRATCH=true + # os: linux + # sudo: required + # dist: trusty + # - env: + # - SALT_NODE_ID=servo-master1 + # - SALT_FROM_SCRATCH=true + # os: linux + # sudo: required + # dist: trusty + # # Salt from previous configuration + # - env: + # - SALT_NODE_ID=servo-linux-cross1 + # - SALT_FROM_SCRATCH=false + # os: linux + # sudo: required + # dist: trusty + # - env: + # - SALT_NODE_ID=servo-mac1 + # - SALT_FROM_SCRATCH=false + # os: osx + # osx_image: xcode7.3 + # - env: + # - SALT_NODE_ID=servo-linux1 + # - SALT_FROM_SCRATCH=false + # os: linux + # sudo: required + # dist: trusty + # - env: + # - SALT_NODE_ID=servo-master1 + # - SALT_FROM_SCRATCH=false + # os: linux + # sudo: required + # dist: trusty + # # Salt inside Docker + # - env: + # - SALT_NODE_ID=servo-linux1 + # - SALT_FROM_SCRATCH=true + # # ubuntu/14.04 + # - SALT_DOCKER_IMAGE=ubuntu@sha256:edf05697d8ea17028a69726b4b450ad48da8b29884cd640fec950c904bfb50ce + # os: linux + # sudo: required + # dist: trusty + # - env: + # - SALT_NODE_ID=servo-linux1 + # - SALT_FROM_SCRATCH=true + # # ubuntu/16.04 + # - SALT_DOCKER_IMAGE=ubuntu@sha256:f3a61450ae43896c4332bda5e78b453f4a93179045f20c8181043b26b5e79028 + # os: linux + # sudo: required + # dist: trusty + # # Not a Salt node, runs test suite instead + # - env: + # - SALT_NODE_ID=test + # os: linux + # sudo: required + # dist: trusty # Check that mach bootstrap works - env: - SALT_NODE_ID=mach-bootstrap diff --git a/.travis/dispatch.sh b/.travis/dispatch.sh index dc29d81e..48802c46 100755 --- a/.travis/dispatch.sh +++ b/.travis/dispatch.sh @@ -45,7 +45,7 @@ run_inside_docker() { # Reexec this script inside docker # (without exporting the `SALT_DOCKER_IMAGE` environment variable # to prevent recursion) - local -r DOCKER_SALT_ROOT="/tmp/salt" + local -r DOCKER_SALT_ROOT="/tmp/saltfs" # Use an env file for variables which may or may not be present local -r DOCKER_ENV_FILE="/tmp/docker-env-file" @@ -100,9 +100,9 @@ elif [[ "${SALT_NODE_ID}" == "mach-bootstrap" ]]; then # so that `python-apt` is available export PATH="/usr/bin:${PATH}" - # Install git (not present by default in Docker) - sudo apt-get update - sudo apt-get -y install git + # Install basic deps (not present by default in Docker) + ${SUDO} apt-get update + ${SUDO} apt-get -y install git python-apt python-dev python-pip python-virtualenv # Run mach bootstrap test separately from `test` because it installs things git clone --depth 1 https://github.com/servo/servo.git ../servo