diff --git a/.travis.yml b/.travis.yml index c8930413..91c29c7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ script: # Minimally validate YAML and Jinja at a basic level - sudo salt-call --id="${SALT_NODE_ID}" --retcode-passthrough state.show_highstate # Full on installation test - - sudo salt-call --id="${SALT_NODE_ID}" --retcode-passthrough state.highstate + - sudo salt-call --id="${SALT_NODE_ID}" --retcode-passthrough --log-level=warning state.highstate notifications: webhooks: http://build.servo.org:54856/travis diff --git a/README.md b/README.md index cd4aa6fa..07d3ac57 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## What's going on? Salt is a configuration management tool that we use to automate Servo's -infrastructure. See [the tutorials](http://docs.saltstack.com/en/latest/topics/tutorials/) to get started. +infrastructure. See [the tutorials](https://docs.saltstack.com/en/2015.5/topics/tutorials/index.html) to get started. ## Contributing diff --git a/STYLE_GUIDE.md b/STYLE_GUIDE.md index 2863316d..fae7238a 100644 --- a/STYLE_GUIDE.md +++ b/STYLE_GUIDE.md @@ -4,6 +4,19 @@ Style guide for Salt states (and other code) in this repo. Unfortunately, no linter exists yet for Salt states, so there is no automated way to check for compliance with this guide. +## General + +### Downloads + +URLs used for downloads should always use HTTPS. + +Note that APT repos are currently an exception - they don't seem to like +HTTPS urls, but they're GPG signed so this is OK. + +### Hash functions + +Hashes used for download/file verification should be SHA512 or stronger. + ## Jinja Usage ### Imports diff --git a/android-dependencies.sls b/android-dependencies.sls deleted file mode 100644 index 127fa32e..00000000 --- a/android-dependencies.sls +++ /dev/null @@ -1,88 +0,0 @@ -{% if '64' in grains['cpuarch'] %} -enable-i386-architecture: - cmd.run: - - name: 'dpkg --add-architecture i386' - - require_in: - - pkg: android-dependencies -{% endif %} - -android-dependencies: - pkg.installed: - - pkgs: - {% if '64' in grains['cpuarch'] %} - - libc6:i386 - - libstdc++6:i386 - {% endif %} - - default-jdk - - ant - - expect - - gcc - - g++ - - lib32z1 - - libstdc++6 - - libgl1-mesa-dev - - refresh: True - pip.installed: - - pkgs: - - s3cmd - - require: - - pkg: pip - -android-sdk: - archive.extracted: - - name: /home/servo # Directory to extract into - - source: http://dl.google.com/android/android-sdk_r24.4.1-linux.tgz - - source_hash: sha512=96fb71d78a8c2833afeba6df617edcd6cc4e37ecd0c3bec38c39e78204ed3c2bd54b138a56086bf5ccd95e372e3c36e72c1550c13df8232ec19537da93049284 - - archive_format: tar - - archive_user: servo # 2015.8 moves these to the standard user and group parameters - - if_missing: /home/servo/android-sdk_r24.4.1-linux.tgz - cmd.wait: - # The arguments to --filter are from running 'android list sdk' - # Currently these are: - # platform-tool: Android SDK Platform-tools, revision 23.0.1 - # 9: SDK Platform Android 4.3.1, API 18, revision 3 - - name: | - expect -c ' - set timeout -1; - spawn /home/servo/android-sdk-linux/tools/android - update sdk --no-ui --filter platform-tool,9; - expect { - "Do you accept the license" { exp_send "y\r" ; exp_continue } - eof - } - ' - - user: servo - - require: - - pkg: android-dependencies - - watch: - - archive: android-sdk - -android-ndk: - file.managed: - - name: /home/servo/android-ndk-r10e-linux-x86_64.bin - - source: http://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin - - source_hash: sha512=8948c7bd1621e32dce554d5cd1268ffda2e9c5e6b2dda5b8cf0266ea60aa2dd6fddf8d290683fc1ef0b69d66c898226c7f52cc567dbb14352b4191ac19dfb371 - - user: servo - - group: servo - - mode: 777 - cmd.wait: - # Need to filter log output to avoid hitting log limits on Travis CI - - name: /home/servo/android-ndk-r10e-linux-x86_64.bin | grep -v Extracting - - user: servo - - watch: - - file: android-ndk - -android-ndk-toolset-configuration: - cmd.wait: - - name: bash /home/servo/android-ndk-r10e/build/tools/make-standalone-toolchain.sh --platform=android-18 --toolchain=arm-linux-androideabi-4.8 --install-dir='/home/servo/ndk-toolchain' --ndk-dir='/home/servo/android-ndk-r10e' - - user: servo - - require: - - cmd: android-sdk - - watch: - - cmd: android-ndk - -/home/servo/.bash_profile: - file.managed: - - source: salt://bash/dot.bash_profile - - user: servo - - group: servo - - mode: 0644 diff --git a/bash/dot.bash_profile b/bash/dot.bash_profile deleted file mode 100644 index 3babb243..00000000 --- a/bash/dot.bash_profile +++ /dev/null @@ -1,4 +0,0 @@ -export ANDROID_SDK=$HOME/android-sdk-linux/ -export ANDROID_NDK=$HOME/android-ndk-r10c/ -export ANDROID_TOOLCHAIN=$HOME/ndk-toolchain/ -PATH="$PATH:$HOME/android-sdk-linux/platform-tools:$HOME/ndk-toolchain/bin" diff --git a/buildbot/master.sls b/buildbot/master.sls index e465cb19..cf77cfea 100644 --- a/buildbot/master.sls +++ b/buildbot/master.sls @@ -1,3 +1,5 @@ +{% from 'common/map.jinja' import common %} + buildbot-master: pip.installed: - pkgs: @@ -17,11 +19,13 @@ buildbot-master: /home/servo/buildbot/master: file.recurse: - source: salt://buildbot/master - - template: jinja - user: servo - group: servo - dir_mode: 755 - file_mode: 644 + - template: jinja + - context: + common: {{ common }} /etc/init/buildbot-master.conf: file.managed: diff --git a/buildbot/master/master.cfg b/buildbot/master/master.cfg index 15203e24..a4dfcd3c 100644 --- a/buildbot/master/master.cfg +++ b/buildbot/master/master.cfg @@ -133,9 +133,9 @@ common_test_env = { linux_test_env = dict({ 'DISPLAY': ':0', - 'CARGO_HOME': '/home/servo/.cargo', + 'CARGO_HOME': '{{ common.servo_home }}/.cargo', 'CCACHE': '/usr/bin/ccache', - 'SERVO_CACHE_DIR': '/home/servo/.servo', + 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', 'SHELL': '/bin/bash' }, **common_test_env) linux_headless_env = dict({'SERVO_HEADLESS': '1'}, **linux_test_env) @@ -210,10 +210,10 @@ mac_rel_css_factory = create_servo_factory([ steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), ]) -android_compile_env = dict({'ANDROID_SDK': '/home/servo/android-sdk-linux/', - 'ANDROID_NDK': '/home/servo/android-ndk-r10e/', - 'ANDROID_TOOLCHAIN': '/home/servo/ndk-toolchain/', - 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:/home/servo/android-sdk-linux/platform-tools:/home/servo/ndk-toolchain/bin'}, +android_compile_env = dict({'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/', + 'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/', + 'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/', + 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:{{ common.servo_home }}/android/sdk/current/platform-tools:{{ common.servo_home }}/android/toolchain/current/bin'}, **linux_test_env) android_factory = create_servo_factory([ steps.Compile(command=["./mach", "build", "--android", "--dev"], env=android_compile_env), @@ -226,15 +226,15 @@ android_nightly_factory = create_servo_factory([ steps.Compile(command=["./mach", "build", "--android", "--release"], env=android_compile_env), steps.Compile(command=["./mach", "package", "-r"], env=android_compile_env), steps.ShellCommand(command=["s3cmd", "put", - "/home/servo/buildbot/slave/android-nightly/build/target/arm-linux-androideabi/release/servo.apk", + "{{ common.servo_home }}/buildbot/slave/android-nightly/build/target/arm-linux-androideabi/release/servo.apk", "s3://servo-rust/nightly/servo.apk"]), ]) -gonk_compile_env = dict({'ANDROID_SDK': '/home/servo/android-sdk-linux/', - 'ANDROID_NDK': '/home/servo/android-ndk-r10e/', - 'ANDROID_TOOLCHAIN': '/home/servo/ndk-toolchain/', - 'GONKDIR': '/home/servo/B2G/', - 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:/home/servo/B2G/prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.7/bin'}, +gonk_compile_env = dict({'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/', + 'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/', + 'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/', + 'GONKDIR': '{{ common.servo_home }}/B2G/', + 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:{{ common.servo_home }}/B2G/prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.7/bin'}, **linux_test_env) gonk_factory = create_servo_factory([ @@ -243,7 +243,7 @@ gonk_factory = create_servo_factory([ steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=gonk_compile_env), ]) -arm32_compile_env = dict({'PATH': '/home/servo/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', +arm32_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', 'BUILD_TARGET': 'arm-unknown-linux-gnueabihf', 'PKG_CONFIG_ALLOW_CROSS': '1', 'PKG_CONFIG_PATH': '/usr/lib/arm-linux-gnueabihf/pkgconfig', @@ -257,7 +257,7 @@ arm32_factory = create_servo_factory([ steps.Compile(command=["./mach", "build", "--rel", "--target=arm-unknown-linux-gnueabihf"], env=arm32_compile_env), ]) -arm64_compile_env = dict({'PATH': '/home/servo/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', +arm64_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', 'BUILD_TARGET': 'aarch64-unknown-linux-gnu', 'PKG_CONFIG_ALLOW_CROSS': '1', 'PKG_CONFIG_PATH': '/usr/lib/aarch64-linux-gnu/pkgconfig', @@ -274,8 +274,8 @@ arm64_factory = create_servo_factory([ doc_factory = create_servo_factory([ steps.ShellCommand(command=["etc/ci/upload_docs.sh"], env={'TOKEN': GITHUB_DOC_TOKEN, - 'CARGO_HOME': '/home/servo/.cargo', - 'SERVO_CACHE_DIR': '/home/servo/.servo', + 'CARGO_HOME': '{{ common.servo_home }}/.cargo', + 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', 'SHELL': '/bin/bash'}, # important not to leak token logEnviron=False), diff --git a/git/init.sls b/git/init.sls new file mode 100644 index 00000000..19c05d55 --- /dev/null +++ b/git/init.sls @@ -0,0 +1,5 @@ +# Used to keep the Salt tree up to date on the master +git: + pkg.installed: + - pkgs: + - git diff --git a/gonk-dependencies.sls b/gonk-dependencies.sls deleted file mode 100644 index 3829ae07..00000000 --- a/gonk-dependencies.sls +++ /dev/null @@ -1,12 +0,0 @@ -b2g-download: - file.managed: - - name: /home/servo/B2G.tgz - - source: https://servo-rust.s3.amazonaws.com/B2G/B2G.tgz - - source_hash: sha1=2f871c23ddff795938cf9f4764f926bcf91b5938 - - user: servo - - group: servo - cmd.wait: - - name: tar xzf /home/servo/B2G.tgz - - user: servo - - watch: - - file: b2g-download diff --git a/servo-build-dependencies/android.sls b/servo-build-dependencies/android.sls new file mode 100644 index 00000000..e8d2432f --- /dev/null +++ b/servo-build-dependencies/android.sls @@ -0,0 +1,123 @@ +{% from 'common/map.jinja' import common %} +{% from tpldir ~ '/map.jinja' import android %} + +{% if '64' in grains['cpuarch'] %} +enable-i386-architecture: + cmd.run: + - name: 'dpkg --add-architecture i386' + - require_in: + - pkg: android-dependencies +{% endif %} + +android-dependencies: + pkg.installed: + - pkgs: + {% if '64' in grains['cpuarch'] %} + - libc6:i386 + - libstdc++6:i386 + {% endif %} + - default-jdk + - ant + - expect + - gcc + - g++ + - lib32z1 + - libstdc++6 + - libgl1-mesa-dev + - refresh: True + pip.installed: + - pkgs: + - s3cmd + - require: + - pkg: pip + + +android-sdk: + archive.extracted: + - name: {{ common.servo_home }}/android/sdk/{{ android.sdk.version }} + - source: https://dl.google.com/android/android-sdk_{{ android.sdk.version }}-linux.tgz + - source_hash: sha512={{ android.sdk.sha512 }} + - archive_format: tar + - archive_user: servo + - if_missing: {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux + - require: + - user: servo + cmd.run: + # The arguments to --filter are from running 'android list sdk' + # Currently these are: + # platform-tool: Android SDK Platform-tools, revision 23.0.1 + # 9: SDK Platform Android 4.3.1, API 18, revision 3 + - name: | + expect -c ' + set timeout -1; + spawn {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux/tools/android - update sdk --no-ui --filter platform-tool,9; + expect { + "Do you accept the license" { exp_send "y\r" ; exp_continue } + eof + } + ' + - user: servo + - creates: {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux/platform-tools + - require: + - pkg: android-dependencies + - archive: android-sdk + +android-sdk-current: + file.symlink: + - name: {{ common.servo_home }}/android/sdk/current + - target: {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux + - user: servo + - group: servo + - require: + - cmd: android-sdk + + +android-ndk: + file.managed: + - name: {{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}-linux-x86_64.bin + - source: https://dl.google.com/android/ndk/android-ndk-{{ android.ndk.version }}-linux-x86_64.bin + - source_hash: sha512={{ android.ndk.sha512 }} + - user: servo + - group: servo + - mode: 744 + - dir_mode: 755 + - makedirs: True + - require: + - user: servo + cmd.run: + # Need to filter log output to avoid hitting log limits on Travis CI + - name: '{{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}-linux-x86_64.bin | grep -v Extracting' + - user: servo + - cwd: {{ common.servo_home }}/android/ndk/{{ android.ndk.version }} + - creates: {{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }} + - require: + - file: android-ndk + +android-toolchain: + cmd.run: + - name: bash {{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}/build/tools/make-standalone-toolchain.sh --platform=android-18 --toolchain=arm-linux-androideabi-4.8 --install-dir='{{ common.servo_home }}/android/toolchain/{{ android.ndk.version }}/android-toolchain' --ndk-dir='{{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}' + - user: servo + - creates: {{ common.servo_home }}/android/toolchain/{{ android.ndk.version }}/android-toolchain + - require: + - cmd: android-ndk + +# Toolchain depends on NDK so update the symlinks together +android-ndk-current: + file.symlink: + - name: {{ common.servo_home }}/android/ndk/current + - target: {{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }} + - user: servo + - group: servo + - require: + - cmd: android-ndk + - cmd: android-toolchain + +android-toolchain-current: + file.symlink: + - name: {{ common.servo_home }}/android/toolchain/current + - target: {{ common.servo_home }}/android/toolchain/{{ android.ndk.version }}/android-toolchain + - user: servo + - group: servo + - require: + - cmd: android-ndk + - cmd: android-toolchain diff --git a/arm-dependencies.sls b/servo-build-dependencies/arm.sls similarity index 100% rename from arm-dependencies.sls rename to servo-build-dependencies/arm.sls diff --git a/servo-build-dependencies/gonk.sls b/servo-build-dependencies/gonk.sls new file mode 100644 index 00000000..9cb2337c --- /dev/null +++ b/servo-build-dependencies/gonk.sls @@ -0,0 +1,11 @@ +{% from 'common/map.jinja' import common %} +{% from tpldir ~ '/map.jinja' import b2g %} + +b2g: + archive.extracted: + - name: {{ common.servo_home }} + - source: https://servo-rust.s3.amazonaws.com/B2G/B2G.tgz + - source_hash: sha512={{ b2g.sha512 }} + - archive_format: tar + - archive_user: servo + - if_missing: {{ common.servo_home }}/B2G diff --git a/servo-dependencies.sls b/servo-build-dependencies/init.sls similarity index 100% rename from servo-dependencies.sls rename to servo-build-dependencies/init.sls diff --git a/servo-build-dependencies/map.jinja b/servo-build-dependencies/map.jinja new file mode 100644 index 00000000..8c2a08c2 --- /dev/null +++ b/servo-build-dependencies/map.jinja @@ -0,0 +1,18 @@ +{% + set android = { + 'sdk': { + 'version': 'r24.4.1', + 'sha512': '96fb71d78a8c2833afeba6df617edcd6cc4e37ecd0c3bec38c39e78204ed3c2bd54b138a56086bf5ccd95e372e3c36e72c1550c13df8232ec19537da93049284' + }, + 'ndk': { + 'version': 'r10e', + 'sha512': '8948c7bd1621e32dce554d5cd1268ffda2e9c5e6b2dda5b8cf0266ea60aa2dd6fddf8d290683fc1ef0b69d66c898226c7f52cc567dbb14352b4191ac19dfb371' + } + } +%} + +{% + set b2g = { + 'sha512': '092a3c9210ce4eae864dc07a4a17f5fc38aebdd0179d64d7b77a65bad2adc004a9599ef7bf1e6ce610abaecae87cd61af5d58ea5ac609028b429e514dfe96e77' + } +%} diff --git a/top.sls b/top.sls index c25cc611..97f4715d 100644 --- a/top.sls +++ b/top.sls @@ -3,7 +3,6 @@ base: '*': - common - - servo-dependencies - salt.common 'os:Ubuntu': @@ -12,26 +11,31 @@ base: 'servo-head': - buildbot.slave - - android-dependencies + - servo-build-dependencies + - servo-build-dependencies.android 'servo-linux-cross\d+': - match: pcre - buildbot.slave - - android-dependencies - - gonk-dependencies - - arm-dependencies + - servo-build-dependencies + - servo-build-dependencies.android + - servo-build-dependencies.gonk + - servo-build-dependencies.arm 'servo-(mac|macpro)\d+': - match: pcre - osx - buildbot.slave + - servo-build-dependencies 'servo-linux\d+': - match: pcre - buildbot.slave + - servo-build-dependencies - xvfb 'servo-master': + - git - buildbot.master - homu - nginx