From caa1d410e4bb67a1cb1e156e91d605f911e27b7a Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 16 Apr 2016 13:16:53 -0400 Subject: [PATCH 01/12] Remove gonk In b7fc797f2f43a334ce35e5b9b96bad5733342dc7, we disabled blocking on gonk; since we're not using the results, removing the builder will use less resources. --- buildbot/master/files/config/master.cfg | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index f525d63f..08bc519e 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -48,7 +48,7 @@ c['schedulers'] = [] c['schedulers'].append(schedulers.AnyBranchScheduler( name="servo-auto", treeStableTimer=None, - builderNames=["linux-dev", "linux-rel", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", "android", "gonk", "arm32", "arm64", "windows"], + builderNames=["linux-dev", "linux-rel", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", "android", "arm32", "arm64", "windows"], change_filter=util.ChangeFilter(filter_fn=servo_auto_try_filter), )) c['schedulers'].append(schedulers.SingleBranchScheduler( @@ -59,7 +59,7 @@ c['schedulers'].append(schedulers.SingleBranchScheduler( )) c['schedulers'].append(schedulers.ForceScheduler( name="force", - builderNames=["linux-dev", "linux-rel", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", "android", "gonk", "arm32", "arm64", "android-nightly", "windows"] + builderNames=["linux-dev", "linux-rel", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", "android", "arm32", "arm64", "android-nightly", "windows"] )) c['schedulers'].append(schedulers.Nightly( name="Nightly", @@ -202,19 +202,6 @@ android_nightly_factory = create_servo_factory([ "s3://servo-rust/nightly/servo.apk"]), ]) -gonk_compile_env = dict({'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/', - 'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/', - 'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/', - 'GONKDIR': '{{ common.servo_home }}/B2G/', - 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:{{ common.servo_home }}/B2G/prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.7/bin'}, - **linux_test_env) - -gonk_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build-gonk"], env=gonk_compile_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=gonk_compile_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=gonk_compile_env), -]) - arm32_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', 'BUILD_TARGET': 'arm-unknown-linux-gnueabihf', 'PKG_CONFIG_ALLOW_CROSS': '1', @@ -292,13 +279,6 @@ c['builders'].append(util.BuilderConfig( nextBuild=branch_priority, category="auto", )) -c['builders'].append(util.BuilderConfig( - name="gonk", - slavenames=CROSS_SLAVES, - factory=gonk_factory, - nextBuild=branch_priority, - category="auto", -)) c['builders'].append(util.BuilderConfig( name="arm32", slavenames=CROSS_SLAVES, From 36b16ffdcc6b4bfb8b7d30475eb9555356d8d2a8 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 16 Apr 2016 13:19:23 -0400 Subject: [PATCH 02/12] Temporarily remove pkill steps We should be running pkill (or the platform-specific equivalent) at the start and end of each build, but this is really a CI detail, not something that is integral to the list of steps, so remove it for now. --- buildbot/master/files/config/master.cfg | 5 ----- 1 file changed, 5 deletions(-) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 08bc519e..0f9f138f 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -2,7 +2,6 @@ # ex: set syntax=python: from buildbot.plugins import * -from buildbot.status.results import SUCCESS from passwords import HTTP_USERNAME, HTTP_PASSWORD, SLAVE_PASSWORD, CHANGE_PASSWORD from passwords import GITHUB_DOC_TOKEN, GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET @@ -124,7 +123,6 @@ linux_rel_factory = create_servo_factory([ env=linux_headless_env, logfiles={"test-wpt.log": "test-wpt.log", "wpt_errorsummary.log": "wpt_errorsummary.log"}), - #steps.ShellCommand(command=["pkill", "-x", "servo"], decodeRC={0: SUCCESS, 1: SUCCESS}), steps.ShellCommand(command=["gzip", "test-wpt.log"], env=linux_headless_env), steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "16", "--log-raw", "test-css.log", @@ -132,7 +130,6 @@ linux_rel_factory = create_servo_factory([ env=linux_headless_env, logfiles={"test-css.log": "test-css.log", "css_errorsummary.log": "css_errorsummary.log"}), - #steps.ShellCommand(command=["pkill", "-x", "servo"], decodeRC={0: SUCCESS, 1: SUCCESS}), steps.ShellCommand(command=["gzip", "test-css.log"], env=linux_headless_env), steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env), steps.Compile(command=["./mach", "build-geckolib", "--release"], env=linux_headless_env), @@ -151,7 +148,6 @@ mac_rel_wpt_factory = create_servo_factory([ env=mac_test_env, logfiles={"test-wpt.log": "test-wpt.log", "wpt_errorsummary.log": "wpt_errorsummary.log"}), - steps.ShellCommand(command=["pkill", "-x", "servo"], decodeRC={0: SUCCESS, 1: SUCCESS}), steps.ShellCommand(command=["gzip", "test-wpt.log"], env=mac_test_env), steps.ShellCommand(command=["./mach", "build-cef", "--release"], env=mac_test_env), steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), @@ -175,7 +171,6 @@ mac_rel_css_factory = create_servo_factory([ env=mac_test_env, logfiles={"test-css.log": "test-css.log", "css_errorsummary.log": "css_errorsummary.log"}), - steps.ShellCommand(command=["pkill", "-x", "servo"], decodeRC={0: SUCCESS, 1: SUCCESS}), steps.ShellCommand(command=["gzip", "test-css.log"], env=mac_test_env), steps.ShellCommand(command=["./mach", "build-geckolib", "--release"], env=mac_test_env), steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), From ceff384021c75d8f95b9b5319addb27ac6d627a6 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 16 Apr 2016 13:20:54 -0400 Subject: [PATCH 03/12] Remove gzip steps The steps gzipping build logs did not have logfiles listed, so they were not available for download from the web interface. The only way to get the logs is via ssh/scp, which works just as well for non-gzipped logs. This will cut down on build times. --- buildbot/master/files/config/master.cfg | 4 ---- 1 file changed, 4 deletions(-) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 0f9f138f..0b15e7d2 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -123,14 +123,12 @@ linux_rel_factory = create_servo_factory([ env=linux_headless_env, logfiles={"test-wpt.log": "test-wpt.log", "wpt_errorsummary.log": "wpt_errorsummary.log"}), - steps.ShellCommand(command=["gzip", "test-wpt.log"], env=linux_headless_env), steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "16", "--log-raw", "test-css.log", "--log-errorsummary", "css_errorsummary.log"], env=linux_headless_env, logfiles={"test-css.log": "test-css.log", "css_errorsummary.log": "css_errorsummary.log"}), - steps.ShellCommand(command=["gzip", "test-css.log"], env=linux_headless_env), steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env), steps.Compile(command=["./mach", "build-geckolib", "--release"], env=linux_headless_env), steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=linux_headless_env), @@ -148,7 +146,6 @@ mac_rel_wpt_factory = create_servo_factory([ env=mac_test_env, logfiles={"test-wpt.log": "test-wpt.log", "wpt_errorsummary.log": "wpt_errorsummary.log"}), - steps.ShellCommand(command=["gzip", "test-wpt.log"], env=mac_test_env), steps.ShellCommand(command=["./mach", "build-cef", "--release"], env=mac_test_env), steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), @@ -171,7 +168,6 @@ mac_rel_css_factory = create_servo_factory([ env=mac_test_env, logfiles={"test-css.log": "test-css.log", "css_errorsummary.log": "css_errorsummary.log"}), - steps.ShellCommand(command=["gzip", "test-css.log"], env=mac_test_env), steps.ShellCommand(command=["./mach", "build-geckolib", "--release"], env=mac_test_env), steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), From 451f3f974da71983c040e322cbf591eb74a640b9 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 16 Apr 2016 14:46:06 -0400 Subject: [PATCH 04/12] Create separate file for environments The existing mechanism for chaining environment dictionaries is a hack that Guido has noted as abuse of the '**' mechanism: https://mail.python.org/pipermail/python-dev/2010-April/099459.html Create a subclass that allows 'adding' in a single expression, as copy() + update() takes two lines. Use str.join for listing PATHs to keep lines short. This also exposed an error in the Windows PATH, with a missing semicolon between r'C:\msys64\usr\bin' and r'C:\Windows\system32'. Move the environments into a separate file to keep them all together. Rename the various environment variables for clarity and consistency. The environments.py file now passes flake8; this should be added to our test suite. --- buildbot/master/files/config/environments.py | 119 +++++++++++++++ buildbot/master/files/config/master.cfg | 148 ++++++------------- 2 files changed, 167 insertions(+), 100 deletions(-) create mode 100644 buildbot/master/files/config/environments.py diff --git a/buildbot/master/files/config/environments.py b/buildbot/master/files/config/environments.py new file mode 100644 index 00000000..965405e1 --- /dev/null +++ b/buildbot/master/files/config/environments.py @@ -0,0 +1,119 @@ +from passwords import GITHUB_DOC_TOKEN + + +class Environment(dict): + """ + Wrapper that allows 'adding' environment dictionaries + to make it easy to build up environments piece by piece. + """ + + def __init__(self, *args, **kwargs): + super(Environment, self).__init__(*args, **kwargs) + + def __add__(self, other): + assert type(self) == type(other) + combined = self.copy() + combined.update(other) # other takes precedence over self + return Environment(combined) + + +doc = Environment({ + 'CARGO_HOME': '{{ common.servo_home }}/.cargo', + 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', + 'SHELL': '/bin/bash', + 'TOKEN': GITHUB_DOC_TOKEN, +}) + +build_common = Environment({ + 'RUST_BACKTRACE': '1', +}) + +build_windows = build_common + Environment({ + 'MSYS': 'winsymlinks=lnk', + 'MSYSTEM': 'MINGW64', + 'PATH': ';'.join([ + r'C:\msys64\mingw64\bin', + r'C:\msys64\usr\bin', + r'C:\Windows\system32', + r'C:\Windows', + r'C:\Windows\System32\Wbem', + r'C:\Windows\System32\WindowsPowerShell\v1.0', + r'C:\Program Files\Amazon\cfn-bootstrap', + ]), +}) + +build_mac = build_common + Environment({ + 'CARGO_HOME': '/Users/servo/.cargo', + 'CCACHE': '/usr/local/bin/ccache', + 'SERVO_CACHE_DIR': '/Users/servo/.servo', +}) + + +build_linux = build_common + Environment({ + 'CARGO_HOME': '{{ common.servo_home }}/.cargo', + 'CCACHE': '/usr/bin/ccache', + 'DISPLAY': ':0', + 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', + 'SHELL': '/bin/bash', +}) + +build_linux_headless = build_linux + Environment({ + 'SERVO_HEADLESS': '1', +}) + +build_android = build_linux + Environment({ + 'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/', + 'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/', + 'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/', + 'PATH': ':'.join([ + '/usr/local/sbin', + '/usr/local/bin', + '/usr/bin', + '/usr/sbin', + '/sbin', + '/bin', + '{{ common.servo_home }}/android/sdk/current/platform-tools', + '{{ common.servo_home }}/android/toolchain/current/bin', + ]), +}) + +build_arm = build_linux + Environment({ + 'EXPAT_NO_PKG_CONFIG': '1', + 'FONTCONFIG_NO_PKG_CONFIG': '1', + 'FREETYPE2_NO_PKG_CONFIG': '1', + 'PKG_CONFIG_ALLOW_CROSS': '1', +}) + + +build_arm32 = build_arm + Environment({ + 'BUILD_TARGET': 'arm-unknown-linux-gnueabihf', + 'CC': 'arm-linux-gnueabihf-gcc', + 'CXX': 'arm-linux-gnueabihf-g++', + 'PATH': ':'.join([ + '{{ common.servo_home }}/bin', + '/usr/local/sbin', + '/usr/local/bin', + '/usr/bin', + '/usr/sbin', + '/sbin', + '/bin', + ]), + 'PKG_CONFIG_PATH': '/usr/lib/arm-linux-gnueabihf/pkgconfig', +}) + +build_arm64 = build_arm + Environment({ + 'BUILD_TARGET': 'aarch64-unknown-linux-gnu', + 'CC': 'aarch64-linux-gnu-gcc', + 'CXX': 'aarch64-linux-gnu-g++', + 'PATH': ':'.join([ + '{{ common.servo_home }}/bin', + '/usr/local/sbin', + '/usr/local/bin', + '/usr/bin', + '/usr/sbin', + '/sbin', + '/bin', + ]), + 'PKG_CONFIG_PATH': '/usr/lib/aarch64-linux-gnu/pkgconfig', + 'SERVO_RUSTC_WITH_GOLD': 'False', +}) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 0b15e7d2..5d95525b 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -3,8 +3,9 @@ from buildbot.plugins import * +import environments from passwords import HTTP_USERNAME, HTTP_PASSWORD, SLAVE_PASSWORD, CHANGE_PASSWORD -from passwords import GITHUB_DOC_TOKEN, GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET +from passwords import GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET SERVO_REPO = "https://github.com/servo/servo" LINUX_RESERVED_SLAVES = ["servo-linux1", "servo-linux2"] @@ -83,161 +84,108 @@ def create_servo_factory(commands): ] + commands) -common_test_env = { - 'RUST_BACKTRACE': '1' -} - -linux_test_env = dict({ - 'DISPLAY': ':0', - 'CARGO_HOME': '{{ common.servo_home }}/.cargo', - 'CCACHE': '/usr/bin/ccache', - 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', - 'SHELL': '/bin/bash' -}, **common_test_env) -linux_headless_env = dict({'SERVO_HEADLESS': '1'}, **linux_test_env) - -mac_test_env = dict({ - 'CARGO_HOME': '/Users/servo/.cargo', - 'CCACHE': '/usr/local/bin/ccache', - 'SERVO_CACHE_DIR': '/Users/servo/.servo' -}, **common_test_env) - linux_dev_factory = create_servo_factory([ - steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress", "--self-test"], env=linux_headless_env), - steps.Compile(command=["./mach", "build", "--dev"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-compiletest"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-unit"], env=linux_headless_env), - steps.Compile(command=["./mach", "build-cef"], env=linux_headless_env), - steps.Compile(command=["./mach", "build-geckolib"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=linux_headless_env), + steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"], env=environments.build_linux_headless), + steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress", "--self-test"], env=environments.build_linux_headless), + steps.Compile(command=["./mach", "build", "--dev"], env=environments.build_linux_headless), + steps.ShellCommand(command=["./mach", "test-compiletest"], env=environments.build_linux_headless), + steps.ShellCommand(command=["./mach", "test-unit"], env=environments.build_linux_headless), + steps.Compile(command=["./mach", "build-cef"], env=environments.build_linux_headless), + steps.Compile(command=["./mach", "build-geckolib"], env=environments.build_linux_headless), + steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_linux_headless), + steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_linux_headless), ]) linux_rel_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--release"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=linux_headless_env), + steps.Compile(command=["./mach", "build", "--release"], env=environments.build_linux_headless), + steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=environments.build_linux_headless), steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "24", "--log-raw", "test-wpt.log", "--log-errorsummary", "wpt_errorsummary.log"], - env=linux_headless_env, + env=environments.build_linux_headless, logfiles={"test-wpt.log": "test-wpt.log", "wpt_errorsummary.log": "wpt_errorsummary.log"}), steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "16", "--log-raw", "test-css.log", "--log-errorsummary", "css_errorsummary.log"], - env=linux_headless_env, + env=environments.build_linux_headless, logfiles={"test-css.log": "test-css.log", "css_errorsummary.log": "css_errorsummary.log"}), - steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env), - steps.Compile(command=["./mach", "build-geckolib", "--release"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/check_no_unwrap.sh"], env=linux_headless_env), + steps.Compile(command=["./mach", "build-cef", "--release"], env=environments.build_linux_headless), + steps.Compile(command=["./mach", "build-geckolib", "--release"], env=environments.build_linux_headless), + steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_linux_headless), + steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_linux_headless), + steps.ShellCommand(command=["bash", "./etc/ci/check_no_unwrap.sh"], env=environments.build_linux_headless), ]) mac_rel_wpt_factory = create_servo_factory([ steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"]), - steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env), - steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=mac_test_env), + steps.Compile(command=["./mach", "build", "--release"], env=environments.build_mac), + steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=environments.build_mac), steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", "--log-raw", "test-wpt.log", "--log-errorsummary", "wpt_errorsummary.log"], - env=mac_test_env, + env=environments.build_mac, logfiles={"test-wpt.log": "test-wpt.log", "wpt_errorsummary.log": "wpt_errorsummary.log"}), - steps.ShellCommand(command=["./mach", "build-cef", "--release"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), + steps.ShellCommand(command=["./mach", "build-cef", "--release"], env=environments.build_mac), + steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac), + steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac), ]) mac_dev_unit_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--dev"], env=mac_test_env), - steps.ShellCommand(command=["./mach", "test-unit"], env=mac_test_env), - steps.Compile(command=["./mach", "build-cef"], env=mac_test_env), - steps.Compile(command=["./mach", "build-geckolib"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), + steps.Compile(command=["./mach", "build", "--dev"], env=environments.build_mac), + steps.ShellCommand(command=["./mach", "test-unit"], env=environments.build_mac), + steps.Compile(command=["./mach", "build-cef"], env=environments.build_mac), + steps.Compile(command=["./mach", "build-geckolib"], env=environments.build_mac), + steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac), + steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac), ]) mac_rel_css_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env), + steps.Compile(command=["./mach", "build", "--release"], env=environments.build_mac), steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "4", "--log-raw", "test-css.log", "--log-errorsummary", "css_errorsummary.log"], - env=mac_test_env, + env=environments.build_mac, logfiles={"test-css.log": "test-css.log", "css_errorsummary.log": "css_errorsummary.log"}), - steps.ShellCommand(command=["./mach", "build-geckolib", "--release"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), + steps.ShellCommand(command=["./mach", "build-geckolib", "--release"], env=environments.build_mac), + steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac), + steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac), ]) -android_compile_env = dict({'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/', - 'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/', - 'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/', - 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:{{ common.servo_home }}/android/sdk/current/platform-tools:{{ common.servo_home }}/android/toolchain/current/bin'}, - **linux_test_env) android_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--android", "--dev"], env=android_compile_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=android_compile_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=android_compile_env), - steps.ShellCommand(command=["python", "./etc/ci/check_dynamic_symbols.py"], env=android_compile_env), + steps.Compile(command=["./mach", "build", "--android", "--dev"], env=environments.build_android), + steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_android), + steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_android), + steps.ShellCommand(command=["python", "./etc/ci/check_dynamic_symbols.py"], env=environments.build_android), ]) android_nightly_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--android", "--release"], env=android_compile_env), - steps.Compile(command=["./mach", "package", "-r"], env=android_compile_env), + steps.Compile(command=["./mach", "build", "--android", "--release"], env=environments.build_android), + steps.Compile(command=["./mach", "package", "-r"], env=environments.build_android), steps.ShellCommand(command=["s3cmd", "put", "{{ common.servo_home }}/buildbot/slave/android-nightly/build/target/arm-linux-androideabi/release/servo.apk", "s3://servo-rust/nightly/servo.apk"]), ]) -arm32_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', - 'BUILD_TARGET': 'arm-unknown-linux-gnueabihf', - 'PKG_CONFIG_ALLOW_CROSS': '1', - 'PKG_CONFIG_PATH': '/usr/lib/arm-linux-gnueabihf/pkgconfig', - 'EXPAT_NO_PKG_CONFIG': '1', - 'FREETYPE2_NO_PKG_CONFIG': '1', - 'FONTCONFIG_NO_PKG_CONFIG': '1', - 'CC': 'arm-linux-gnueabihf-gcc', - 'CXX': 'arm-linux-gnueabihf-g++'}, - **linux_test_env) arm32_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--rel", "--target=arm-unknown-linux-gnueabihf"], env=arm32_compile_env), + steps.Compile(command=["./mach", "build", "--rel", "--target=arm-unknown-linux-gnueabihf"], env=environments.build_arm32), ]) -arm64_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', - 'BUILD_TARGET': 'aarch64-unknown-linux-gnu', - 'PKG_CONFIG_ALLOW_CROSS': '1', - 'PKG_CONFIG_PATH': '/usr/lib/aarch64-linux-gnu/pkgconfig', - 'EXPAT_NO_PKG_CONFIG': '1', - 'FREETYPE2_NO_PKG_CONFIG': '1', - 'FONTCONFIG_NO_PKG_CONFIG': '1', - 'CC': 'aarch64-linux-gnu-gcc', - 'SERVO_RUSTC_WITH_GOLD': 'False', - 'CXX': 'aarch64-linux-gnu-g++'}, - **linux_test_env) arm64_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--rel", "--target=aarch64-unknown-linux-gnu"], env=arm64_compile_env), + steps.Compile(command=["./mach", "build", "--rel", "--target=aarch64-unknown-linux-gnu"], env=environments.build_arm64), ]) -windows_compile_env = dict({'PATH': r'C:\msys64\mingw64\bin;C:\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0;C:\Program Files\Amazon\cfn-bootstrap', - 'MSYSTEM': 'MINGW64', - 'MSYS': 'winsymlinks=lnk'}, - **common_test_env) - windows_factory = create_servo_factory([ - steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"], env=windows_compile_env), - steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], env=windows_compile_env), + steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"], env=environments.build_windows), + steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], env=environments.build_windows), ]) doc_factory = create_servo_factory([ steps.ShellCommand(command=["etc/ci/upload_docs.sh"], - env={'TOKEN': GITHUB_DOC_TOKEN, - 'CARGO_HOME': '{{ common.servo_home }}/.cargo', - 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', - 'SHELL': '/bin/bash'}, + env=environments.doc, # important not to leak token logEnviron=False), ]) From 85979c4437721adc103110701107b38fd7fe6eb8 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 16 Apr 2016 19:53:39 -0400 Subject: [PATCH 05/12] Create separate file for steps In order to move the build steps to the main servo repository, they must be converted from python code that requires the Buildbot library to be installed (to manually build steps) into more declarative steps. This commit starts this conversion by using heuristics to convert a list of shell commands (akin to those in a .travis.yml or .appveyor.yml file) into the appropriate Buildbot Steps. The doc factory and windows factory still use hardcoded steps; see the comments in the code for more details. The android nightly factory now uses a relative instead of absolute path for the .apk file to make the Jinja templating unnecessary. --- buildbot/master/files/config/factories.py | 187 ++++++++++++++++++++++ buildbot/master/files/config/master.cfg | 144 ++--------------- 2 files changed, 199 insertions(+), 132 deletions(-) create mode 100644 buildbot/master/files/config/factories.py diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py new file mode 100644 index 00000000..cf1776f0 --- /dev/null +++ b/buildbot/master/files/config/factories.py @@ -0,0 +1,187 @@ +import re + +from buildbot.plugins import steps, util +from buildbot.process import buildstep + +import environments as envs + + +SERVO_REPO = "https://github.com/servo/servo" + + +class ServoFactory(util.BuildFactory): + """ + Build factory which checks out the servo repo as the first build step. + """ + + def __init__(self, build_steps): + """ + Takes a list of Buildbot steps. + Prefer using DynamicServoFactory to using this class directly. + """ + all_steps = [ + steps.Git(repourl=SERVO_REPO, mode="full", method="clobber"), + ] + build_steps + # util.BuildFactory is an old-style class so we cannot use super() + # but must hardcode the superclass here + util.BuildFactory.__init__(self, all_steps) + + +class BadConfigurationStep(buildstep.BuildStep): + """ + Step which immediately fails the build due to a bad configuration. + """ + + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, exception): + self.exception = exception + + def run(self): + raise Exception("Bad configuration, unable to convert to steps" + + str(self.exception)) + + +class DynamicServoFactory(ServoFactory): + """ + Smart factory which takes a list of shell commands and creates the + appropriate Buildbot Steps. Uses heuristics to infer Step type, if + there are any logfiles, etc. + """ + + def __init__(self, environment, commands): + self.environment = environment + try: + dynamic_steps = [self.make_step(command) for command in commands] + except Exception as e: # Bad step configuration, fail build + print(str(e)) + dynamic_steps = [BadConfigurationStep(e)] + + # util.BuildFactory is an old-style class so we cannot use super() + # but must hardcode the superclass here + ServoFactory.__init__(self, dynamic_steps) + + def make_step(self, command): + step_kwargs = {} + step_kwargs['env'] = self.environment + + command = command.split(' ') + step_kwargs['command'] = command + + step_class = steps.ShellCommand + args = iter(command) + for arg in args: + # Change Step class to capture warnings as needed + # (steps.Compile and steps.Test catch warnings) + if arg == './mach': + mach_arg = next(args) + if re.match('build(-.*)?', mach_arg): + step_class = steps.Compile + elif re.match('test-.*', mach_arg): + step_class = steps.Test + + # Capture any logfiles + elif re.match('--log-.*', arg): + logfile = next(args) + if 'logfiles' not in step_kwargs: + step_kwargs['logfiles'] = {} + step_kwargs['logfiles'][logfile] = logfile + + return step_class(**step_kwargs) + + +doc = ServoFactory([ + # This is not dynamic because a) we need to pass the logEnviron kwarg + # and b) changes to the documentation build are already encapsulated + # in the upload_docs.sh script; any further changes should go through + # the saltfs repo to avoid leaking the token. + steps.ShellCommand(command=["etc/ci/upload_docs.sh"], + env=envs.doc, + # important not to leak token + logEnviron=False), +]) + +windows = ServoFactory([ + # TODO: convert this to use DynamicServoFactory + # We need to run each command in a bash login shell, which breaks the + # heuristics used by DynamicServoFactory.make_step + steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"], + env=envs.build_windows), + steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], + env=envs.build_windows), +]) + +mac_rel_wpt = DynamicServoFactory(envs.build_mac, [ + "./mach test-tidy --no-progress", + "./mach build --release", + "./mach test-wpt-failure", + "./mach test-wpt --release --processes 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log", + "./mach build-cef --release", + "bash ./etc/ci/lockfile_changed.sh", + "bash ./etc/ci/manifest_changed.sh", +]) + +mac_dev_unit = DynamicServoFactory(envs.build_mac, [ + "./mach build --dev", + "./mach test-unit", + "./mach build-cef", + "./mach build-geckolib", + "bash ./etc/ci/lockfile_changed.sh", + "bash ./etc/ci/manifest_changed.sh", +]) + +mac_rel_css = DynamicServoFactory(envs.build_mac, [ + "./mach build --release", + "./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log", + "./mach build-geckolib --release", + "bash ./etc/ci/lockfile_changed.sh", + "bash ./etc/ci/manifest_changed.sh", +]) + +linux_dev = DynamicServoFactory(envs.build_linux_headless, [ + "./mach test-tidy --no-progress", + "./mach test-tidy --no-progress --self-test", + "./mach build --dev", + "./mach test-compiletest", + "./mach test-unit", + "./mach build-cef", + "./mach build-geckolib", + "bash ./etc/ci/lockfile_changed.sh", + "bash ./etc/ci/manifest_changed.sh", +]) + +linux_rel = DynamicServoFactory(envs.build_linux_headless, [ + "./mach build --release", + "./mach test-wpt-failure", + "./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log", + "./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log", + "./mach build-cef --release", + "./mach build-geckolib --release", + "bash ./etc/ci/lockfile_changed.sh", + "bash ./etc/ci/manifest_changed.sh", + "bash ./etc/ci/check_no_unwrap.sh", +]) + +android = DynamicServoFactory(envs.build_android, [ + "./mach build --android --dev", + "bash ./etc/ci/lockfile_changed.sh", + "bash ./etc/ci/manifest_changed.sh", + "python ./etc/ci/check_dynamic_symbols.py", +]) + +android_nightly = DynamicServoFactory(envs.build_android, [ + "./mach build --android --release", + "./mach package -r", + ("s3cmd put " + "./target/arm-linux-androideabi/release/servo.apk " + "s3://servo-rust/nightly/servo.apk"), +]) + +arm32 = DynamicServoFactory(envs.build_arm32, [ + "./mach build --rel --target=arm-unknown-linux-gnueabihf", +]) + +arm64 = DynamicServoFactory(envs.build_arm64, [ + "./mach build --rel --target=aarch64-unknown-linux-gnu", +]) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 5d95525b..38737a46 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -3,11 +3,10 @@ from buildbot.plugins import * -import environments +import factories from passwords import HTTP_USERNAME, HTTP_PASSWORD, SLAVE_PASSWORD, CHANGE_PASSWORD from passwords import GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET -SERVO_REPO = "https://github.com/servo/servo" LINUX_RESERVED_SLAVES = ["servo-linux1", "servo-linux2"] MAC_SLAVES = ["servo-mac1", "servo-mac2", "servo-mac3"] CROSS_SLAVES = ["servo-linux-cross1", "servo-linux-cross2"] @@ -71,125 +70,6 @@ c['schedulers'].append(schedulers.Nightly( ####### BUILDERS -def create_factory(commands): - factory = util.BuildFactory() - for command in commands: - factory.addStep(command) - return factory - - -def create_servo_factory(commands): - return create_factory([ - steps.Git(repourl=SERVO_REPO, mode="full", method="clobber"), - ] + commands) - - -linux_dev_factory = create_servo_factory([ - steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"], env=environments.build_linux_headless), - steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress", "--self-test"], env=environments.build_linux_headless), - steps.Compile(command=["./mach", "build", "--dev"], env=environments.build_linux_headless), - steps.ShellCommand(command=["./mach", "test-compiletest"], env=environments.build_linux_headless), - steps.ShellCommand(command=["./mach", "test-unit"], env=environments.build_linux_headless), - steps.Compile(command=["./mach", "build-cef"], env=environments.build_linux_headless), - steps.Compile(command=["./mach", "build-geckolib"], env=environments.build_linux_headless), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_linux_headless), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_linux_headless), -]) - -linux_rel_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--release"], env=environments.build_linux_headless), - steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=environments.build_linux_headless), - steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "24", - "--log-raw", "test-wpt.log", - "--log-errorsummary", "wpt_errorsummary.log"], - env=environments.build_linux_headless, - logfiles={"test-wpt.log": "test-wpt.log", - "wpt_errorsummary.log": "wpt_errorsummary.log"}), - steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "16", - "--log-raw", "test-css.log", - "--log-errorsummary", "css_errorsummary.log"], - env=environments.build_linux_headless, - logfiles={"test-css.log": "test-css.log", - "css_errorsummary.log": "css_errorsummary.log"}), - steps.Compile(command=["./mach", "build-cef", "--release"], env=environments.build_linux_headless), - steps.Compile(command=["./mach", "build-geckolib", "--release"], env=environments.build_linux_headless), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_linux_headless), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_linux_headless), - steps.ShellCommand(command=["bash", "./etc/ci/check_no_unwrap.sh"], env=environments.build_linux_headless), -]) - -mac_rel_wpt_factory = create_servo_factory([ - steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"]), - steps.Compile(command=["./mach", "build", "--release"], env=environments.build_mac), - steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=environments.build_mac), - steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", - "--log-raw", "test-wpt.log", - "--log-errorsummary", "wpt_errorsummary.log"], - env=environments.build_mac, - logfiles={"test-wpt.log": "test-wpt.log", - "wpt_errorsummary.log": "wpt_errorsummary.log"}), - steps.ShellCommand(command=["./mach", "build-cef", "--release"], env=environments.build_mac), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac), -]) - -mac_dev_unit_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--dev"], env=environments.build_mac), - steps.ShellCommand(command=["./mach", "test-unit"], env=environments.build_mac), - steps.Compile(command=["./mach", "build-cef"], env=environments.build_mac), - steps.Compile(command=["./mach", "build-geckolib"], env=environments.build_mac), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac), -]) - -mac_rel_css_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--release"], env=environments.build_mac), - steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "4", - "--log-raw", "test-css.log", - "--log-errorsummary", "css_errorsummary.log"], - env=environments.build_mac, - logfiles={"test-css.log": "test-css.log", - "css_errorsummary.log": "css_errorsummary.log"}), - steps.ShellCommand(command=["./mach", "build-geckolib", "--release"], env=environments.build_mac), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac), -]) - -android_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--android", "--dev"], env=environments.build_android), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_android), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_android), - steps.ShellCommand(command=["python", "./etc/ci/check_dynamic_symbols.py"], env=environments.build_android), -]) - -android_nightly_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--android", "--release"], env=environments.build_android), - steps.Compile(command=["./mach", "package", "-r"], env=environments.build_android), - steps.ShellCommand(command=["s3cmd", "put", - "{{ common.servo_home }}/buildbot/slave/android-nightly/build/target/arm-linux-androideabi/release/servo.apk", - "s3://servo-rust/nightly/servo.apk"]), -]) - -arm32_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--rel", "--target=arm-unknown-linux-gnueabihf"], env=environments.build_arm32), -]) - -arm64_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--rel", "--target=aarch64-unknown-linux-gnu"], env=environments.build_arm64), -]) - -windows_factory = create_servo_factory([ - steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"], env=environments.build_windows), - steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], env=environments.build_windows), -]) - -doc_factory = create_servo_factory([ - steps.ShellCommand(command=["etc/ci/upload_docs.sh"], - env=environments.doc, - # important not to leak token - logEnviron=False), -]) - def branch_priority(builder, requests): for r in requests: if r.source.branch != "try": @@ -200,76 +80,76 @@ c['builders'] = [] c['builders'].append(util.BuilderConfig( name="linux-dev", slavenames=LINUX_RESERVED_SLAVES, - factory=linux_dev_factory, + factory=factories.linux_dev, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="linux-rel", slavenames=LINUX_RESERVED_SLAVES, - factory=linux_rel_factory, + factory=factories.linux_rel, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="android", slavenames=CROSS_SLAVES, - factory=android_factory, + factory=factories.android, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="arm32", slavenames=CROSS_SLAVES, - factory=arm32_factory, + factory=factories.arm32, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="arm64", slavenames=CROSS_SLAVES, - factory=arm64_factory, + factory=factories.arm64, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="mac-rel-wpt", slavenames=MAC_SLAVES, - factory=mac_rel_wpt_factory, + factory=factories.mac_rel_wpt, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="mac-dev-unit", slavenames=MAC_SLAVES, - factory=mac_dev_unit_factory, + factory=factories.mac_dev_unit, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="mac-rel-css", slavenames=MAC_SLAVES, - factory=mac_rel_css_factory, + factory=factories.mac_rel_css, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="windows", slavenames=WINDOWS_SLAVES, - factory=windows_factory, + factory=factories.windows, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="doc", slavenames=LINUX_RESERVED_SLAVES, - factory=doc_factory, + factory=factories.doc, category="auto", )) c['builders'].append(util.BuilderConfig( name="android-nightly", slavenames=CROSS_SLAVES, - factory=android_nightly_factory, + factory=factories.android_nightly, nextBuild=branch_priority, category="auto", )) From d8955508eaf94aa235f682d08f78f6708b58d82e Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Mon, 18 Apr 2016 19:39:09 -0400 Subject: [PATCH 06/12] Re-add pkill step This automatically runs pkill for each build. It is cleaner to put this in the main configuration instead of the lists of steps because this is a CI-related cleanup task, not a necessary step to run the build + test suite. Also, this ensures that the pkill step cannot be forgotten when adding new builders. Note: this doesn't work on Windows yet. --- buildbot/master/files/config/factories.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py index cf1776f0..87b1bad4 100644 --- a/buildbot/master/files/config/factories.py +++ b/buildbot/master/files/config/factories.py @@ -2,6 +2,7 @@ from buildbot.plugins import steps, util from buildbot.process import buildstep +from buildbot.status.results import SUCCESS import environments as envs @@ -58,9 +59,13 @@ def __init__(self, environment, commands): print(str(e)) dynamic_steps = [BadConfigurationStep(e)] + # TODO: windows compatibility (use a custom script for this?) + pkill_step = [steps.ShellCommand(command=["pkill", "-x", "servo"], + decodeRC={0: SUCCESS, 1: SUCCESS})] + # util.BuildFactory is an old-style class so we cannot use super() # but must hardcode the superclass here - ServoFactory.__init__(self, dynamic_steps) + ServoFactory.__init__(self, pkill_step + dynamic_steps) def make_step(self, command): step_kwargs = {} From 90937e6be420bdc1e5d8d5a65eabb241d5868b38 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 16 Apr 2016 20:31:39 -0400 Subject: [PATCH 07/12] Remove GITHUB_STATUS_TOKEN This was used a long time ago for bors, but Buildbot now reports status indirectly via Homu. --- .travis/test_pillars/buildbot/master.sls | 1 - buildbot/master/files/config/master.cfg | 2 +- buildbot/master/files/config/passwords.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis/test_pillars/buildbot/master.sls b/.travis/test_pillars/buildbot/master.sls index 4a5f31b4..643ffc9d 100644 --- a/.travis/test_pillars/buildbot/master.sls +++ b/.travis/test_pillars/buildbot/master.sls @@ -4,6 +4,5 @@ buildbot: 'http-pass': 'TEST_BUILDBOT_HTTP_PASS' 'change-pass': 'TEST_BUILDBOT_CHANGE_PASS' 'gh-doc-token': 'TEST_BUILDBOT_GH_DOC_TOKEN' - 'gh-status-token': 'TEST_BUILDBOT_GH_STATUS_TOKEN' 'gh-hook-secret': 'TEST_BUILDBOT_GH_HOOK_SECRET' 'homu-secret': 'TEST_BUILDBOT_HOMU_SECRET' diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 38737a46..71d9efe0 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -5,7 +5,7 @@ from buildbot.plugins import * import factories from passwords import HTTP_USERNAME, HTTP_PASSWORD, SLAVE_PASSWORD, CHANGE_PASSWORD -from passwords import GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET +from passwords import HOMU_BUILDBOT_SECRET LINUX_RESERVED_SLAVES = ["servo-linux1", "servo-linux2"] MAC_SLAVES = ["servo-mac1", "servo-mac2", "servo-mac3"] diff --git a/buildbot/master/files/config/passwords.py b/buildbot/master/files/config/passwords.py index d957ee54..544f2d40 100644 --- a/buildbot/master/files/config/passwords.py +++ b/buildbot/master/files/config/passwords.py @@ -3,5 +3,4 @@ SLAVE_PASSWORD = "{{ pillar['buildbot']['credentials']['slave-pass'] }}" CHANGE_PASSWORD = "{{ pillar['buildbot']['credentials']['change-pass'] }}" GITHUB_DOC_TOKEN = "{{pillar['buildbot']['credentials']['gh-doc-token'] }}" -GITHUB_STATUS_TOKEN = "{{pillar['buildbot']['credentials']['gh-status-token'] }}" HOMU_BUILDBOT_SECRET = "{{pillar['buildbot']['credentials']['homu-secret'] }}" From 056ed34ec10c9275054cc0364cca7ce3063dfe9c Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 16 Apr 2016 20:56:34 -0400 Subject: [PATCH 08/12] Clean up master.cfg to pass flake8 Also rename LINUX_RESERVED_SLAVES to LINUX_SLAVES, since all of our builders are now on reserved instances. --- buildbot/master/files/config/master.cfg | 156 +++++++++++++++--------- 1 file changed, 100 insertions(+), 56 deletions(-) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 71d9efe0..6d3e7443 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -1,17 +1,18 @@ -# -*- python -*- -# ex: set syntax=python: - -from buildbot.plugins import * +from buildbot.plugins import buildslave, changes, schedulers, util +from buildbot.status import html, status_push, web, words import factories -from passwords import HTTP_USERNAME, HTTP_PASSWORD, SLAVE_PASSWORD, CHANGE_PASSWORD +from passwords import HTTP_USERNAME, HTTP_PASSWORD +from passwords import SLAVE_PASSWORD, CHANGE_PASSWORD from passwords import HOMU_BUILDBOT_SECRET -LINUX_RESERVED_SLAVES = ["servo-linux1", "servo-linux2"] + +LINUX_SLAVES = ["servo-linux1", "servo-linux2"] MAC_SLAVES = ["servo-mac1", "servo-mac2", "servo-mac3"] CROSS_SLAVES = ["servo-linux-cross1", "servo-linux-cross2"] WINDOWS_SLAVES = ["servo-windows1"] + c = BuildmasterConfig = {} c['protocols'] = {'pb': {'port': 9001}} c['caches'] = { @@ -20,34 +21,57 @@ c['caches'] = { 'chdicts': 1000, } -####### BUILDSLAVES + +################## +# BUILDSLAVES +################## + c['slaves'] = [] -for s in MAC_SLAVES + CROSS_SLAVES + LINUX_RESERVED_SLAVES + WINDOWS_SLAVES: +for s in MAC_SLAVES + CROSS_SLAVES + LINUX_SLAVES + WINDOWS_SLAVES: c['slaves'].append(buildslave.BuildSlave(s, SLAVE_PASSWORD, max_builds=1)) -####### CHANGESOURCES -c['change_source'] = [] -c['change_source'].append(changes.PBChangeSource(passwd=CHANGE_PASSWORD)) +################## +# CHANGESOURCES +################## + + +c['change_source'] = [changes.PBChangeSource(passwd=CHANGE_PASSWORD)] + + +################## +# SCHEDULERS +################## -####### SCHEDULERS def servo_auto_try_filter(c): - if c.project == 'servo/servo' and c.who.startswith('bors-servo') and c.branch in ["auto", "try"]: - return True - return False + return (c.project == 'servo/servo' and + c.who.startswith('bors-servo') and + c.branch in ["auto", "try"]) + def servo_master_filter(c): - if c.project == 'servo/servo' and c.who.startswith('bors-servo') and c.branch == "master": - return True - return False + return (c.project == 'servo/servo' and + c.who.startswith('bors-servo') and + c.branch == "master") + c['schedulers'] = [] c['schedulers'].append(schedulers.AnyBranchScheduler( name="servo-auto", treeStableTimer=None, - builderNames=["linux-dev", "linux-rel", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", "android", "arm32", "arm64", "windows"], + builderNames=[ + "linux-dev", + "linux-rel", + "mac-rel-wpt", + "mac-dev-unit", + "mac-rel-css", + "android", + "arm32", + "arm64", + "windows", + ], change_filter=util.ChangeFilter(filter_fn=servo_auto_try_filter), )) c['schedulers'].append(schedulers.SingleBranchScheduler( @@ -58,17 +82,32 @@ c['schedulers'].append(schedulers.SingleBranchScheduler( )) c['schedulers'].append(schedulers.ForceScheduler( name="force", - builderNames=["linux-dev", "linux-rel", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", "android", "arm32", "arm64", "android-nightly", "windows"] + builderNames=[ + "linux-dev", + "linux-rel", + "mac-rel-wpt", + "mac-dev-unit", + "mac-rel-css", + "android", + "arm32", + "arm64", + "android-nightly", + "windows", + ], )) c['schedulers'].append(schedulers.Nightly( name="Nightly", branch="master", builderNames=["android-nightly"], hour=1, - minute=0 + minute=0, )) -####### BUILDERS + +################## +# BUILDERS +################## + def branch_priority(builder, requests): for r in requests: @@ -76,17 +115,18 @@ def branch_priority(builder, requests): return r return requests[0] + c['builders'] = [] c['builders'].append(util.BuilderConfig( name="linux-dev", - slavenames=LINUX_RESERVED_SLAVES, + slavenames=LINUX_SLAVES, factory=factories.linux_dev, nextBuild=branch_priority, category="auto", )) c['builders'].append(util.BuilderConfig( name="linux-rel", - slavenames=LINUX_RESERVED_SLAVES, + slavenames=LINUX_SLAVES, factory=factories.linux_rel, nextBuild=branch_priority, category="auto", @@ -142,7 +182,7 @@ c['builders'].append(util.BuilderConfig( )) c['builders'].append(util.BuilderConfig( name="doc", - slavenames=LINUX_RESERVED_SLAVES, + slavenames=LINUX_SLAVES, factory=factories.doc, category="auto", )) @@ -154,53 +194,57 @@ c['builders'].append(util.BuilderConfig( category="auto", )) -####### STATUS TARGETS -from buildbot.status.status_push import HttpStatusPush +################## +# STATUS TARGETS +################## + c['status'] = [] -c['status'].append(HttpStatusPush( +c['status'].append(status_push.HttpStatusPush( serverUrl='http://build.servo.org:54856/buildbot', extra_post_params={'secret': HOMU_BUILDBOT_SECRET}, )) -from buildbot.status import html -from buildbot.status.web import authz, auth - - - -authz_cfg=authz.Authz( - auth=auth.BasicAuth([(HTTP_USERNAME, HTTP_PASSWORD)]), - gracefulShutdown = 'auth', - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = 'auth', - stopBuild = 'auth', - stopAllBuilds = 'auth', - cancelPendingBuild = 'auth', +authz_cfg = web.authz.Authz( + auth=web.auth.BasicAuth([(HTTP_USERNAME, HTTP_PASSWORD)]), + gracefulShutdown='auth', + forceBuild='auth', + forceAllBuilds='auth', + pingBuilder='auth', + stopBuild='auth', + stopAllBuilds='auth', + cancelPendingBuild='auth', ) c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) -from buildbot.status import words c['status'].append(words.IRC(host="irc.mozilla.org", - port=6697, - useSSL=True, - nick="servo_buildbot", - channels=["#servo-bots"], - notify_events={ - 'exception':1, - 'finished':1, - 'success':1, - 'failure':1 - })) - -####### PROJECT IDENTITY + port=6697, + useSSL=True, + nick="servo_buildbot", + channels=["#servo-bots"], + notify_events={ + 'exception': 1, + 'finished': 1, + 'success': 1, + 'failure': 1, + })) + + +################## +# PROJECT IDENTITY +################## + c['title'] = "Servo" c['titleURL'] = "http://github.com/servo/servo" c['buildbotURL'] = "http://build.servo.org/" -####### DB URL + +################## +# DATABASE URL +################## + c['db'] = { 'db_url': "sqlite:///state.sqlite", From cf3adb1f07d18db405d81830e9d6692d2cdf3a04 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Mon, 18 Apr 2016 03:34:13 -0400 Subject: [PATCH 09/12] Use a YAML file to drive dynamic builders As a stepping-stone to having builder steps live in the main Servo repo, switch to a simple data format to express builder configuration (lists of strings accessed via yaml.safe_load) instead of needing to run or import arbitrary untrusted Python code. Currently, the YAML file is still part of the master configuration, but it is now ready to be moved to the main Servo repo. --- buildbot/master/files/config/factories.py | 89 ++-------------- buildbot/master/files/config/master.cfg | 120 ++++++++-------------- buildbot/master/files/config/steps.yml | 62 +++++++++++ buildbot/master/init.sls | 1 + 4 files changed, 117 insertions(+), 155 deletions(-) create mode 100644 buildbot/master/files/config/steps.yml diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py index 87b1bad4..167d5b8c 100644 --- a/buildbot/master/files/config/factories.py +++ b/buildbot/master/files/config/factories.py @@ -1,8 +1,10 @@ +import os.path import re from buildbot.plugins import steps, util from buildbot.process import buildstep from buildbot.status.results import SUCCESS +import yaml import environments as envs @@ -46,14 +48,19 @@ def run(self): class DynamicServoFactory(ServoFactory): """ - Smart factory which takes a list of shell commands and creates the - appropriate Buildbot Steps. Uses heuristics to infer Step type, if - there are any logfiles, etc. + Smart factory which takes a list of shell commands from a yaml file + and creates the appropriate Buildbot Steps. Uses heuristics to infer + Step type, if there are any logfiles, etc. """ - def __init__(self, environment, commands): + def __init__(self, builder_name, environment): self.environment = environment try: + config_dir = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(config_dir, 'steps.yml') + with open(yaml_path) as steps_file: + builder_steps = yaml.safe_load(steps_file) + commands = builder_steps[builder_name] dynamic_steps = [self.make_step(command) for command in commands] except Exception as e: # Bad step configuration, fail build print(str(e)) @@ -116,77 +123,3 @@ def make_step(self, command): steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], env=envs.build_windows), ]) - -mac_rel_wpt = DynamicServoFactory(envs.build_mac, [ - "./mach test-tidy --no-progress", - "./mach build --release", - "./mach test-wpt-failure", - "./mach test-wpt --release --processes 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log", - "./mach build-cef --release", - "bash ./etc/ci/lockfile_changed.sh", - "bash ./etc/ci/manifest_changed.sh", -]) - -mac_dev_unit = DynamicServoFactory(envs.build_mac, [ - "./mach build --dev", - "./mach test-unit", - "./mach build-cef", - "./mach build-geckolib", - "bash ./etc/ci/lockfile_changed.sh", - "bash ./etc/ci/manifest_changed.sh", -]) - -mac_rel_css = DynamicServoFactory(envs.build_mac, [ - "./mach build --release", - "./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log", - "./mach build-geckolib --release", - "bash ./etc/ci/lockfile_changed.sh", - "bash ./etc/ci/manifest_changed.sh", -]) - -linux_dev = DynamicServoFactory(envs.build_linux_headless, [ - "./mach test-tidy --no-progress", - "./mach test-tidy --no-progress --self-test", - "./mach build --dev", - "./mach test-compiletest", - "./mach test-unit", - "./mach build-cef", - "./mach build-geckolib", - "bash ./etc/ci/lockfile_changed.sh", - "bash ./etc/ci/manifest_changed.sh", -]) - -linux_rel = DynamicServoFactory(envs.build_linux_headless, [ - "./mach build --release", - "./mach test-wpt-failure", - "./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log", - "./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log", - "./mach build-cef --release", - "./mach build-geckolib --release", - "bash ./etc/ci/lockfile_changed.sh", - "bash ./etc/ci/manifest_changed.sh", - "bash ./etc/ci/check_no_unwrap.sh", -]) - -android = DynamicServoFactory(envs.build_android, [ - "./mach build --android --dev", - "bash ./etc/ci/lockfile_changed.sh", - "bash ./etc/ci/manifest_changed.sh", - "python ./etc/ci/check_dynamic_symbols.py", -]) - -android_nightly = DynamicServoFactory(envs.build_android, [ - "./mach build --android --release", - "./mach package -r", - ("s3cmd put " - "./target/arm-linux-androideabi/release/servo.apk " - "s3://servo-rust/nightly/servo.apk"), -]) - -arm32 = DynamicServoFactory(envs.build_arm32, [ - "./mach build --rel --target=arm-unknown-linux-gnueabihf", -]) - -arm64 = DynamicServoFactory(envs.build_arm64, [ - "./mach build --rel --target=aarch64-unknown-linux-gnu", -]) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 6d3e7443..2df21fce 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -1,6 +1,7 @@ from buildbot.plugins import buildslave, changes, schedulers, util from buildbot.status import html, status_push, web, words +import environments as envs import factories from passwords import HTTP_USERNAME, HTTP_PASSWORD from passwords import SLAVE_PASSWORD, CHANGE_PASSWORD @@ -116,83 +117,48 @@ def branch_priority(builder, requests): return requests[0] -c['builders'] = [] -c['builders'].append(util.BuilderConfig( - name="linux-dev", - slavenames=LINUX_SLAVES, - factory=factories.linux_dev, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="linux-rel", - slavenames=LINUX_SLAVES, - factory=factories.linux_rel, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="android", - slavenames=CROSS_SLAVES, - factory=factories.android, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="arm32", - slavenames=CROSS_SLAVES, - factory=factories.arm32, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="arm64", - slavenames=CROSS_SLAVES, - factory=factories.arm64, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="mac-rel-wpt", - slavenames=MAC_SLAVES, - factory=factories.mac_rel_wpt, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="mac-dev-unit", - slavenames=MAC_SLAVES, - factory=factories.mac_dev_unit, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="mac-rel-css", - slavenames=MAC_SLAVES, - factory=factories.mac_rel_css, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="windows", - slavenames=WINDOWS_SLAVES, - factory=factories.windows, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="doc", - slavenames=LINUX_SLAVES, - factory=factories.doc, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="android-nightly", - slavenames=CROSS_SLAVES, - factory=factories.android_nightly, - nextBuild=branch_priority, - category="auto", -)) +class DynamicServoBuilder(util.BuilderConfig): + """ + Builder which uses DynamicServoFactory to run steps from a yaml file. + """ + def __init__(self, name, slavenames, environment): + # util.BuilderConfig is an old-style class so we cannot use super() + # but must hardcode the superclass here + util.BuilderConfig.__init__( + self, + name=name, + slavenames=slavenames, + factory=factories.DynamicServoFactory(name, environment), + nextBuild=branch_priority, + category="auto", + ) + + +c['builders'] = [ + DynamicServoBuilder("linux-dev", LINUX_SLAVES, envs.build_linux_headless), + DynamicServoBuilder("linux-rel", LINUX_SLAVES, envs.build_linux_headless), + DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android), + DynamicServoBuilder("arm32", CROSS_SLAVES, envs.build_arm32), + DynamicServoBuilder("arm64", CROSS_SLAVES, envs.build_arm64), + DynamicServoBuilder("mac-dev-unit", MAC_SLAVES, envs.build_mac), + DynamicServoBuilder("mac-rel-wpt", MAC_SLAVES, envs.build_mac), + DynamicServoBuilder("mac-rel-css", MAC_SLAVES, envs.build_mac), + DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android), + # The below builders are not dynamic but rather have hard-coded factories + util.BuilderConfig( + name="windows", + slavenames=WINDOWS_SLAVES, + factory=factories.windows, + nextBuild=branch_priority, + category="auto", + ), + util.BuilderConfig( + name="doc", + slavenames=LINUX_SLAVES, + factory=factories.doc, + category="auto", + ), +] ################## diff --git a/buildbot/master/files/config/steps.yml b/buildbot/master/files/config/steps.yml new file mode 100644 index 00000000..e22aca72 --- /dev/null +++ b/buildbot/master/files/config/steps.yml @@ -0,0 +1,62 @@ +mac-rel-wpt: + - ./mach test-tidy --no-progress + - ./mach build --release + - ./mach test-wpt-failure + - ./mach test-wpt --release --processes 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log + - ./mach build-cef --release + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +mac-dev-unit: + - ./mach build --dev + - ./mach test-unit + - ./mach build-cef + - ./mach build-geckolib + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +mac-rel-css: + - ./mach build --release + - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log + - ./mach build-geckolib --release + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +linux-dev: + - ./mach test-tidy --no-progress + - ./mach test-tidy --no-progress --self-test + - ./mach build --dev + - ./mach test-compiletest + - ./mach test-unit + - ./mach build-cef + - ./mach build-geckolib + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +linux-rel: + - ./mach build --release + - ./mach test-wpt-failure + - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log + - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log + - ./mach build-cef --release + - ./mach build-geckolib --release + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + - bash ./etc/ci/check_no_unwrap.sh + +android: + - ./mach build --android --dev + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + - python ./etc/ci/check_dynamic_symbols.py + +android-nightly: + - ./mach build --android --release + - ./mach package -r + - s3cmd put target/arm-linux-androideabi/release/servo.apk s3://servo-rust/nightly/servo.apk + +arm32: + - ./mach build --rel --target=arm-unknown-linux-gnueabihf + +arm64: + - ./mach build --rel --target=aarch64-unknown-linux-gnu diff --git a/buildbot/master/init.sls b/buildbot/master/init.sls index fe8010f4..c7905c0d 100644 --- a/buildbot/master/init.sls +++ b/buildbot/master/init.sls @@ -7,6 +7,7 @@ buildbot-master: - service_identity == 14.0.0 - txgithub == 15.0.0 - boto == 2.38.0 + - pyyaml == 3.11 - require: - pkg: pip service.running: From 9d71e2d6ce6120032d603c31eaa0910af1fa9de6 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Mon, 18 Apr 2016 11:21:55 -0400 Subject: [PATCH 10/12] Tweak testing steps Use linux-dev to run all lints because it is the fastest builder; these are platform-independent. Ensure that the lockfile_changed.sh and manifest_changed.sh scripts are run for every builder, since these are platform-dependent. Note: these should also be added for the Windows builder. This is also meant as an example of how much easier a YAML file makes it to edit our build scripts; note the lack of needing to know about `logfiles` or other parameters which have been forgotten in the past. --- buildbot/master/files/config/factories.py | 1 + buildbot/master/files/config/steps.yml | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py index 167d5b8c..041380d8 100644 --- a/buildbot/master/files/config/factories.py +++ b/buildbot/master/files/config/factories.py @@ -122,4 +122,5 @@ def make_step(self, command): env=envs.build_windows), steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], env=envs.build_windows), + # TODO: run lockfile_changed.sh and manifest_changed.sh scripts ]) diff --git a/buildbot/master/files/config/steps.yml b/buildbot/master/files/config/steps.yml index e22aca72..006623e4 100644 --- a/buildbot/master/files/config/steps.yml +++ b/buildbot/master/files/config/steps.yml @@ -1,5 +1,4 @@ mac-rel-wpt: - - ./mach test-tidy --no-progress - ./mach build --release - ./mach test-wpt-failure - ./mach test-wpt --release --processes 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log @@ -32,6 +31,7 @@ linux-dev: - ./mach build-geckolib - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh + - bash ./etc/ci/check_no_unwrap.sh linux-rel: - ./mach build --release @@ -42,7 +42,6 @@ linux-rel: - ./mach build-geckolib --release - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh - - bash ./etc/ci/check_no_unwrap.sh android: - ./mach build --android --dev @@ -57,6 +56,10 @@ android-nightly: arm32: - ./mach build --rel --target=arm-unknown-linux-gnueabihf + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh arm64: - ./mach build --rel --target=aarch64-unknown-linux-gnu + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh From bab18c34cabc88a9d270a1e1eba04258dba13084 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Mon, 18 Apr 2016 04:09:18 -0400 Subject: [PATCH 11/12] Add test for buildbot configuration passing flake8 This test requires the flake8 python module; to ensure it is installed, add a requirements.txt file. Travis-CI will automatically install the dependencies specified in that file. --- requirements.txt | 3 +++ tests/sls/buildbot/master/config_lint.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 requirements.txt create mode 100644 tests/sls/buildbot/master/config_lint.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..275d0fd3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# Python modules needed for testing +# (Travis-CI will auto-install them from this file) +flake8 == 2.5.4 diff --git a/tests/sls/buildbot/master/config_lint.py b/tests/sls/buildbot/master/config_lint.py new file mode 100644 index 00000000..af044a2c --- /dev/null +++ b/tests/sls/buildbot/master/config_lint.py @@ -0,0 +1,20 @@ +import os +import subprocess + +from tests.util import Failure, Success, project_path + + +def run(): + CONF_DIR = os.path.join(project_path(), + 'buildbot', 'master', 'files', 'config') + # Have to specify master.cfg separately because it is not a .py file + command = ['flake8', CONF_DIR, os.path.join(CONF_DIR, 'master.cfg')] + ret = subprocess.run(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + + if ret.returncode == 0: + return Success("Buildbot master config passed linting") + else: + return Failure("Buildbot master config lint check failed:", ret.stdout) From acf76a1b9625895654145654ea8b5a02ac9c6903 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Mon, 18 Apr 2016 21:00:55 -0400 Subject: [PATCH 12/12] Add flake8 lint test for other python files Currently, the only other python files are the test suite. --- tests/lint/flake8.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/lint/flake8.py diff --git a/tests/lint/flake8.py b/tests/lint/flake8.py new file mode 100644 index 00000000..5eb523df --- /dev/null +++ b/tests/lint/flake8.py @@ -0,0 +1,19 @@ +import os +import subprocess + +from tests.util import Failure, Success, project_path + + +def run(): + paths = ['test.py', 'tests'] + paths = [os.path.join(project_path(), path) for path in paths] + command = ['flake8'] + paths + ret = subprocess.run(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + + if ret.returncode == 0: + return Success("Tests passed flake8 lint") + else: + return Failure("Tests failed flake8 lint:", ret.stdout)