diff --git a/.travis/test_pillars/buildbot/master.sls b/.travis/test_pillars/buildbot/master.sls index 4a5f31b4..643ffc9d 100644 --- a/.travis/test_pillars/buildbot/master.sls +++ b/.travis/test_pillars/buildbot/master.sls @@ -4,6 +4,5 @@ buildbot: 'http-pass': 'TEST_BUILDBOT_HTTP_PASS' 'change-pass': 'TEST_BUILDBOT_CHANGE_PASS' 'gh-doc-token': 'TEST_BUILDBOT_GH_DOC_TOKEN' - 'gh-status-token': 'TEST_BUILDBOT_GH_STATUS_TOKEN' 'gh-hook-secret': 'TEST_BUILDBOT_GH_HOOK_SECRET' 'homu-secret': 'TEST_BUILDBOT_HOMU_SECRET' diff --git a/buildbot/master/files/config/environments.py b/buildbot/master/files/config/environments.py new file mode 100644 index 00000000..965405e1 --- /dev/null +++ b/buildbot/master/files/config/environments.py @@ -0,0 +1,119 @@ +from passwords import GITHUB_DOC_TOKEN + + +class Environment(dict): + """ + Wrapper that allows 'adding' environment dictionaries + to make it easy to build up environments piece by piece. + """ + + def __init__(self, *args, **kwargs): + super(Environment, self).__init__(*args, **kwargs) + + def __add__(self, other): + assert type(self) == type(other) + combined = self.copy() + combined.update(other) # other takes precedence over self + return Environment(combined) + + +doc = Environment({ + 'CARGO_HOME': '{{ common.servo_home }}/.cargo', + 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', + 'SHELL': '/bin/bash', + 'TOKEN': GITHUB_DOC_TOKEN, +}) + +build_common = Environment({ + 'RUST_BACKTRACE': '1', +}) + +build_windows = build_common + Environment({ + 'MSYS': 'winsymlinks=lnk', + 'MSYSTEM': 'MINGW64', + 'PATH': ';'.join([ + r'C:\msys64\mingw64\bin', + r'C:\msys64\usr\bin', + r'C:\Windows\system32', + r'C:\Windows', + r'C:\Windows\System32\Wbem', + r'C:\Windows\System32\WindowsPowerShell\v1.0', + r'C:\Program Files\Amazon\cfn-bootstrap', + ]), +}) + +build_mac = build_common + Environment({ + 'CARGO_HOME': '/Users/servo/.cargo', + 'CCACHE': '/usr/local/bin/ccache', + 'SERVO_CACHE_DIR': '/Users/servo/.servo', +}) + + +build_linux = build_common + Environment({ + 'CARGO_HOME': '{{ common.servo_home }}/.cargo', + 'CCACHE': '/usr/bin/ccache', + 'DISPLAY': ':0', + 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', + 'SHELL': '/bin/bash', +}) + +build_linux_headless = build_linux + Environment({ + 'SERVO_HEADLESS': '1', +}) + +build_android = build_linux + Environment({ + 'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/', + 'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/', + 'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/', + 'PATH': ':'.join([ + '/usr/local/sbin', + '/usr/local/bin', + '/usr/bin', + '/usr/sbin', + '/sbin', + '/bin', + '{{ common.servo_home }}/android/sdk/current/platform-tools', + '{{ common.servo_home }}/android/toolchain/current/bin', + ]), +}) + +build_arm = build_linux + Environment({ + 'EXPAT_NO_PKG_CONFIG': '1', + 'FONTCONFIG_NO_PKG_CONFIG': '1', + 'FREETYPE2_NO_PKG_CONFIG': '1', + 'PKG_CONFIG_ALLOW_CROSS': '1', +}) + + +build_arm32 = build_arm + Environment({ + 'BUILD_TARGET': 'arm-unknown-linux-gnueabihf', + 'CC': 'arm-linux-gnueabihf-gcc', + 'CXX': 'arm-linux-gnueabihf-g++', + 'PATH': ':'.join([ + '{{ common.servo_home }}/bin', + '/usr/local/sbin', + '/usr/local/bin', + '/usr/bin', + '/usr/sbin', + '/sbin', + '/bin', + ]), + 'PKG_CONFIG_PATH': '/usr/lib/arm-linux-gnueabihf/pkgconfig', +}) + +build_arm64 = build_arm + Environment({ + 'BUILD_TARGET': 'aarch64-unknown-linux-gnu', + 'CC': 'aarch64-linux-gnu-gcc', + 'CXX': 'aarch64-linux-gnu-g++', + 'PATH': ':'.join([ + '{{ common.servo_home }}/bin', + '/usr/local/sbin', + '/usr/local/bin', + '/usr/bin', + '/usr/sbin', + '/sbin', + '/bin', + ]), + 'PKG_CONFIG_PATH': '/usr/lib/aarch64-linux-gnu/pkgconfig', + 'SERVO_RUSTC_WITH_GOLD': 'False', +}) diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py new file mode 100644 index 00000000..041380d8 --- /dev/null +++ b/buildbot/master/files/config/factories.py @@ -0,0 +1,126 @@ +import os.path +import re + +from buildbot.plugins import steps, util +from buildbot.process import buildstep +from buildbot.status.results import SUCCESS +import yaml + +import environments as envs + + +SERVO_REPO = "https://github.com/servo/servo" + + +class ServoFactory(util.BuildFactory): + """ + Build factory which checks out the servo repo as the first build step. + """ + + def __init__(self, build_steps): + """ + Takes a list of Buildbot steps. + Prefer using DynamicServoFactory to using this class directly. + """ + all_steps = [ + steps.Git(repourl=SERVO_REPO, mode="full", method="clobber"), + ] + build_steps + # util.BuildFactory is an old-style class so we cannot use super() + # but must hardcode the superclass here + util.BuildFactory.__init__(self, all_steps) + + +class BadConfigurationStep(buildstep.BuildStep): + """ + Step which immediately fails the build due to a bad configuration. + """ + + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, exception): + self.exception = exception + + def run(self): + raise Exception("Bad configuration, unable to convert to steps" + + str(self.exception)) + + +class DynamicServoFactory(ServoFactory): + """ + Smart factory which takes a list of shell commands from a yaml file + and creates the appropriate Buildbot Steps. Uses heuristics to infer + Step type, if there are any logfiles, etc. + """ + + def __init__(self, builder_name, environment): + self.environment = environment + try: + config_dir = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(config_dir, 'steps.yml') + with open(yaml_path) as steps_file: + builder_steps = yaml.safe_load(steps_file) + commands = builder_steps[builder_name] + dynamic_steps = [self.make_step(command) for command in commands] + except Exception as e: # Bad step configuration, fail build + print(str(e)) + dynamic_steps = [BadConfigurationStep(e)] + + # TODO: windows compatibility (use a custom script for this?) + pkill_step = [steps.ShellCommand(command=["pkill", "-x", "servo"], + decodeRC={0: SUCCESS, 1: SUCCESS})] + + # util.BuildFactory is an old-style class so we cannot use super() + # but must hardcode the superclass here + ServoFactory.__init__(self, pkill_step + dynamic_steps) + + def make_step(self, command): + step_kwargs = {} + step_kwargs['env'] = self.environment + + command = command.split(' ') + step_kwargs['command'] = command + + step_class = steps.ShellCommand + args = iter(command) + for arg in args: + # Change Step class to capture warnings as needed + # (steps.Compile and steps.Test catch warnings) + if arg == './mach': + mach_arg = next(args) + if re.match('build(-.*)?', mach_arg): + step_class = steps.Compile + elif re.match('test-.*', mach_arg): + step_class = steps.Test + + # Capture any logfiles + elif re.match('--log-.*', arg): + logfile = next(args) + if 'logfiles' not in step_kwargs: + step_kwargs['logfiles'] = {} + step_kwargs['logfiles'][logfile] = logfile + + return step_class(**step_kwargs) + + +doc = ServoFactory([ + # This is not dynamic because a) we need to pass the logEnviron kwarg + # and b) changes to the documentation build are already encapsulated + # in the upload_docs.sh script; any further changes should go through + # the saltfs repo to avoid leaking the token. + steps.ShellCommand(command=["etc/ci/upload_docs.sh"], + env=envs.doc, + # important not to leak token + logEnviron=False), +]) + +windows = ServoFactory([ + # TODO: convert this to use DynamicServoFactory + # We need to run each command in a bash login shell, which breaks the + # heuristics used by DynamicServoFactory.make_step + steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"], + env=envs.build_windows), + steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], + env=envs.build_windows), + # TODO: run lockfile_changed.sh and manifest_changed.sh scripts +]) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index f525d63f..2df21fce 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -1,18 +1,19 @@ -# -*- python -*- -# ex: set syntax=python: +from buildbot.plugins import buildslave, changes, schedulers, util +from buildbot.status import html, status_push, web, words -from buildbot.plugins import * -from buildbot.status.results import SUCCESS +import environments as envs +import factories +from passwords import HTTP_USERNAME, HTTP_PASSWORD +from passwords import SLAVE_PASSWORD, CHANGE_PASSWORD +from passwords import HOMU_BUILDBOT_SECRET -from passwords import HTTP_USERNAME, HTTP_PASSWORD, SLAVE_PASSWORD, CHANGE_PASSWORD -from passwords import GITHUB_DOC_TOKEN, GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET -SERVO_REPO = "https://github.com/servo/servo" -LINUX_RESERVED_SLAVES = ["servo-linux1", "servo-linux2"] +LINUX_SLAVES = ["servo-linux1", "servo-linux2"] MAC_SLAVES = ["servo-mac1", "servo-mac2", "servo-mac3"] CROSS_SLAVES = ["servo-linux-cross1", "servo-linux-cross2"] WINDOWS_SLAVES = ["servo-windows1"] + c = BuildmasterConfig = {} c['protocols'] = {'pb': {'port': 9001}} c['caches'] = { @@ -21,34 +22,57 @@ c['caches'] = { 'chdicts': 1000, } -####### BUILDSLAVES + +################## +# BUILDSLAVES +################## + c['slaves'] = [] -for s in MAC_SLAVES + CROSS_SLAVES + LINUX_RESERVED_SLAVES + WINDOWS_SLAVES: +for s in MAC_SLAVES + CROSS_SLAVES + LINUX_SLAVES + WINDOWS_SLAVES: c['slaves'].append(buildslave.BuildSlave(s, SLAVE_PASSWORD, max_builds=1)) -####### CHANGESOURCES -c['change_source'] = [] -c['change_source'].append(changes.PBChangeSource(passwd=CHANGE_PASSWORD)) +################## +# CHANGESOURCES +################## + + +c['change_source'] = [changes.PBChangeSource(passwd=CHANGE_PASSWORD)] + + +################## +# SCHEDULERS +################## -####### SCHEDULERS def servo_auto_try_filter(c): - if c.project == 'servo/servo' and c.who.startswith('bors-servo') and c.branch in ["auto", "try"]: - return True - return False + return (c.project == 'servo/servo' and + c.who.startswith('bors-servo') and + c.branch in ["auto", "try"]) + def servo_master_filter(c): - if c.project == 'servo/servo' and c.who.startswith('bors-servo') and c.branch == "master": - return True - return False + return (c.project == 'servo/servo' and + c.who.startswith('bors-servo') and + c.branch == "master") + c['schedulers'] = [] c['schedulers'].append(schedulers.AnyBranchScheduler( name="servo-auto", treeStableTimer=None, - builderNames=["linux-dev", "linux-rel", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", "android", "gonk", "arm32", "arm64", "windows"], + builderNames=[ + "linux-dev", + "linux-rel", + "mac-rel-wpt", + "mac-dev-unit", + "mac-rel-css", + "android", + "arm32", + "arm64", + "windows", + ], change_filter=util.ChangeFilter(filter_fn=servo_auto_try_filter), )) c['schedulers'].append(schedulers.SingleBranchScheduler( @@ -59,349 +83,134 @@ c['schedulers'].append(schedulers.SingleBranchScheduler( )) c['schedulers'].append(schedulers.ForceScheduler( name="force", - builderNames=["linux-dev", "linux-rel", "mac-rel-wpt", "mac-dev-unit", "mac-rel-css", "android", "gonk", "arm32", "arm64", "android-nightly", "windows"] + builderNames=[ + "linux-dev", + "linux-rel", + "mac-rel-wpt", + "mac-dev-unit", + "mac-rel-css", + "android", + "arm32", + "arm64", + "android-nightly", + "windows", + ], )) c['schedulers'].append(schedulers.Nightly( name="Nightly", branch="master", builderNames=["android-nightly"], hour=1, - minute=0 + minute=0, )) -####### BUILDERS -def create_factory(commands): - factory = util.BuildFactory() - for command in commands: - factory.addStep(command) - return factory +################## +# BUILDERS +################## -def create_servo_factory(commands): - return create_factory([ - steps.Git(repourl=SERVO_REPO, mode="full", method="clobber"), - ] + commands) - - -common_test_env = { - 'RUST_BACKTRACE': '1' -} - -linux_test_env = dict({ - 'DISPLAY': ':0', - 'CARGO_HOME': '{{ common.servo_home }}/.cargo', - 'CCACHE': '/usr/bin/ccache', - 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', - 'SHELL': '/bin/bash' -}, **common_test_env) -linux_headless_env = dict({'SERVO_HEADLESS': '1'}, **linux_test_env) - -mac_test_env = dict({ - 'CARGO_HOME': '/Users/servo/.cargo', - 'CCACHE': '/usr/local/bin/ccache', - 'SERVO_CACHE_DIR': '/Users/servo/.servo' -}, **common_test_env) - -linux_dev_factory = create_servo_factory([ - steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress", "--self-test"], env=linux_headless_env), - steps.Compile(command=["./mach", "build", "--dev"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-compiletest"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-unit"], env=linux_headless_env), - steps.Compile(command=["./mach", "build-cef"], env=linux_headless_env), - steps.Compile(command=["./mach", "build-geckolib"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=linux_headless_env), -]) - -linux_rel_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--release"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "24", - "--log-raw", "test-wpt.log", - "--log-errorsummary", "wpt_errorsummary.log"], - env=linux_headless_env, - logfiles={"test-wpt.log": "test-wpt.log", - "wpt_errorsummary.log": "wpt_errorsummary.log"}), - #steps.ShellCommand(command=["pkill", "-x", "servo"], decodeRC={0: SUCCESS, 1: SUCCESS}), - steps.ShellCommand(command=["gzip", "test-wpt.log"], env=linux_headless_env), - steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "16", - "--log-raw", "test-css.log", - "--log-errorsummary", "css_errorsummary.log"], - env=linux_headless_env, - logfiles={"test-css.log": "test-css.log", - "css_errorsummary.log": "css_errorsummary.log"}), - #steps.ShellCommand(command=["pkill", "-x", "servo"], decodeRC={0: SUCCESS, 1: SUCCESS}), - steps.ShellCommand(command=["gzip", "test-css.log"], env=linux_headless_env), - steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env), - steps.Compile(command=["./mach", "build-geckolib", "--release"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=linux_headless_env), - steps.ShellCommand(command=["bash", "./etc/ci/check_no_unwrap.sh"], env=linux_headless_env), -]) - -mac_rel_wpt_factory = create_servo_factory([ - steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"]), - steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env), - steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=mac_test_env), - steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", - "--log-raw", "test-wpt.log", - "--log-errorsummary", "wpt_errorsummary.log"], - env=mac_test_env, - logfiles={"test-wpt.log": "test-wpt.log", - "wpt_errorsummary.log": "wpt_errorsummary.log"}), - steps.ShellCommand(command=["pkill", "-x", "servo"], decodeRC={0: SUCCESS, 1: SUCCESS}), - steps.ShellCommand(command=["gzip", "test-wpt.log"], env=mac_test_env), - steps.ShellCommand(command=["./mach", "build-cef", "--release"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), -]) - -mac_dev_unit_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--dev"], env=mac_test_env), - steps.ShellCommand(command=["./mach", "test-unit"], env=mac_test_env), - steps.Compile(command=["./mach", "build-cef"], env=mac_test_env), - steps.Compile(command=["./mach", "build-geckolib"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), -]) - -mac_rel_css_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env), - steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "4", - "--log-raw", "test-css.log", - "--log-errorsummary", "css_errorsummary.log"], - env=mac_test_env, - logfiles={"test-css.log": "test-css.log", - "css_errorsummary.log": "css_errorsummary.log"}), - steps.ShellCommand(command=["pkill", "-x", "servo"], decodeRC={0: SUCCESS, 1: SUCCESS}), - steps.ShellCommand(command=["gzip", "test-css.log"], env=mac_test_env), - steps.ShellCommand(command=["./mach", "build-geckolib", "--release"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env), -]) - -android_compile_env = dict({'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/', - 'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/', - 'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/', - 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:{{ common.servo_home }}/android/sdk/current/platform-tools:{{ common.servo_home }}/android/toolchain/current/bin'}, - **linux_test_env) -android_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--android", "--dev"], env=android_compile_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=android_compile_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=android_compile_env), - steps.ShellCommand(command=["python", "./etc/ci/check_dynamic_symbols.py"], env=android_compile_env), -]) - -android_nightly_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--android", "--release"], env=android_compile_env), - steps.Compile(command=["./mach", "package", "-r"], env=android_compile_env), - steps.ShellCommand(command=["s3cmd", "put", - "{{ common.servo_home }}/buildbot/slave/android-nightly/build/target/arm-linux-androideabi/release/servo.apk", - "s3://servo-rust/nightly/servo.apk"]), -]) - -gonk_compile_env = dict({'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/', - 'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/', - 'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/', - 'GONKDIR': '{{ common.servo_home }}/B2G/', - 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:{{ common.servo_home }}/B2G/prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.7/bin'}, - **linux_test_env) - -gonk_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build-gonk"], env=gonk_compile_env), - steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=gonk_compile_env), - steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=gonk_compile_env), -]) - -arm32_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', - 'BUILD_TARGET': 'arm-unknown-linux-gnueabihf', - 'PKG_CONFIG_ALLOW_CROSS': '1', - 'PKG_CONFIG_PATH': '/usr/lib/arm-linux-gnueabihf/pkgconfig', - 'EXPAT_NO_PKG_CONFIG': '1', - 'FREETYPE2_NO_PKG_CONFIG': '1', - 'FONTCONFIG_NO_PKG_CONFIG': '1', - 'CC': 'arm-linux-gnueabihf-gcc', - 'CXX': 'arm-linux-gnueabihf-g++'}, - **linux_test_env) -arm32_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--rel", "--target=arm-unknown-linux-gnueabihf"], env=arm32_compile_env), -]) - -arm64_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin', - 'BUILD_TARGET': 'aarch64-unknown-linux-gnu', - 'PKG_CONFIG_ALLOW_CROSS': '1', - 'PKG_CONFIG_PATH': '/usr/lib/aarch64-linux-gnu/pkgconfig', - 'EXPAT_NO_PKG_CONFIG': '1', - 'FREETYPE2_NO_PKG_CONFIG': '1', - 'FONTCONFIG_NO_PKG_CONFIG': '1', - 'CC': 'aarch64-linux-gnu-gcc', - 'SERVO_RUSTC_WITH_GOLD': 'False', - 'CXX': 'aarch64-linux-gnu-g++'}, - **linux_test_env) -arm64_factory = create_servo_factory([ - steps.Compile(command=["./mach", "build", "--rel", "--target=aarch64-unknown-linux-gnu"], env=arm64_compile_env), -]) - -windows_compile_env = dict({'PATH': r'C:\msys64\mingw64\bin;C:\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0;C:\Program Files\Amazon\cfn-bootstrap', - 'MSYSTEM': 'MINGW64', - 'MSYS': 'winsymlinks=lnk'}, - **common_test_env) - -windows_factory = create_servo_factory([ - steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"], env=windows_compile_env), - steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], env=windows_compile_env), -]) - -doc_factory = create_servo_factory([ - steps.ShellCommand(command=["etc/ci/upload_docs.sh"], - env={'TOKEN': GITHUB_DOC_TOKEN, - 'CARGO_HOME': '{{ common.servo_home }}/.cargo', - 'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo', - 'SHELL': '/bin/bash'}, - # important not to leak token - logEnviron=False), -]) - def branch_priority(builder, requests): for r in requests: if r.source.branch != "try": return r return requests[0] -c['builders'] = [] -c['builders'].append(util.BuilderConfig( - name="linux-dev", - slavenames=LINUX_RESERVED_SLAVES, - factory=linux_dev_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="linux-rel", - slavenames=LINUX_RESERVED_SLAVES, - factory=linux_rel_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="android", - slavenames=CROSS_SLAVES, - factory=android_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="gonk", - slavenames=CROSS_SLAVES, - factory=gonk_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="arm32", - slavenames=CROSS_SLAVES, - factory=arm32_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="arm64", - slavenames=CROSS_SLAVES, - factory=arm64_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="mac-rel-wpt", - slavenames=MAC_SLAVES, - factory=mac_rel_wpt_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="mac-dev-unit", - slavenames=MAC_SLAVES, - factory=mac_dev_unit_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="mac-rel-css", - slavenames=MAC_SLAVES, - factory=mac_rel_css_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="windows", - slavenames=WINDOWS_SLAVES, - factory=windows_factory, - nextBuild=branch_priority, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="doc", - slavenames=LINUX_RESERVED_SLAVES, - factory=doc_factory, - category="auto", -)) -c['builders'].append(util.BuilderConfig( - name="android-nightly", - slavenames=CROSS_SLAVES, - factory=android_nightly_factory, - nextBuild=branch_priority, - category="auto", -)) -####### STATUS TARGETS +class DynamicServoBuilder(util.BuilderConfig): + """ + Builder which uses DynamicServoFactory to run steps from a yaml file. + """ + def __init__(self, name, slavenames, environment): + # util.BuilderConfig is an old-style class so we cannot use super() + # but must hardcode the superclass here + util.BuilderConfig.__init__( + self, + name=name, + slavenames=slavenames, + factory=factories.DynamicServoFactory(name, environment), + nextBuild=branch_priority, + category="auto", + ) + + +c['builders'] = [ + DynamicServoBuilder("linux-dev", LINUX_SLAVES, envs.build_linux_headless), + DynamicServoBuilder("linux-rel", LINUX_SLAVES, envs.build_linux_headless), + DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android), + DynamicServoBuilder("arm32", CROSS_SLAVES, envs.build_arm32), + DynamicServoBuilder("arm64", CROSS_SLAVES, envs.build_arm64), + DynamicServoBuilder("mac-dev-unit", MAC_SLAVES, envs.build_mac), + DynamicServoBuilder("mac-rel-wpt", MAC_SLAVES, envs.build_mac), + DynamicServoBuilder("mac-rel-css", MAC_SLAVES, envs.build_mac), + DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android), + # The below builders are not dynamic but rather have hard-coded factories + util.BuilderConfig( + name="windows", + slavenames=WINDOWS_SLAVES, + factory=factories.windows, + nextBuild=branch_priority, + category="auto", + ), + util.BuilderConfig( + name="doc", + slavenames=LINUX_SLAVES, + factory=factories.doc, + category="auto", + ), +] + + +################## +# STATUS TARGETS +################## -from buildbot.status.status_push import HttpStatusPush c['status'] = [] -c['status'].append(HttpStatusPush( +c['status'].append(status_push.HttpStatusPush( serverUrl='http://build.servo.org:54856/buildbot', extra_post_params={'secret': HOMU_BUILDBOT_SECRET}, )) -from buildbot.status import html -from buildbot.status.web import authz, auth - - - -authz_cfg=authz.Authz( - auth=auth.BasicAuth([(HTTP_USERNAME, HTTP_PASSWORD)]), - gracefulShutdown = 'auth', - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = 'auth', - stopBuild = 'auth', - stopAllBuilds = 'auth', - cancelPendingBuild = 'auth', +authz_cfg = web.authz.Authz( + auth=web.auth.BasicAuth([(HTTP_USERNAME, HTTP_PASSWORD)]), + gracefulShutdown='auth', + forceBuild='auth', + forceAllBuilds='auth', + pingBuilder='auth', + stopBuild='auth', + stopAllBuilds='auth', + cancelPendingBuild='auth', ) c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) -from buildbot.status import words c['status'].append(words.IRC(host="irc.mozilla.org", - port=6697, - useSSL=True, - nick="servo_buildbot", - channels=["#servo-bots"], - notify_events={ - 'exception':1, - 'finished':1, - 'success':1, - 'failure':1 - })) - -####### PROJECT IDENTITY + port=6697, + useSSL=True, + nick="servo_buildbot", + channels=["#servo-bots"], + notify_events={ + 'exception': 1, + 'finished': 1, + 'success': 1, + 'failure': 1, + })) + + +################## +# PROJECT IDENTITY +################## + c['title'] = "Servo" c['titleURL'] = "http://github.com/servo/servo" c['buildbotURL'] = "http://build.servo.org/" -####### DB URL + +################## +# DATABASE URL +################## + c['db'] = { 'db_url': "sqlite:///state.sqlite", diff --git a/buildbot/master/files/config/passwords.py b/buildbot/master/files/config/passwords.py index d957ee54..544f2d40 100644 --- a/buildbot/master/files/config/passwords.py +++ b/buildbot/master/files/config/passwords.py @@ -3,5 +3,4 @@ SLAVE_PASSWORD = "{{ pillar['buildbot']['credentials']['slave-pass'] }}" CHANGE_PASSWORD = "{{ pillar['buildbot']['credentials']['change-pass'] }}" GITHUB_DOC_TOKEN = "{{pillar['buildbot']['credentials']['gh-doc-token'] }}" -GITHUB_STATUS_TOKEN = "{{pillar['buildbot']['credentials']['gh-status-token'] }}" HOMU_BUILDBOT_SECRET = "{{pillar['buildbot']['credentials']['homu-secret'] }}" diff --git a/buildbot/master/files/config/steps.yml b/buildbot/master/files/config/steps.yml new file mode 100644 index 00000000..006623e4 --- /dev/null +++ b/buildbot/master/files/config/steps.yml @@ -0,0 +1,65 @@ +mac-rel-wpt: + - ./mach build --release + - ./mach test-wpt-failure + - ./mach test-wpt --release --processes 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log + - ./mach build-cef --release + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +mac-dev-unit: + - ./mach build --dev + - ./mach test-unit + - ./mach build-cef + - ./mach build-geckolib + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +mac-rel-css: + - ./mach build --release + - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log + - ./mach build-geckolib --release + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +linux-dev: + - ./mach test-tidy --no-progress + - ./mach test-tidy --no-progress --self-test + - ./mach build --dev + - ./mach test-compiletest + - ./mach test-unit + - ./mach build-cef + - ./mach build-geckolib + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + - bash ./etc/ci/check_no_unwrap.sh + +linux-rel: + - ./mach build --release + - ./mach test-wpt-failure + - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log + - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log + - ./mach build-cef --release + - ./mach build-geckolib --release + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +android: + - ./mach build --android --dev + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + - python ./etc/ci/check_dynamic_symbols.py + +android-nightly: + - ./mach build --android --release + - ./mach package -r + - s3cmd put target/arm-linux-androideabi/release/servo.apk s3://servo-rust/nightly/servo.apk + +arm32: + - ./mach build --rel --target=arm-unknown-linux-gnueabihf + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh + +arm64: + - ./mach build --rel --target=aarch64-unknown-linux-gnu + - bash ./etc/ci/lockfile_changed.sh + - bash ./etc/ci/manifest_changed.sh diff --git a/buildbot/master/init.sls b/buildbot/master/init.sls index fe8010f4..c7905c0d 100644 --- a/buildbot/master/init.sls +++ b/buildbot/master/init.sls @@ -7,6 +7,7 @@ buildbot-master: - service_identity == 14.0.0 - txgithub == 15.0.0 - boto == 2.38.0 + - pyyaml == 3.11 - require: - pkg: pip service.running: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..275d0fd3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# Python modules needed for testing +# (Travis-CI will auto-install them from this file) +flake8 == 2.5.4 diff --git a/tests/lint/flake8.py b/tests/lint/flake8.py new file mode 100644 index 00000000..5eb523df --- /dev/null +++ b/tests/lint/flake8.py @@ -0,0 +1,19 @@ +import os +import subprocess + +from tests.util import Failure, Success, project_path + + +def run(): + paths = ['test.py', 'tests'] + paths = [os.path.join(project_path(), path) for path in paths] + command = ['flake8'] + paths + ret = subprocess.run(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + + if ret.returncode == 0: + return Success("Tests passed flake8 lint") + else: + return Failure("Tests failed flake8 lint:", ret.stdout) diff --git a/tests/sls/buildbot/master/config_lint.py b/tests/sls/buildbot/master/config_lint.py new file mode 100644 index 00000000..af044a2c --- /dev/null +++ b/tests/sls/buildbot/master/config_lint.py @@ -0,0 +1,20 @@ +import os +import subprocess + +from tests.util import Failure, Success, project_path + + +def run(): + CONF_DIR = os.path.join(project_path(), + 'buildbot', 'master', 'files', 'config') + # Have to specify master.cfg separately because it is not a .py file + command = ['flake8', CONF_DIR, os.path.join(CONF_DIR, 'master.cfg')] + ret = subprocess.run(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + + if ret.returncode == 0: + return Success("Buildbot master config passed linting") + else: + return Failure("Buildbot master config lint check failed:", ret.stdout)