diff --git a/buildbot/master/master.cfg b/buildbot/master/master.cfg index f2fc7969..dafb19d8 100644 --- a/buildbot/master/master.cfg +++ b/buildbot/master/master.cfg @@ -147,6 +147,19 @@ c['schedulers'].append(schedulers.SingleBranchScheduler( ####### BUILDERS +def create_factory(commands): + factory = util.BuildFactory() + for command in commands: + factory.addStep(command) + return factory + + +def create_servo_factory(commands): + return create_factory([ + steps.Git(repourl=SERVO_REPO, mode="full", method="clobber"), + ] + commands) + + common_test_env = { 'RUST_BACKTRACE': '1' } @@ -164,77 +177,71 @@ mac_test_env = dict({ 'SERVO_CACHE_DIR': '/Users/servo/.servo' }, **common_test_env) -linux_dev_factory = util.BuildFactory() -linux_dev_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -linux_dev_factory.addStep(steps.ShellCommand(command=["./mach", "test-tidy"], env=linux_headless_env)) -linux_dev_factory.addStep(steps.Compile(command=["./mach", "build", "--dev"], env=linux_headless_env)) -linux_dev_factory.addStep(steps.ShellCommand(command=["./mach", "test-unit"], env=linux_headless_env)) -linux_dev_factory.addStep(steps.ShellCommand(command=["./mach", "test-ref", "--kind", "cpu"], - env=linux_headless_env)) -linux_dev_factory.addStep(steps.Compile(command=["./mach", "build-cef"], env=linux_headless_env)) - - -linux_rel_factory = util.BuildFactory() -linux_rel_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -linux_rel_factory.addStep(steps.Compile(command=["./mach", "build", "--release"], env=linux_headless_env)) -linux_rel_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt-failure"], - env=linux_headless_env)) -linux_rel_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "24", - "--log-raw", "test-wpt.log"], - env=linux_headless_env, - logfiles={"test-wpt.log": "test-wpt.log"})) -linux_rel_factory.addStep(steps.ShellCommand(command=["gzip", "test-wpt.log"], env=linux_headless_env)) -linux_rel_factory.addStep(steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "24", - "--log-raw", "test-css.log"], - env=linux_headless_env, - logfiles={"test-css.log": "test-css.log"})) -linux_rel_factory.addStep(steps.ShellCommand(command=["gzip", "test-css.log"], env=linux_headless_env)) -linux_rel_factory.addStep(steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env)) +linux_dev_factory = create_servo_factory([ + steps.ShellCommand(command=["./mach", "test-tidy"], env=linux_headless_env), + steps.Compile(command=["./mach", "build", "--dev"], env=linux_headless_env), + steps.ShellCommand(command=["./mach", "test-unit"], env=linux_headless_env), + steps.ShellCommand(command=["./mach", "test-ref", "--kind", "cpu"], env=linux_headless_env), + steps.Compile(command=["./mach", "build-cef"], env=linux_headless_env), +]) + +linux_rel_factory = create_servo_factory([ + steps.Compile(command=["./mach", "build", "--release"], env=linux_headless_env), + steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=linux_headless_env), + steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "24", + "--log-raw", "test-wpt.log"], + env=linux_headless_env, logfiles={"test-wpt.log": "test-wpt.log"}), + steps.ShellCommand(command=["gzip", "test-wpt.log"], env=linux_headless_env), + steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "24", + "--log-raw", "test-css.log"], + env=linux_headless_env, logfiles={"test-css.log": "test-css.log"}), + steps.ShellCommand(command=["gzip", "test-css.log"], env=linux_headless_env), + steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env), +]) task_limited_test_env = dict({'RUST_TEST_TASKS': '1'}, **mac_test_env) -mac_rel_wpt_factory = util.BuildFactory() -mac_rel_wpt_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -mac_rel_wpt_factory.addStep(steps.ShellCommand(command=["./mach", "test-tidy"])) -mac_rel_wpt_factory.addStep(steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env)) -mac_rel_wpt_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt-failure"], - env=mac_test_env)) -mac_rel_wpt_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", - "--log-raw", "test-wpt.log"], - env=mac_test_env, - logfiles={"test-wpt.log": "test-wpt.log"})) -mac_rel_wpt_factory.addStep(steps.ShellCommand(command=["gzip", "test-wpt.log"], env=mac_test_env)) - -mac_dev_ref_unit_factory = util.BuildFactory() -mac_dev_ref_unit_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -mac_dev_ref_unit_factory.addStep(steps.Compile(command=["./mach", "build", "--dev"], env=mac_test_env)) -mac_dev_ref_unit_factory.addStep(steps.ShellCommand(command=["./mach", "test-ref", "--kind", "cpu"], - env=task_limited_test_env)) -mac_dev_ref_unit_factory.addStep(steps.ShellCommand(command=["./mach", "test-unit"], env=mac_test_env)) -mac_dev_ref_unit_factory.addStep(steps.Compile(command=["./mach", "build-cef"], env=mac_test_env)) - -mac_rel_css_factory = util.BuildFactory() -mac_rel_css_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -mac_rel_css_factory.addStep(steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env)) -mac_rel_css_factory.addStep(steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "4", - "--log-raw", "test-css.log"], - env=mac_test_env, - logfiles={"test-css.log": "test-css.log"})) -mac_rel_css_factory.addStep(steps.ShellCommand(command=["gzip", "test-css.log"], env=mac_test_env)) +mac_rel_wpt_factory = create_servo_factory([ + steps.ShellCommand(command=["./mach", "test-tidy"]), + steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env), + steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=mac_test_env), + steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", + "--log-raw", "test-wpt.log"], + env=mac_test_env, logfiles={"test-wpt.log": "test-wpt.log"}), + steps.ShellCommand(command=["gzip", "test-wpt.log"], env=mac_test_env), +]) + +mac_dev_ref_unit_factory = create_servo_factory([ + steps.Compile(command=["./mach", "build", "--dev"], env=mac_test_env), + steps.ShellCommand(command=["./mach", "test-ref", "--kind", "cpu"], + env=task_limited_test_env), + steps.ShellCommand(command=["./mach", "test-unit"], env=mac_test_env), + steps.Compile(command=["./mach", "build-cef"], env=mac_test_env), +]) + +mac_rel_css_factory = create_servo_factory([ + steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env), + steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "4", + "--log-raw", "test-css.log"], + env=mac_test_env, logfiles={"test-css.log": "test-css.log"}), + steps.ShellCommand(command=["gzip", "test-css.log"], env=mac_test_env), +]) android_compile_env = dict({'ANDROID_SDK': '/home/servo/android-sdk-linux/', 'ANDROID_NDK': '/home/servo/android-ndk-r10c/', 'ANDROID_TOOLCHAIN': '/home/servo/ndk-toolchain/', 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:/home/servo/android-sdk-linux/platform-tools:/home/servo/ndk-toolchain/bin'}, **linux_test_env) -android_factory = util.BuildFactory() -android_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -android_factory.addStep(steps.Compile(command=["./mach", "build", "--android", "--dev"], env=android_compile_env)) +android_factory = create_servo_factory([ + steps.Compile(command=["./mach", "build", "--android", "--dev"], env=android_compile_env), +]) -android_nightly_factory = util.BuildFactory() -android_nightly_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -android_nightly_factory.addStep(steps.Compile(command=["./mach", "build", "--android", "--release"], env=android_compile_env)) -android_nightly_factory.addStep(steps.ShellCommand(command=["s3cmd", "put", "/home/servo/buildbot/slave/android-nightly/build/target/arm-linux-androideabi/release/servo", "s3://servo-rust/nightly/servo.apk"])) +android_nightly_factory = create_servo_factory([ + steps.Compile(command=["./mach", "build", "--android", "--release"], env=android_compile_env), + steps.ShellCommand(command=["s3cmd", "put", + "/home/servo/buildbot/slave/android-nightly/build/target/arm-linux-androideabi/release/servo", + "s3://servo-rust/nightly/servo.apk"]), +]) gonk_compile_env = dict({'ANDROID_SDK': '/home/servo/android-sdk-linux/', 'ANDROID_NDK': '/home/servo/android-ndk-r10c/', @@ -243,89 +250,52 @@ gonk_compile_env = dict({'ANDROID_SDK': '/home/servo/android-sdk-linux/', 'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:/home/servo/B2G/prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.7/bin'}, **linux_test_env) -gonk_factory = util.BuildFactory() -gonk_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -gonk_factory.addStep(steps.Compile(command=["./mach", "build-gonk"], env=gonk_compile_env)) +gonk_factory = create_servo_factory([ + steps.Compile(command=["./mach", "build-gonk"], env=gonk_compile_env), +]) -doc_factory = util.BuildFactory() -doc_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -doc_factory.addStep(steps.ShellCommand(command=["etc/ci/upload_docs.sh"], - env={'TOKEN': GITHUB_DOC_TOKEN, - 'CARGO_HOME': '/home/servo/.cargo', - 'SERVO_CACHE_DIR': '/home/servo/.servo', - 'SHELL': '/bin/bash'}, - logEnviron=False)) # important not to leak token +doc_factory = create_servo_factory([ + steps.ShellCommand(command=["etc/ci/upload_docs.sh"], + env={'TOKEN': GITHUB_DOC_TOKEN, + 'CARGO_HOME': '/home/servo/.cargo', + 'SERVO_CACHE_DIR': '/home/servo/.servo', + 'SHELL': '/bin/bash'}, + # important not to leak token + logEnviron=False), +]) # Cargo builds # NOTE: We use this instead of Travis CI for some projects, either for platform reasons or because # Travis CI can't handle them. - -cargo_linux_env = {'SHELL': '/bin/bash', - 'PATH': ['${HOME}/.rust/bin', '${PATH}'], - 'LD_LIBRARY_PATH': '${HOME}/.rust/lib'} -cargo_mac_env = {'PATH': ['${HOME}/.rust/bin', '${PATH}'], - 'DYLD_LIBRARY_PATH': '${HOME}/.rust/lib'} - -cargo_linux_factory = util.BuildFactory() -cargo_linux_factory.addStep(steps.Git(repourl=util.Property('repository'), mode="full", method="clobber")) -cargo_linux_factory.addStep(steps.ShellCommand( - name="download rustup", - command=["curl","-sL","https://static.rust-lang.org/rustup.sh","-o","/tmp/rustup.sh"] -)) -cargo_linux_factory.addStep(steps.ShellCommand( - name="install rust", - command=["sh", "/tmp/rustup.sh", "--prefix=~/.rust", "--channel=nightly", "-y", "--disable-sudo", "--save"], - env={'SHELL': '/bin/bash'}, - want_stderr=False -)) -cargo_linux_factory.addStep(steps.Compile( - name="cargo build", - description="cargo building", - descriptionDone="cargo build", - command=["cargo","build","--verbose"], - env=cargo_linux_env -)) -cargo_linux_factory.addStep(steps.ShellCommand( - name="cargo test", - description="cargo testing", - descriptionDone="cargo test", - command=["cargo","test","--verbose"], - env=cargo_linux_env -)) -cargo_linux_factory.addStep(steps.RemoveDirectory( - name="cleanup", - dir="~/buildbot/rust" -)) - -cargo_mac_factory = util.BuildFactory() -cargo_mac_factory.addStep(steps.Git(repourl=util.Property('repository'), mode="full", method="clobber")) -cargo_mac_factory.addStep(steps.ShellCommand( - name="download rustup", - command=["curl","-sL","https://static.rust-lang.org/rustup.sh","-o","/tmp/rustup.sh"] -)) -cargo_mac_factory.addStep(steps.ShellCommand( - name="install rust", - command=["sh", "/tmp/rustup.sh", "--prefix=~/.rust", "--channel=nightly", "-y", "--disable-sudo", "--save"], - want_stderr=False -)) -cargo_mac_factory.addStep(steps.Compile( - name="cargo build", - description="cargo building", - descriptionDone="cargo build", - command=["cargo","build","--verbose"], - env=cargo_mac_env -)) -cargo_mac_factory.addStep(steps.ShellCommand( - name="cargo test", - description="cargo testing", - descriptionDone="cargo test", - command=["cargo","test","--verbose"], - env=cargo_mac_env -)) -cargo_mac_factory.addStep(steps.RemoveDirectory( - name="cleanup", - dir="~/buildbot/rust" -)) +def create_cargo_factory(env, cargo_env): + cargo_env = dict(env, **cargo_env) + + return create_factory([ + steps.Git(repourl=util.Property('repository'), mode="full", method="clobber"), + steps.ShellCommand(name="download rustup", + command=["curl", "-sL", "https://static.rust-lang.org/rustup.sh", + "-o", "/tmp/rustup.sh"]), + steps.ShellCommand(name="install rust", + command=["sh", "/tmp/rustup.sh", "--prefix=~/.rust", "--channel=nightly", + "-y", "--disable-sudo", "--save"], + env=env, want_stderr=False), + steps.Compile(name="cargo build", description="cargo building", descriptionDone="cargo build", + command=["cargo", "build", "--verbose"], env=cargo_env), + steps.ShellCommand(name="cargo test", description="cargo testing", + descriptionDone="cargo test", command=["cargo", "test", "--verbose"], + env=cargo_env), + steps.RemoveDirectory(name="cleanup", dir="~/buildbot/rust"), + ]) + + +cargo_linux_factory = create_cargo_factory({'SHELL': '/bin/bash'}, { + 'PATH': ['${HOME}/.rust/bin', '${PATH}'], + 'LD_LIBRARY_PATH': '${HOME}/.rust/lib', +}) +cargo_mac_factory = create_cargo_factory({'SHELL': '/bin/bash'}, { + 'PATH': ['${HOME}/.rust/bin', '${PATH}'], + 'DYLD_LIBRARY_PATH': '${HOME}/.rust/lib', +}) def branch_priority(builder, requests): for r in requests: