From 22a10ef735bff4d66eefd72695941a108a3f3542 Mon Sep 17 00:00:00 2001 From: Lars Bergstrom Date: Thu, 6 Aug 2015 21:20:55 +0000 Subject: [PATCH 1/3] Switch linux builders to EC2 --- buildbot-master.sls | 3 + buildbot/master/master.cfg | 114 +++++++++++++++++++++++------------ buildbot/master/passwords.py | 5 +- homu/cfg.toml | 4 +- top.sls | 5 ++ 5 files changed, 91 insertions(+), 40 deletions(-) diff --git a/buildbot-master.sls b/buildbot-master.sls index f0a67509..c718a6a0 100644 --- a/buildbot-master.sls +++ b/buildbot-master.sls @@ -5,6 +5,9 @@ buildbot: txgithub: pip.installed +boto: + pip.installed + buildbot-master: service: - running diff --git a/buildbot/master/master.cfg b/buildbot/master/master.cfg index 7c76f7ee..cc3acc61 100644 --- a/buildbot/master/master.cfg +++ b/buildbot/master/master.cfg @@ -5,10 +5,12 @@ from buildbot.plugins import * from passwords import HTTP_USERNAME, HTTP_PASSWORD, SLAVE_PASSWORD, CHANGE_PASSWORD from passwords import GITHUB_DOC_TOKEN, GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET +from passwords import AWS_ACCESS_KEY, AWS_SECRET_KEY +from passwords import MINION_PUBLIC_KEY, MINION_PRIVATE_KEY SERVO_REPO = "https://github.com/servo/servo" HEAD_SLAVES = ["servo-head"] -LINUX_SLAVES = ["servo-linux1", "servo-linux2", "servo-linux3"] +LINUX_SLAVES = ["linux1", "linux2"] MAC_SLAVES = ["servo-mac1", "servo-mac2", "servo-mac3"] ANDROID_SLAVES = ["servo-linux-android1"] @@ -28,9 +30,41 @@ c['caches'] = { ####### BUILDSLAVES +def make_user_data(s): + lines = [] + lines.append("#!/bin/bash") + lines.append("add-apt-repository -y ppa:saltstack/salt") + lines.append("apt-get update") + lines.append("apt-get -y install python-pip salt-minion") + lines.append("echo 96.126.125.232 salt >> /etc/hosts") + lines.append("echo %s > /etc/salt/minion_id" % s) + lines.append("cat < /etc/salt/pki/minion/minion.pem") + lines.append(MINION_PRIVATE_KEY.strip()) + lines.append("EOF") + lines.append("cat < /etc/salt/pki/minion/minion.pub") + lines.append(MINION_PUBLIC_KEY.strip()) + lines.append("EOF") + lines.append("service salt-minion restart") + lines.append("sleep 5") + lines.append("salt-call state.highstate") + return "\n".join(lines) + c['slaves'] = [] -for s in LINUX_SLAVES + MAC_SLAVES + ANDROID_SLAVES + HEAD_SLAVES: +for s in MAC_SLAVES + ANDROID_SLAVES + HEAD_SLAVES: c['slaves'].append(buildslave.BuildSlave(s, SLAVE_PASSWORD, max_builds=1)) +for s in LINUX_SLAVES: + c['slaves'].append(buildslave.EC2LatentBuildSlave( + s, SLAVE_PASSWORD, "c4.8xlarge", + max_builds=1, + build_wait_timeout=60*60, + ami="ami-01020a31", + region="us-west-2", + identifier=AWS_ACCESS_KEY, + secret_identifier=AWS_SECRET_KEY, + keypair_name="servo", + security_name="servo-test", + tags={"Name": "servo-%s" % s}, + user_data=make_user_data(s))) ####### CHANGESOURCES @@ -71,7 +105,7 @@ c['schedulers'] = [] c['schedulers'].append(schedulers.AnyBranchScheduler( name="servo_auto", treeStableTimer=None, - builderNames=["linux1", "linux2", "linux3", "mac1", "mac2", "mac3", "android", "gonk"], + builderNames=["linux1", "linux2", "mac1", "mac2", "mac3", "android", "gonk"], change_filter=util.ChangeFilter(filter_fn=servo_auto_try_filter), )) c['schedulers'].append(schedulers.SingleBranchScheduler( @@ -82,7 +116,7 @@ c['schedulers'].append(schedulers.SingleBranchScheduler( )) c['schedulers'].append(schedulers.ForceScheduler( name="force", - builderNames=["linux1", "linux2", "linux3", "mac1", "mac2", "mac3", "android", "gonk", "linux1-rel", "linux2-rel", "mac1-rel", "mac2-rel","android-nightly"] + builderNames=["linux1", "linux2", "mac1", "mac2", "mac3", "android", "gonk", "linux1-rel", "linux2-rel", "mac1-rel", "mac2-rel","android-nightly"] )) c['schedulers'].append(schedulers.Nightly( name="Nightly", @@ -137,7 +171,7 @@ linux1_factory.addStep(steps.Compile(command=["./mach", "build", "--dev"], env=l linux1_factory.addStep(steps.ShellCommand(command=["./mach", "test-unit"], env=linux_headless_env)) linux1_factory.addStep(steps.ShellCommand(command=["./mach", "test-ref", "--kind", "cpu"], env=linux_headless_env)) -linux1_factory.addStep(steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env)) +linux1_factory.addStep(steps.Compile(command=["./mach", "build-cef"], env=linux_headless_env)) linux2_factory = util.BuildFactory() @@ -145,20 +179,26 @@ linux2_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobbe linux2_factory.addStep(steps.Compile(command=["./mach", "build", "--release"], env=linux_headless_env)) linux2_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=linux_headless_env)) -linux2_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", - "--log-raw", "wpt_raw.log"], +linux2_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "24", + "--log-raw", "test-wpt.log"], env=linux_headless_env, - logfiles={"wpt_raw.log": "wpt_raw.log"})) -linux2_factory.addStep(steps.ShellCommand(command=["gzip", "wpt_raw.log"], env=linux_headless_env)) - -linux3_factory = util.BuildFactory() -linux3_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) -linux3_factory.addStep(steps.Compile(command=["./mach", "build", "--release"], env=linux_headless_env)) -linux3_factory.addStep(steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "4", - "--log-raw", "wpt_raw.log"], + logfiles={"test-wpt.log": "test-wpt.log"})) +linux2_factory.addStep(steps.ShellCommand(command=["gzip", "test-wpt.log"], env=linux_headless_env)) +linux2_factory.addStep(steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "24", + "--log-raw", "test-css.log"], env=linux_headless_env, - logfiles={"wpt_raw.log": "wpt_raw.log"})) -linux3_factory.addStep(steps.ShellCommand(command=["gzip", "wpt_raw.log"], env=linux_headless_env)) + logfiles={"test-css.log": "test-css.log"})) +linux2_factory.addStep(steps.ShellCommand(command=["gzip", "test-css.log"], env=linux_headless_env)) +linux2_factory.addStep(steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env)) + +#linux3_factory = util.BuildFactory() +#linux3_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) +#linux3_factory.addStep(steps.Compile(command=["./mach", "build", "--release"], env=linux_headless_env)) +#linux3_factory.addStep(steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "24", +# "--log-raw", "test-css.log"], +# env=linux_headless_env, +# logfiles={"test-css.log": "test-css.log"})) +#linux3_factory.addStep(steps.ShellCommand(command=["gzip", "test-css.log"], env=linux_headless_env)) task_limited_test_env = dict({'RUST_TEST_TASKS': '1'}, **mac_test_env) @@ -170,10 +210,10 @@ mac1_factory.addStep(steps.Compile(command=["./mach", "build", "--release"], env mac1_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=mac_test_env)) mac1_factory.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", - "--log-raw", "wpt_raw.log"], + "--log-raw", "test-wpt.log"], env=mac_test_env, - logfiles={"wpt_raw.log": "wpt_raw.log"})) -mac1_factory.addStep(steps.ShellCommand(command=["gzip", "wpt_raw.log"], env=mac_test_env)) + logfiles={"test-wpt.log": "test-wpt.log"})) +mac1_factory.addStep(steps.ShellCommand(command=["gzip", "test-wpt.log"], env=mac_test_env)) mac2_factory = util.BuildFactory() mac2_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) @@ -187,10 +227,10 @@ mac3_factory = util.BuildFactory() mac3_factory.addStep(steps.Git(repourl=SERVO_REPO, mode="full", method="clobber")) mac3_factory.addStep(steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env)) mac3_factory.addStep(steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "4", - "--log-raw", "wpt_raw.log"], + "--log-raw", "test-css.log"], env=mac_test_env, - logfiles={"wpt_raw.log": "wpt_raw.log"})) -mac3_factory.addStep(steps.ShellCommand(command=["gzip", "wpt_raw.log"], env=mac_test_env)) + logfiles={"test-css.log": "test-css.log"})) +mac3_factory.addStep(steps.ShellCommand(command=["gzip", "test-css.log"], env=mac_test_env)) android_compile_env = dict({'ANDROID_SDK': '/home/servo/android-sdk-linux/', 'ANDROID_NDK': '/home/servo/android-ndk-r10c/', @@ -247,10 +287,10 @@ linux2_factory_rel.addStep(steps.Compile(command=["./mach", "build", "--release" linux2_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=linux_test_env)) linux2_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--release", - "--processes", "4", - "--log-raw", "wpt_raw.log"], + "--processes", "24", + "--log-raw", "test-wpt.log"], env=linux_test_env, - logfiles={"wpt_raw.log": "wpt_raw.log"})) + logfiles={"test-wpt.log": "test-wpt.log"})) linux2_factory_rel.addStep(steps.Compile(command=["./mach", "build-cef", "--release"])) mac1_factory_rel = util.BuildFactory() @@ -259,9 +299,9 @@ mac1_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-tidy"])) mac1_factory_rel.addStep(steps.Compile(command=["./mach", "build", "--release"])) mac1_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=common_test_env)) -mac1_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", "--total-chunks", "2", "--this-chunk", "1", "--log-raw", "wpt_raw.log"], +mac1_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4", "--total-chunks", "2", "--this-chunk", "1", "--log-raw", "test-wpt.log"], env=common_test_env, - logfiles={"wpt_raw.log": "wpt_raw.log"})) + logfiles={"test-wpt.log": "test-wpt.log"})) # https://github.com/rust-lang/cargo/issues/691 # mac1_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-unit"], env=common_test_env)) mac1_factory_rel.addStep(steps.Compile(command=["./mach", "build-cef"])) @@ -272,9 +312,9 @@ mac2_factory_rel.addStep(steps.Compile(command=["./mach", "build", "--release"]) mac2_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-content"], env=common_test_env)) mac2_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-ref", "--kind", "cpu"], env=task_limited_test_env)) -mac2_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--processes", "4", "--total-chunks", "2", "--this-chunk", "2", "--log-raw", "wpt_raw.log"], +mac2_factory_rel.addStep(steps.ShellCommand(command=["./mach", "test-wpt", "--processes", "4", "--total-chunks", "2", "--this-chunk", "2", "--log-raw", "test-wpt.log"], env=common_test_env, - logfiles={"wpt_raw.log": "wpt_raw.log"})) + logfiles={"test-wpt.log": "test-wpt.log"})) # Cargo builds # NOTE: We use this instead of Travis CI for some projects, either for platform reasons or because @@ -368,13 +408,13 @@ c['builders'].append(util.BuilderConfig( nextBuild=branch_priority, category="auto", )) -c['builders'].append(util.BuilderConfig( - name="linux3", - slavenames=LINUX_SLAVES, - factory=linux3_factory, - nextBuild=branch_priority, - category="auto", -)) +#c['builders'].append(util.BuilderConfig( +# name="linux3", +# slavenames=LINUX_SLAVES, +# factory=linux3_factory, +# nextBuild=branch_priority, +# category="auto", +#)) c['builders'].append(util.BuilderConfig( name="android", slavenames=ANDROID_SLAVES, diff --git a/buildbot/master/passwords.py b/buildbot/master/passwords.py index 3297470e..3ef7ad70 100644 --- a/buildbot/master/passwords.py +++ b/buildbot/master/passwords.py @@ -5,4 +5,7 @@ GITHUB_DOC_TOKEN = "{{pillar['buildbot']['credentials']['gh-doc-token'] }}" GITHUB_STATUS_TOKEN = "{{pillar['buildbot']['credentials']['gh-status-token'] }}" HOMU_BUILDBOT_SECRET = "{{pillar['buildbot']['credentials']['homu-secret'] }}" - +AWS_ACCESS_KEY = "{{pillar['buildbot']['credentials']['aws-access-key'] }}" +AWS_SECRET_KEY = "{{pillar['buildbot']['credentials']['aws-secret-key'] }}" +MINION_PUBLIC_KEY = """{{pillar['buildbot']['credentials']['minion-public-key'] }}""" +MINION_PRIVATE_KEY = """{{pillar['buildbot']['credentials']['minion-private-key'] }}""" diff --git a/homu/cfg.toml b/homu/cfg.toml index aa34d4c9..d3b68e00 100644 --- a/homu/cfg.toml +++ b/homu/cfg.toml @@ -36,8 +36,8 @@ secret = "{{ pillar["homu"]["gh-webhook-secret"] }}" [repo.servo.buildbot] url = "http://build.servo.org" secret = "{{ pillar["homu"]["buildbot-secret"] }}" -builders = ["linux1", "linux2", "linux3", "android", "mac1", "mac2", "mac3", "gonk"] -try_builders = ["linux1", "linux2", "linux3", "android", "mac1", "mac2", "mac3", "gonk"] +builders = ["linux1", "linux2", "android", "mac1", "mac2", "mac3", "gonk"] +try_builders = ["linux1", "linux2", "android", "mac1", "mac2", "mac3", "gonk"] username = "{{ pillar["homu"]["buildbot-http-user"] }}" password = "{{ pillar["homu"]["buildbot-http-pass"] }}" diff --git a/top.sls b/top.sls index 6b7dc21b..fffea297 100644 --- a/top.sls +++ b/top.sls @@ -16,6 +16,11 @@ base: - match: pcre - xvfb + 'linux\d+': + - match: pcre + - buildbot-slave + - xvfb + 'servo-linux-android\d+': - match: pcre - buildbot-slave From d0c2318c4d81c839a00b1f7cbaf63dd41951fe30 Mon Sep 17 00:00:00 2001 From: Lars Bergstrom Date: Thu, 27 Aug 2015 16:15:04 +0000 Subject: [PATCH 2/3] Add heapsize and rust-url to Homu --- homu/cfg.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/homu/cfg.toml b/homu/cfg.toml index d3b68e00..d568cb49 100644 --- a/homu/cfg.toml +++ b/homu/cfg.toml @@ -21,7 +21,8 @@ port = 54856 ('servo', 'libfontconfig'), ('servo', 'skia'), ('servo', 'cocoa-rs'), ('servo', 'cgl-rs'), ('servo', 'io-surface-rs'), ('servo', 'rust-cssparser'), ('servo', 'unicode-bidi'), ('servo', 'rust-smallvec'), ('servo', 'core-text-rs'), - ('servo', 'libexpat'), ('servo', 'rust-harfbuzz')] %} + ('servo', 'libexpat'), ('servo', 'rust-harfbuzz'), ('servo', 'heapsize'), + ('servo', 'rust-url')] %} {% set buildbot_repos = [('servo', 'mozjs'), ('servo', 'rust-mozjs')] %} {% set reviewers = ["glennw","jdm","kmcallister","larsbergstrom","Manishearth","metajack","mbrubeck","Ms2ger","pcwalton","SimonSapin","mrobinson","brunoabinader","saneyuki","nox","zmike","edunham"] %} From c1a0c2ee1632f8c402149af44ba47d41cd4409d5 Mon Sep 17 00:00:00 2001 From: Lars Bergstrom Date: Thu, 27 Aug 2015 16:15:30 +0000 Subject: [PATCH 3/3] Add macpro names to pattern for mac builders --- top.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/top.sls b/top.sls index fffea297..3b849666 100644 --- a/top.sls +++ b/top.sls @@ -8,7 +8,7 @@ base: - homu - nginx - 'servo-(linux|mac)\d+': + 'servo-(linux|mac|macpro)\d+': - match: pcre - buildbot-slave