From e6e1a3d4c95af3e1243c1a146e601bb101704878 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 29 Jun 2016 21:18:15 -0400 Subject: [PATCH 1/2] Alphabetize list of builder names --- buildbot/master/files/config/master.cfg | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index da2b0436..1c212e81 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -85,23 +85,25 @@ c['schedulers'].append(schedulers.SingleBranchScheduler( c['schedulers'].append(schedulers.ForceScheduler( name="force", builderNames=[ + "android", + "android-nightly", + "arm32", + "arm64", "linux-dev", - "linux-rel", "linux-dev-yaml", - "mac-rel-wpt", + "linux-rel", "mac-dev-unit", "mac-rel-css", - "android", - "arm32", - "arm64", - "android-nightly", + "mac-rel-wpt", "windows", ], )) c['schedulers'].append(schedulers.Nightly( name="Nightly", branch="master", - builderNames=["android-nightly"], + builderNames=[ + "android-nightly", + ], hour=1, minute=0, )) From f9f12304ad3bb393812c825c9c0df667f38e544a Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 29 Jun 2016 21:53:48 -0400 Subject: [PATCH 2/2] Add Linux and Mac nightlies Uses new etc/ci/upload_nightly.sh script. Passes credentials more securely via environment variables. --- .travis/test_pillars/buildbot/master.sls | 2 ++ buildbot/master/files/config/factories.py | 20 +++++++++++++++++-- buildbot/master/files/config/master.cfg | 4 ++++ buildbot/master/files/config/passwords.py | 24 +++++++++++++++++------ buildbot/master/files/config/steps.yml | 12 +++++++++++- 5 files changed, 53 insertions(+), 9 deletions(-) diff --git a/.travis/test_pillars/buildbot/master.sls b/.travis/test_pillars/buildbot/master.sls index 643ffc9d..7dc80a37 100644 --- a/.travis/test_pillars/buildbot/master.sls +++ b/.travis/test_pillars/buildbot/master.sls @@ -6,3 +6,5 @@ buildbot: 'gh-doc-token': 'TEST_BUILDBOT_GH_DOC_TOKEN' 'gh-hook-secret': 'TEST_BUILDBOT_GH_HOOK_SECRET' 'homu-secret': 'TEST_BUILDBOT_HOMU_SECRET' + 's3-upload-access-key-id': 'TEST_BUILDBOT_S3_UPLOAD_ACCESS_KEY_ID' + 's3-upload-secret-access-key': 'TEST_BUILDBOT_S3_UPLOAD_SECRET_ACCESS_KEY' diff --git a/buildbot/master/files/config/factories.py b/buildbot/master/files/config/factories.py index 97d752a1..20e9759b 100644 --- a/buildbot/master/files/config/factories.py +++ b/buildbot/master/files/config/factories.py @@ -1,3 +1,4 @@ +import copy import os.path import re @@ -8,6 +9,7 @@ import yaml import environments as envs +from passwords import S3_UPLOAD_ACCESS_KEY_ID, S3_UPLOAD_SECRET_ACCESS_KEY SERVO_REPO = "https://github.com/servo/servo" @@ -78,7 +80,7 @@ def __init__(self, builder_name, environment): def make_step(self, command): step_kwargs = {} - step_kwargs['env'] = self.environment + step_env = copy.deepcopy(self.environment) command = command.split(' ') step_kwargs['command'] = command @@ -102,6 +104,13 @@ def make_step(self, command): step_kwargs['logfiles'] = {} step_kwargs['logfiles'][logfile] = logfile + # Provide environment variables for s3cmd + elif arg == './etc/ci/upload_nightly.sh': + step_kwargs['logEnviron'] = False + step_env['AWS_ACCESS_KEY_ID'] = S3_UPLOAD_ACCESS_KEY_ID + step_env['AWS_SECRET_ACCESS_KEY'] = S3_UPLOAD_SECRET_ACCESS_KEY + + step_kwargs['env'] = self.environment return step_class(**step_kwargs) @@ -158,7 +167,7 @@ def run(self): def make_step(self, command): step_kwargs = {} - step_kwargs['env'] = self.environment + step_env = copy.deepcopy(self.environment) command = command.split(' ') step_kwargs['command'] = command @@ -182,6 +191,13 @@ def make_step(self, command): step_kwargs['logfiles'] = {} step_kwargs['logfiles'][logfile] = logfile + # Provide environment variables for s3cmd + elif arg == './etc/ci/upload_nightly.sh': + step_kwargs['logEnviron'] = False + step_env['AWS_ACCESS_KEY_ID'] = S3_UPLOAD_ACCESS_KEY_ID + step_env['AWS_SECRET_ACCESS_KEY'] = S3_UPLOAD_SECRET_ACCESS_KEY + + step_kwargs['env'] = self.environment return step_class(**step_kwargs) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index 1c212e81..812179f4 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -103,6 +103,8 @@ c['schedulers'].append(schedulers.Nightly( branch="master", builderNames=[ "android-nightly", + "linux-nightly", + "mac-nightly", ], hour=1, minute=0, @@ -158,12 +160,14 @@ class DynamicServoYAMLBuilder(util.BuilderConfig): c['builders'] = [ DynamicServoBuilder("linux-dev", LINUX_SLAVES, envs.build_linux), DynamicServoBuilder("linux-rel", LINUX_SLAVES, envs.build_linux), + DynamicServoBuilder("linux-nightly", LINUX_SLAVES, envs.build_linux), DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android), DynamicServoBuilder("arm32", CROSS_SLAVES, envs.build_arm32), DynamicServoBuilder("arm64", CROSS_SLAVES, envs.build_arm64), DynamicServoBuilder("mac-dev-unit", MAC_SLAVES, envs.build_mac), DynamicServoBuilder("mac-rel-wpt", MAC_SLAVES, envs.build_mac), DynamicServoBuilder("mac-rel-css", MAC_SLAVES, envs.build_mac), + DynamicServoBuilder("mac-nightly", MAC_SLAVES, envs.build_mac), DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android), # The below builders are not dynamic but rather have hard-coded factories util.BuilderConfig( diff --git a/buildbot/master/files/config/passwords.py b/buildbot/master/files/config/passwords.py index 544f2d40..c16fb737 100644 --- a/buildbot/master/files/config/passwords.py +++ b/buildbot/master/files/config/passwords.py @@ -1,6 +1,18 @@ -HTTP_USERNAME = "{{ pillar['buildbot']['credentials']['http-user'] }}" -HTTP_PASSWORD = "{{ pillar['buildbot']['credentials']['http-pass'] }}" -SLAVE_PASSWORD = "{{ pillar['buildbot']['credentials']['slave-pass'] }}" -CHANGE_PASSWORD = "{{ pillar['buildbot']['credentials']['change-pass'] }}" -GITHUB_DOC_TOKEN = "{{pillar['buildbot']['credentials']['gh-doc-token'] }}" -HOMU_BUILDBOT_SECRET = "{{pillar['buildbot']['credentials']['homu-secret'] }}" +import json + +# Jinja will replace the inside with double-quote-using JSON, +# so use single quotes to delimit the string. +# Use double quotes inside to keep the expression as a single string. +credentials = json.loads('{{ pillar["buildbot"]["credentials"]|json }}') +# json.loads creates unicode strings but Buildbot requires bytestrings. +# Python 2's Unicode situation makes me sad. +credentials = {k: v.encode('utf-8') for k, v in credentials.items()} + +HTTP_USERNAME = credentials['http-user'] +HTTP_PASSWORD = credentials['http-pass'] +SLAVE_PASSWORD = credentials['slave-pass'] +CHANGE_PASSWORD = credentials['change-pass'] +GITHUB_DOC_TOKEN = credentials['gh-doc-token'] +HOMU_BUILDBOT_SECRET = credentials['homu-secret'] +S3_UPLOAD_ACCESS_KEY_ID = credentials['s3-upload-access-key-id'] +S3_UPLOAD_SECRET_ACCESS_KEY = credentials['s3-upload-secret-access-key'] diff --git a/buildbot/master/files/config/steps.yml b/buildbot/master/files/config/steps.yml index 7668b3c2..ff911f33 100644 --- a/buildbot/master/files/config/steps.yml +++ b/buildbot/master/files/config/steps.yml @@ -22,6 +22,11 @@ mac-rel-css: - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh +mac-nightly: + - ./mach build --release + - ./mach package --release + - ./etc/ci/upload_nightly.sh mac + linux-dev: - ./mach test-tidy --no-progress --all - ./mach test-tidy --no-progress --self-test @@ -45,6 +50,11 @@ linux-rel: - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh +linux-nightly: + - ./mach build --release + - ./mach package --release + - ./etc/ci/upload_nightly.sh linux + android: - ./mach build --android --dev - ./mach package --android --dev @@ -55,7 +65,7 @@ android: android-nightly: - ./mach build --android --release - ./mach package --android --release - - s3cmd put target/arm-linux-androideabi/release/servo.apk s3://servo-rust/nightly/servo.apk + - ./etc/ci/upload_nightly.sh android arm32: - ./mach build --rel --target=arm-unknown-linux-gnueabihf