From 281a9e39d65248a0e681b81086aa45330d1892cc Mon Sep 17 00:00:00 2001 From: "E. Dunham" Date: Fri, 24 Jun 2016 11:00:04 -0700 Subject: [PATCH 1/5] Add schedulers for Linux, OSX nightlies Windows should wait on factories.py refactor to fix:: windows = ServoFactory([ # TODO: convert this to use DynamicServoFactory # We need to run each command in a bash login shell, which breaks the # heuristics used by DynamicServoFactory.make_step steps.Compile(command=make_win_command("./mach build -d -v"), env=envs.build_windows), steps.Test(command=make_win_command("./mach test-unit"), env=envs.build_windows), # TODO: run lockfile_changed.sh and manifest_changed.sh scripts ]) --- buildbot/master/files/config/master.cfg | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/buildbot/master/files/config/master.cfg b/buildbot/master/files/config/master.cfg index da2b0436..1ac5a66f 100644 --- a/buildbot/master/files/config/master.cfg +++ b/buildbot/master/files/config/master.cfg @@ -85,23 +85,29 @@ c['schedulers'].append(schedulers.SingleBranchScheduler( c['schedulers'].append(schedulers.ForceScheduler( name="force", builderNames=[ + "android", + "android-nightly", + "arm32", + "arm64", "linux-dev", - "linux-rel", "linux-dev-yaml", - "mac-rel-wpt", + "linux-nightly", + "linux-rel", "mac-dev-unit", + "mac-nightly", "mac-rel-css", - "android", - "arm32", - "arm64", - "android-nightly", + "mac-rel-wpt", "windows", ], )) c['schedulers'].append(schedulers.Nightly( name="Nightly", branch="master", - builderNames=["android-nightly"], + builderNames=[ + "android-nightly", + "linux-nightly", + "mac-nightly" + ], hour=1, minute=0, )) From 0516df17ef91d4113baa2eb05df343c034105a7e Mon Sep 17 00:00:00 2001 From: "E. Dunham" Date: Fri, 24 Jun 2016 15:11:44 -0700 Subject: [PATCH 2/5] add build steps for nightlies --- buildbot/master/files/config/steps.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/buildbot/master/files/config/steps.yml b/buildbot/master/files/config/steps.yml index 7668b3c2..8ad38ec8 100644 --- a/buildbot/master/files/config/steps.yml +++ b/buildbot/master/files/config/steps.yml @@ -22,6 +22,12 @@ mac-rel-css: - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh +mac-nightly: + - ./mach build --release + - ./mach package --release + - s3cmd -c ~/.s3cfg-servo put target/release/*.tar.gz s3://servo-developer-preview/nightly/`date +%Y`/ + - rm -rf target/ + linux-dev: - ./mach test-tidy --no-progress --all - ./mach test-tidy --no-progress --self-test @@ -45,6 +51,12 @@ linux-rel: - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh +linux-nightly: + - ./mach build --release + - ./mach package --release + - s3cmd -c ~/.s3cfg-servo put target/release/*.tar.gz s3://servo-developer-preview/nightly/`date +%Y`/ + - rm -rf target/ + android: - ./mach build --android --dev - ./mach package --android --dev From 72f805f39e2a55e880f19fea484920586e512114 Mon Sep 17 00:00:00 2001 From: "E. Dunham" Date: Mon, 27 Jun 2016 20:35:21 -0700 Subject: [PATCH 3/5] Install and configure s3cmd on linux and osx slaves, for nightlies Already added the credentials to /srv/pillar on the master. --- nightly/files/s3cfg-rust | 75 +++++++++++++++++++++++++++++++++++++++ nightly/files/s3cfg-servo | 75 +++++++++++++++++++++++++++++++++++++++ nightly/init.sls | 21 +++++++++++ top.sls | 6 ++-- 4 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 nightly/files/s3cfg-rust create mode 100644 nightly/files/s3cfg-servo create mode 100644 nightly/init.sls diff --git a/nightly/files/s3cfg-rust b/nightly/files/s3cfg-rust new file mode 100644 index 00000000..60006460 --- /dev/null +++ b/nightly/files/s3cfg-rust @@ -0,0 +1,75 @@ +[default] +access_key = {{ pillar['nightly']['rust_aws_access_key_id'] }} +access_token = +add_encoding_exts = +add_headers = +bucket_location = us-west-2 +ca_certs_file = +cache_file = +check_ssl_certificate = True +check_ssl_hostname = True +cloudfront_host = cloudfront.amazonaws.com +default_mime_type = binary/octet-stream +delay_updates = False +delete_after = False +delete_after_fetch = False +delete_removed = False +dry_run = False +enable_multipart = True +encoding = UTF-8 +encrypt = False +expiry_date = +expiry_days = +expiry_prefix = +follow_symlinks = False +force = False +get_continue = False +gpg_command = /usr/bin/gpg +gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes +--passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes +--passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_passphrase = +guess_mime_type = True +host_base = s3.amazonaws.com +host_bucket = %(bucket)s.s3.amazonaws.com +human_readable_sizes = False +invalidate_default_index_on_cf = False +invalidate_default_index_root_on_cf = True +invalidate_on_cf = False +kms_key = +limitrate = 0 +list_md5 = False +log_target_prefix = +long_listing = False +max_delete = -1 +mime_type = +multipart_chunk_size_mb = 15 +multipart_max_chunks = 10000 +preserve_attrs = True +progress_meter = True +proxy_host = +proxy_port = 0 +put_continue = False +recursive = False +recv_chunk = 65536 +reduced_redundancy = False +requester_pays = False +restore_days = 1 +secret_key = {{ pillar['nightly']['rust_aws_secret_key'] }} +send_chunk = 65536 +server_side_encryption = False +signature_v2 = False +simpledb_host = sdb.amazonaws.com +skip_existing = False +socket_timeout = 300 +stats = False +stop_on_error = False +storage_class = +urlencoding_mode = normal +use_https = True +use_mime_magic = True +verbosity = WARNING +website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/ +website_error = +website_index = index.html diff --git a/nightly/files/s3cfg-servo b/nightly/files/s3cfg-servo new file mode 100644 index 00000000..7878eea4 --- /dev/null +++ b/nightly/files/s3cfg-servo @@ -0,0 +1,75 @@ +[default] +access_key = {{ pillar['nightly']['aws_access_key_id'] }} +access_token = +add_encoding_exts = +add_headers = +bucket_location = us-west-2 +ca_certs_file = +cache_file = +check_ssl_certificate = True +check_ssl_hostname = True +cloudfront_host = cloudfront.amazonaws.com +default_mime_type = binary/octet-stream +delay_updates = False +delete_after = False +delete_after_fetch = False +delete_removed = False +dry_run = False +enable_multipart = True +encoding = UTF-8 +encrypt = False +expiry_date = +expiry_days = +expiry_prefix = +follow_symlinks = False +force = False +get_continue = False +gpg_command = /usr/bin/gpg +gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes +--passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes +--passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_passphrase = +guess_mime_type = True +host_base = s3.amazonaws.com +host_bucket = %(bucket)s.s3.amazonaws.com +human_readable_sizes = False +invalidate_default_index_on_cf = False +invalidate_default_index_root_on_cf = True +invalidate_on_cf = False +kms_key = +limitrate = 0 +list_md5 = False +log_target_prefix = +long_listing = False +max_delete = -1 +mime_type = +multipart_chunk_size_mb = 15 +multipart_max_chunks = 10000 +preserve_attrs = True +progress_meter = True +proxy_host = +proxy_port = 0 +put_continue = False +recursive = False +recv_chunk = 65536 +reduced_redundancy = False +requester_pays = False +restore_days = 1 +secret_key = {{ pillar['nightly']['aws_secret_key'] }} +send_chunk = 65536 +server_side_encryption = False +signature_v2 = False +simpledb_host = sdb.amazonaws.com +skip_existing = False +socket_timeout = 300 +stats = False +stop_on_error = False +storage_class = +urlencoding_mode = normal +use_https = True +use_mime_magic = True +verbosity = WARNING +website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/ +website_error = +website_index = index.html diff --git a/nightly/init.sls b/nightly/init.sls new file mode 100644 index 00000000..f04a8068 --- /dev/null +++ b/nightly/init.sls @@ -0,0 +1,21 @@ +/home/servo/.s3cfg-servo: + file.managed: + - source: salt://{{ tpldir }}/files/s3cfg-servo + - user: servo + - group: servo + - mode: 644 + +/home/servo/.s3cfg: + file.managed: + - source: salt://{{ tpldir }}/files/s3cfg-rust + - user: servo + - group: servo + - mode: 644 + + +s3cmd: + pip.installed: + - pkgs: + - s3cmd + - require: + - pkg: pip diff --git a/top.sls b/top.sls index 1c84eb05..194e901e 100644 --- a/top.sls +++ b/top.sls @@ -18,19 +18,21 @@ base: 'servo-(mac|macpro)\d+': - match: pcre - - osx - buildbot.slave + - nightly + - osx - servo-build-dependencies 'servo-linux\d+': - match: pcre - buildbot.slave + - nightly - servo-build-dependencies - xvfb 'servo-master\d+': - match: pcre - - git - buildbot.master + - git - homu - nginx From 8b5c1bf01a777446ba96036f3c29174184afa5fa Mon Sep 17 00:00:00 2001 From: "E. Dunham" Date: Wed, 29 Jun 2016 11:35:09 -0700 Subject: [PATCH 4/5] Use Servo nightlies bucket for Android --- nightly/files/{s3cfg-servo => s3cfg} | 0 nightly/files/s3cfg-rust | 75 ---------------------------- nightly/init.sls | 9 +--- 3 files changed, 1 insertion(+), 83 deletions(-) rename nightly/files/{s3cfg-servo => s3cfg} (100%) delete mode 100644 nightly/files/s3cfg-rust diff --git a/nightly/files/s3cfg-servo b/nightly/files/s3cfg similarity index 100% rename from nightly/files/s3cfg-servo rename to nightly/files/s3cfg diff --git a/nightly/files/s3cfg-rust b/nightly/files/s3cfg-rust deleted file mode 100644 index 60006460..00000000 --- a/nightly/files/s3cfg-rust +++ /dev/null @@ -1,75 +0,0 @@ -[default] -access_key = {{ pillar['nightly']['rust_aws_access_key_id'] }} -access_token = -add_encoding_exts = -add_headers = -bucket_location = us-west-2 -ca_certs_file = -cache_file = -check_ssl_certificate = True -check_ssl_hostname = True -cloudfront_host = cloudfront.amazonaws.com -default_mime_type = binary/octet-stream -delay_updates = False -delete_after = False -delete_after_fetch = False -delete_removed = False -dry_run = False -enable_multipart = True -encoding = UTF-8 -encrypt = False -expiry_date = -expiry_days = -expiry_prefix = -follow_symlinks = False -force = False -get_continue = False -gpg_command = /usr/bin/gpg -gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes ---passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s -gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes ---passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s -gpg_passphrase = -guess_mime_type = True -host_base = s3.amazonaws.com -host_bucket = %(bucket)s.s3.amazonaws.com -human_readable_sizes = False -invalidate_default_index_on_cf = False -invalidate_default_index_root_on_cf = True -invalidate_on_cf = False -kms_key = -limitrate = 0 -list_md5 = False -log_target_prefix = -long_listing = False -max_delete = -1 -mime_type = -multipart_chunk_size_mb = 15 -multipart_max_chunks = 10000 -preserve_attrs = True -progress_meter = True -proxy_host = -proxy_port = 0 -put_continue = False -recursive = False -recv_chunk = 65536 -reduced_redundancy = False -requester_pays = False -restore_days = 1 -secret_key = {{ pillar['nightly']['rust_aws_secret_key'] }} -send_chunk = 65536 -server_side_encryption = False -signature_v2 = False -simpledb_host = sdb.amazonaws.com -skip_existing = False -socket_timeout = 300 -stats = False -stop_on_error = False -storage_class = -urlencoding_mode = normal -use_https = True -use_mime_magic = True -verbosity = WARNING -website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/ -website_error = -website_index = index.html diff --git a/nightly/init.sls b/nightly/init.sls index f04a8068..1b9565f0 100644 --- a/nightly/init.sls +++ b/nightly/init.sls @@ -1,13 +1,6 @@ -/home/servo/.s3cfg-servo: - file.managed: - - source: salt://{{ tpldir }}/files/s3cfg-servo - - user: servo - - group: servo - - mode: 644 - /home/servo/.s3cfg: file.managed: - - source: salt://{{ tpldir }}/files/s3cfg-rust + - source: salt://{{ tpldir }}/files/s3cfg - user: servo - group: servo - mode: 644 From 9f5f3a7673b2a0a36eb8863dbc0c21ac1c99eb90 Mon Sep 17 00:00:00 2001 From: "E. Dunham" Date: Wed, 29 Jun 2016 11:38:51 -0700 Subject: [PATCH 5/5] Improve nightly upload steps: * Fix directory structure to sort by target and ditch year directory; realistically we probably won't be in "developer preview" in 2-3 years when the # of files starts getting unmanageable * Upload .dmg from correct location on osx * Upload everything to Servo's AWS account --- buildbot/master/files/config/steps.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/buildbot/master/files/config/steps.yml b/buildbot/master/files/config/steps.yml index 8ad38ec8..d5b5f319 100644 --- a/buildbot/master/files/config/steps.yml +++ b/buildbot/master/files/config/steps.yml @@ -25,8 +25,7 @@ mac-rel-css: mac-nightly: - ./mach build --release - ./mach package --release - - s3cmd -c ~/.s3cfg-servo put target/release/*.tar.gz s3://servo-developer-preview/nightly/`date +%Y`/ - - rm -rf target/ + - s3cmd put target/*.dmg s3://servo-developer-preview/nightly/mac/ linux-dev: - ./mach test-tidy --no-progress --all @@ -54,7 +53,7 @@ linux-rel: linux-nightly: - ./mach build --release - ./mach package --release - - s3cmd -c ~/.s3cfg-servo put target/release/*.tar.gz s3://servo-developer-preview/nightly/`date +%Y`/ + - s3cmd -c ~/.s3cfg-servo put target/release/*.tar.gz s3://servo-developer-preview/nightly/linux/ - rm -rf target/ android: @@ -67,7 +66,7 @@ android: android-nightly: - ./mach build --android --release - ./mach package --android --release - - s3cmd put target/arm-linux-androideabi/release/servo.apk s3://servo-rust/nightly/servo.apk + - s3cmd put target/arm-linux-androideabi/release/*.apk s3://servo-developer-preview/nightly/android/ arm32: - ./mach build --rel --target=arm-unknown-linux-gnueabihf