From 37d5a0c636942a8f0c3dc8437e9277955b742e5e Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Wed, 18 Oct 2017 10:18:39 -0400 Subject: [PATCH 1/3] Add intermittent failure tracker service. --- .../files/config.json | 1 + .../files/tracker.conf | 11 +++++ intermittent-failure-tracker/init.sls | 48 +++++++++++++++++++ intermittent-failure-tracker/map.jinja | 5 ++ nginx/default | 3 ++ top.sls | 1 + 6 files changed, 69 insertions(+) create mode 100644 intermittent-failure-tracker/files/config.json create mode 100644 intermittent-failure-tracker/files/tracker.conf create mode 100644 intermittent-failure-tracker/init.sls create mode 100644 intermittent-failure-tracker/map.jinja diff --git a/intermittent-failure-tracker/files/config.json b/intermittent-failure-tracker/files/config.json new file mode 100644 index 00000000..f310c8f6 --- /dev/null +++ b/intermittent-failure-tracker/files/config.json @@ -0,0 +1 @@ +{"port": 5002} diff --git a/intermittent-failure-tracker/files/tracker.conf b/intermittent-failure-tracker/files/tracker.conf new file mode 100644 index 00000000..2ddc03f3 --- /dev/null +++ b/intermittent-failure-tracker/files/tracker.conf @@ -0,0 +1,11 @@ +exec /home/servo/intermittent-failure-tracker/_venv/bin/intermittent_failure_tracker + +setuid servo +setgid servo + +start on (local-filesystems and net-device-up IFACE!=lo) +stop on runlevel [016] + +env HOME=/home/servo + +chdir /home/servo/intermittent-failure-tracker diff --git a/intermittent-failure-tracker/init.sls b/intermittent-failure-tracker/init.sls new file mode 100644 index 00000000..37fcab1e --- /dev/null +++ b/intermittent-failure-tracker/init.sls @@ -0,0 +1,48 @@ +{% from tpldir ~ '/map.jinja' import tracker %} + +include: + - python + +intermittent-failure-tracker: + virtualenv.managed: + - name: /home/servo/intermittent-failure-tracker/_venv + - venv_bin: virtualenv-3.5 + - python: python3 + - system_site_packages: False + - require: + - pkg: python3 + - pip: virtualenv + pip.installed: + - pkgs: + - git+https://github.com/servo/intermittent-failure-tracker@{{ tracker.rev }} + - bin_env: /home/servo/intermittent-failure-tracker/_venv + - require: + - virtualenv: intermittent-failure-tracker + {% if grains.get('virtual_subtype', '') != 'Docker' %} + service.running: + - enable: True + - name: failure-tracker + - require: + - pip: intermittent-failure-tracker + - watch: + - file: /home/servo/intermittent-failure-tracker/config.json + - file: /etc/init/failure-tracker.conf + {% endif %} + +/home/servo/intermittent-failure-tracker/config.json: + file.managed: + - source: salt://{{ tpldir }}/files/config.json + - template: jinja + - user: servo + - group: servo + - mode: 644 + +/etc/init/failure-tracker.conf: + file.managed: + - source: salt://{{ tpldir }}/files/tracker.conf + - user: root + - group: root + - mode: 644 + - require: + - pip: intermittent-failure-tracker + - file: /home/servo/intermittent-failure-tracker/config.json diff --git a/intermittent-failure-tracker/map.jinja b/intermittent-failure-tracker/map.jinja new file mode 100644 index 00000000..bede9bb1 --- /dev/null +++ b/intermittent-failure-tracker/map.jinja @@ -0,0 +1,5 @@ +{% + set tracker = { + 'rev': '37ef73bb0059e0555a903e758dae8006c46a9916' + } +%} diff --git a/nginx/default b/nginx/default index 5d6bd7e4..cf0ca9ca 100644 --- a/nginx/default +++ b/nginx/default @@ -16,5 +16,8 @@ server { location /intermittent-tracker/ { proxy_pass http://localhost:5000/; } + location /intermittent-failure-tracker/ { + proxy_pass http://localhost:5002/; + } } diff --git a/top.sls b/top.sls index 3ce094d8..ddc70724 100644 --- a/top.sls +++ b/top.sls @@ -43,5 +43,6 @@ base: - buildbot.master - homu - intermittent-tracker + - intermittent-failure-tracker - nginx - salt.master From 039fc7049933ee8e523d87551830b189fa55487c Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Wed, 18 Oct 2017 10:35:36 -0400 Subject: [PATCH 2/3] Add build machine name to environment. --- buildbot/master/files/config/environments.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildbot/master/files/config/environments.py b/buildbot/master/files/config/environments.py index 41dc2901..b1e787e8 100644 --- a/buildbot/master/files/config/environments.py +++ b/buildbot/master/files/config/environments.py @@ -1,3 +1,4 @@ +from buildbot.plugins import util from passwords import GITHUB_DOC_TOKEN, GITHUB_HOMEBREW_TOKEN from passwords import S3_UPLOAD_ACCESS_KEY_ID, S3_UPLOAD_SECRET_ACCESS_KEY @@ -49,6 +50,7 @@ def without(self, to_unset): build_common = Environment({ 'RUST_BACKTRACE': '1', + 'BUILD_MACHINE': str(util.Property('slavename')), }) build_windows_msvc = build_common + Environment({ From 04915fad38d9e7a4c965361976dd3936cc4adf74 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Thu, 19 Oct 2017 11:26:47 -0400 Subject: [PATCH 3/3] Add documentation about service. --- docs/intermittent-failure-tracker.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/intermittent-failure-tracker.md diff --git a/docs/intermittent-failure-tracker.md b/docs/intermittent-failure-tracker.md new file mode 100644 index 00000000..1588a91b --- /dev/null +++ b/docs/intermittent-failure-tracker.md @@ -0,0 +1,13 @@ +# Intermittent Failure Tracker service + +This service runs a flask server that serves two purposes: + +1. an API endpoint for Servo's CI builders to report +instances of intermittent failures that are encountered + +2. an HTML frontend to explore trends in +intermittent failures that have been reported. + +See the project's [README](https://github.com/servo/intermittent-failure-tracker/blob/master/README.md) for more details about interacting with the service, +and [this Servo pull request](https://github.com/servo/servo/pull/16946) +for more details about how failures are reported. \ No newline at end of file