From e2a8efc5eac2b0fe80e43dc75d795f3e6dfc20e2 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Tue, 3 Nov 2015 17:08:49 -0500 Subject: [PATCH 1/3] Standardize python setup, ensure homu gets python3 - Trusty comes with python3, pip and virtualenv by default, but write states defensively to install these dependencies as required and require the pip or virtualenv states as used by other states. - Homu requires a python3 toolchain, use virtualenv-3.4 and pass python: python3 to get the correct setup (including pip3 inside the venv). - Update the style guide example to use pkg.installed instead of pip.installed to avoid muddling it with a require: pip. Fixes #142. --- STYLE_GUIDE.md | 12 +++++------ android-dependencies.sls | 5 ++++- buildbot/master.sls | 38 ++++++++++++++------------------- buildbot/slave.sls | 4 +++- common/init.sls | 31 +++++++++++++++++++++++++-- homu/init.sls | 46 +++++++++++++++++++--------------------- servo-dependencies.sls | 4 +++- 7 files changed, 83 insertions(+), 57 deletions(-) diff --git a/STYLE_GUIDE.md b/STYLE_GUIDE.md index 141b0ced..015661a3 100644 --- a/STYLE_GUIDE.md +++ b/STYLE_GUIDE.md @@ -19,16 +19,16 @@ regardless of the number of packages. *Unsafe*: ```salt -buildbot-dependencies: - pip.installed: - - name: buildbot == 0.8.12 +essential-dependencies: + pkg.installed: + - name: cowsay ``` *Better*: ```salt -buildbot-dependencies: - pip.installed: +essential-dependencies: + pkg.installed: - pkgs: - - buildbot == 0.8.12 + - cowsay ``` diff --git a/android-dependencies.sls b/android-dependencies.sls index c02f9ddb..127fa32e 100644 --- a/android-dependencies.sls +++ b/android-dependencies.sls @@ -23,7 +23,10 @@ android-dependencies: - libgl1-mesa-dev - refresh: True pip.installed: - - name: s3cmd + - pkgs: + - s3cmd + - require: + - pkg: pip android-sdk: archive.extracted: diff --git a/buildbot/master.sls b/buildbot/master.sls index 481bc37c..22a73925 100644 --- a/buildbot/master.sls +++ b/buildbot/master.sls @@ -1,20 +1,19 @@ -buildbot: +buildbot-master: pip.installed: - pkgs: - buildbot == 0.8.12 - service_identity == 14.0.0 - -txgithub: - pip.installed - -boto: - pip.installed - -buildbot-master: + - txgithub == 15.0.0 + - boto == 2.38.0 + - require: + - pkg: pip service.running: - enable: True - require: - - pip: buildbot + - pip: buildbot-master + - watch: + - file: /home/servo/buildbot/master + - file: /etc/init/buildbot-master.conf /home/servo/buildbot/master: file.recurse: @@ -24,8 +23,6 @@ buildbot-master: - group: servo - dir_mode: 755 - file_mode: 644 - - watch_in: - - service: buildbot-master /etc/init/buildbot-master.conf: file.managed: @@ -33,12 +30,6 @@ buildbot-master: - user: root - group: root - mode: 644 - - watch_in: - - service: buildbot-master - -buildbot-github-listener: - service.running: - - enable: True /usr/local/bin/github_buildbot.py: file.managed: @@ -46,8 +37,6 @@ buildbot-github-listener: - user: root - group: root - mode: 755 - - watch_in: - - service: buildbot-github-listener /etc/init/buildbot-github-listener.conf: file.managed: @@ -56,5 +45,10 @@ buildbot-github-listener: - user: root - group: root - mode: 644 - - watch_in: - - service: buildbot-github-listener + +buildbot-github-listener: + service.running: + - enable: True + - watch: + - file: /usr/local/bin/github_buildbot.py + - file: /etc/init/buildbot-github-listener.conf diff --git a/buildbot/slave.sls b/buildbot/slave.sls index 83a6e9f2..f8f4ae08 100644 --- a/buildbot/slave.sls +++ b/buildbot/slave.sls @@ -1,9 +1,11 @@ {% from 'common/map.jinja' import config as common with context %} -buildbot-slave.pip: +buildbot-slave-dependencies: pip.installed: - pkgs: - buildbot-slave == 0.8.12 + - require: + - pkg: pip {{ common.servo_home }}/buildbot/slave: file.recurse: diff --git a/common/init.sls b/common/init.sls index 7aa3d317..e67190b9 100644 --- a/common/init.sls +++ b/common/init.sls @@ -1,11 +1,38 @@ {% from tpldir ~ '/map.jinja' import config with context %} {% if grains['kernel'] != 'Darwin' %} -python-pkgs: +# Ubuntu has python2 as default python +python2: pkg.installed: - pkgs: - - python-pip + - python - python-dev + +python3: + pkg.installed: + - pkgs: + - python3 + +# Ensure pip is default by purging pip3 +pip: + pkg.installed: + - pkgs: + - python-pip + - reload_modules: True + +pip3: + pkg.purged: + - pkgs: + - python3-pip + +# Virtualenv package creates virtualenv and virtualenv-3.4 executables +virtualenv: + pip.installed: + - pkgs: + - virtualenv + - require: + - pkg: pip + - pkg: pip3 {% endif %} servo: diff --git a/homu/init.sls b/homu/init.sls index 74fdaaf7..fe79b8d7 100644 --- a/homu/init.sls +++ b/homu/init.sls @@ -1,38 +1,38 @@ -https://github.com/servo/homu: +homu: git.latest: + - name: https://github.com/servo/homu - rev: e07f74e7a3a185768e71639d6a328ef8ea234f92 - target: /home/servo/homu - user: servo - - require_in: - - pip: install_homu - -/home/servo/homu/cfg.toml: - file.managed: - - source: salt://homu/cfg.toml - - template: jinja - - user: servo - - group: servo - - mode: 644 - - watch_in: - - service: homu - -/home/servo/homu/_venv: virtualenv.managed: + - name: /home/servo/homu/_venv - venv_bin: virtualenv-3.4 + - python: python3 - system_site_packages: False - - require_in: - - pip: install_homu - -install_homu: + - require: + - pkg: python3 + - pip: virtualenv pip.installed: - bin_env: /home/servo/homu/_venv - editable: /home/servo/homu - -homu: + - require: + - git: homu + - virtualenv: /home/servo/homu/_venv service.running: - enable: True - require: - - pip: install_homu + - pip: homu + - watch: + - file: /home/servo/homu/cfg.toml + - file: /etc/init/homu.conf + +/home/servo/homu/cfg.toml: + file.managed: + - source: salt://homu/cfg.toml + - template: jinja + - user: servo + - group: servo + - mode: 644 /etc/init/homu.conf: file.managed: @@ -40,5 +40,3 @@ homu: - user: root - group: root - mode: 644 - - watch_in: - - service: homu diff --git a/servo-dependencies.sls b/servo-dependencies.sls index ec7d70d7..e412782e 100644 --- a/servo-dependencies.sls +++ b/servo-dependencies.sls @@ -28,8 +28,10 @@ servo-dependencies: {% endif %} pip.installed: - pkgs: - - virtualenv - ghp-import + - require: + - pkg: pip + - pip: virtualenv {% if grains['kernel'] == 'Darwin' %} # Workaround for https://github.com/saltstack/salt/issues/26414 From b4e4b521a3c93339e28dc0032aa4101f6673384b Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Tue, 3 Nov 2015 22:29:06 -0500 Subject: [PATCH 2/3] Install python, pip, virtualenv on OS X --- common/init.sls | 21 ++++++++------------- homu/init.sls | 5 +++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/common/init.sls b/common/init.sls index e67190b9..25a353eb 100644 --- a/common/init.sls +++ b/common/init.sls @@ -1,30 +1,27 @@ {% from tpldir ~ '/map.jinja' import config with context %} -{% if grains['kernel'] != 'Darwin' %} -# Ubuntu has python2 as default python python2: pkg.installed: - pkgs: - python - - python-dev -python3: +{% if grains['os'] == 'Ubuntu' %} +python2-dev: pkg.installed: - pkgs: - - python3 + - python-dev +{% endif %} -# Ensure pip is default by purging pip3 pip: pkg.installed: - pkgs: + {% if grains['os'] == 'Ubuntu' %} - python-pip + {% elif grains['os'] == 'MacOS' %} + - python # pip is included with python in homebrew + {% endif %} - reload_modules: True -pip3: - pkg.purged: - - pkgs: - - python3-pip - # Virtualenv package creates virtualenv and virtualenv-3.4 executables virtualenv: pip.installed: @@ -32,8 +29,6 @@ virtualenv: - virtualenv - require: - pkg: pip - - pkg: pip3 -{% endif %} servo: user.present: diff --git a/homu/init.sls b/homu/init.sls index fe79b8d7..01ce440c 100644 --- a/homu/init.sls +++ b/homu/init.sls @@ -1,3 +1,8 @@ +python3: + pkg.installed: + - pkgs: + - python3 + homu: git.latest: - name: https://github.com/servo/homu From a3baeaa481eed4bbac666d23950aea5dd4f9e70a Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 4 Nov 2015 19:30:55 -0500 Subject: [PATCH 3/3] Restart services on package change servo-master is updated in-place instead of a fresh machine being built for every update, so this ensures services are restarted when the underlying code is updated. --- buildbot/master.sls | 3 +-- nginx/init.sls | 2 +- xvfb/init.sls | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/buildbot/master.sls b/buildbot/master.sls index 22a73925..8cc9e38a 100644 --- a/buildbot/master.sls +++ b/buildbot/master.sls @@ -9,9 +9,8 @@ buildbot-master: - pkg: pip service.running: - enable: True - - require: - - pip: buildbot-master - watch: + - pip: buildbot-master - file: /home/servo/buildbot/master - file: /etc/init/buildbot-master.conf diff --git a/nginx/init.sls b/nginx/init.sls index 5138f4cd..94ede988 100644 --- a/nginx/init.sls +++ b/nginx/init.sls @@ -2,7 +2,7 @@ nginx: pkg.installed: [] service.running: - enable: True - - require: + - watch: - pkg: nginx /etc/nginx/sites-available/default: diff --git a/xvfb/init.sls b/xvfb/init.sls index 3a844dc5..30dca514 100644 --- a/xvfb/init.sls +++ b/xvfb/init.sls @@ -9,7 +9,6 @@ xvfb: pkg.installed: [] service.running: - enable: True - - require: - - pkg: xvfb - watch: + - pkg: xvfb - file: /etc/init/xvfb.conf