From 3046c8663b44f2520833b57bc0849c26127c845a Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 11 Jan 2017 19:56:44 -0500 Subject: [PATCH 1/6] Remove obsolete Homebrew code This information about the Homebrew user was needed for a workaround which involved invoking Homebrew directly, since Homebrew cannot be run as root. However, the improved Homebrew versions support means this workaround (to installing autoconf and autoconf@2.13 concurrently) is no longer necessary and was removed in commit a0632484043aba1cbc1a0bb6ce45420f5a090d7d. This code was inadvertently left dangling, and is now cleaned up. --- common/map.jinja | 10 ---------- servo-build-dependencies/init.sls | 2 -- 2 files changed, 12 deletions(-) diff --git a/common/map.jinja b/common/map.jinja index b2b231d2..c4dbd61d 100644 --- a/common/map.jinja +++ b/common/map.jinja @@ -38,13 +38,3 @@ grain='kernel' ) %} -{% - set homebrew = salt['grains.filter_by']({ - 'defaults': {}, - 'Darwin': { 'user': 'administrator' } - }, - base='defaults', - merge=salt.pillar.get('homebrew', {}), - grain='kernel' - ) -%} diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index 1c5ce789..b3cabfd4 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -1,5 +1,3 @@ -{% from 'common/map.jinja' import homebrew %} - servo-dependencies: pkg.installed: - pkgs: From 95ab185a7be4cc5f245ea2c925ecf7185c0cf84a Mon Sep 17 00:00:00 2001 From: Daroc Alden Date: Tue, 10 May 2016 18:37:35 -0400 Subject: [PATCH 2/6] Fully manage /etc/hosts Fully managing the contents of the whole hosts file ensures that unused or obsolete hosts entries are removed promptly. Since we have setup DNS for our build machines, remove the relevant static entries from the hosts file. --- common/files/hosts | 3 +++ common/init.sls | 16 ++++++++++------ common/map.jinja | 9 --------- 3 files changed, 13 insertions(+), 15 deletions(-) create mode 100644 common/files/hosts diff --git a/common/files/hosts b/common/files/hosts new file mode 100644 index 00000000..55eb3dc3 --- /dev/null +++ b/common/files/hosts @@ -0,0 +1,3 @@ +127.0.0.1 localhost +::1 localhost +255.255.255.255 broadcasthost diff --git a/common/init.sls b/common/init.sls index 6210a870..0eb4ed17 100644 --- a/common/init.sls +++ b/common/init.sls @@ -50,12 +50,16 @@ UTC: timezone.system {% endif %} -{% for hostname, ip in common.hosts.items() %} -host-{{ hostname }}: - host.present: - - name: {{ hostname }} - - ip: {{ ip }} -{% endfor %} +/etc/hosts: + file.managed: + - user: root + {% if grains['os'] == 'MacOS' %} + - group: wheel + {% elif grains['os'] == 'Ubuntu' %} + - group: root + {% endif %} + - mode: 644 + - source: salt://{{ tpldir }}/files/hosts {% for ssh_user in common.ssh_users %} sshkey-{{ ssh_user }}: diff --git a/common/map.jinja b/common/map.jinja index c4dbd61d..804243e4 100644 --- a/common/map.jinja +++ b/common/map.jinja @@ -2,15 +2,6 @@ set common = salt['grains.filter_by']({ 'defaults': { 'servo_home': '/home/servo', - 'hosts': { - 'servo-master': '96.126.125.232', - 'servo-linux1': '66.228.48.56', - 'servo-linux2': '173.255.201.95', - 'servo-linux3': '45.79.167.177', - 'servo-linux-android1': '72.14.176.110', - 'servo-mac1': '208.52.161.130', - 'servo-mac3': '63.135.170.19', - }, 'ssh_users': [ 'ajeffrey', 'aneeshusa', From 856c6f5d3200c00c0956095989259d4480dafc29 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 11 Jan 2017 19:32:49 -0500 Subject: [PATCH 3/6] Move hosts and SSH key states to admin These states should only be run for fully managed nodes, e.g. those under our administrative control, so move them to the admin/init.sls file. This is part of a refactor making it possible to run the states installing Salt build dependencies without making other system level changes, for use in mach bootstrap. Use the `os` grain instead of the `kernel` grain for targeting, for slightly better semantics. --- {common => admin}/files/hosts | 0 {common => admin/files}/ssh/Ms2ger.pub | 0 {common => admin/files}/ssh/ajeffrey.pub | 0 {common => admin/files}/ssh/aneeshusa.pub | 0 {common => admin/files}/ssh/avadacatavra.pub | 0 {common => admin/files}/ssh/edunham.pub | 0 {common => admin/files}/ssh/emilio.pub | 0 {common => admin/files}/ssh/gw.pub | 0 {common => admin/files}/ssh/jack.pub | 0 {common => admin/files}/ssh/jdm.pub | 0 {common => admin/files}/ssh/larsberg.pub | 0 {common => admin/files}/ssh/manishearth.pub | 0 {common => admin/files}/ssh/mbrubeck.pub | 0 {common => admin/files}/ssh/mwu.pub | 0 {common => admin/files}/ssh/notriddle.pub | 0 {common => admin/files}/ssh/nox.pub | 0 {common => admin/files}/ssh/pcwalton.pub | 0 {common => admin/files}/ssh/simonsapin.pub | 0 admin/init.sls | 25 ++++++++++++++++++++ admin/map.jinja | 22 +++++++++++++++++ common/init.sls | 23 ------------------ common/map.jinja | 22 ++--------------- 22 files changed, 49 insertions(+), 43 deletions(-) rename {common => admin}/files/hosts (100%) rename {common => admin/files}/ssh/Ms2ger.pub (100%) rename {common => admin/files}/ssh/ajeffrey.pub (100%) rename {common => admin/files}/ssh/aneeshusa.pub (100%) rename {common => admin/files}/ssh/avadacatavra.pub (100%) rename {common => admin/files}/ssh/edunham.pub (100%) rename {common => admin/files}/ssh/emilio.pub (100%) rename {common => admin/files}/ssh/gw.pub (100%) rename {common => admin/files}/ssh/jack.pub (100%) rename {common => admin/files}/ssh/jdm.pub (100%) rename {common => admin/files}/ssh/larsberg.pub (100%) rename {common => admin/files}/ssh/manishearth.pub (100%) rename {common => admin/files}/ssh/mbrubeck.pub (100%) rename {common => admin/files}/ssh/mwu.pub (100%) rename {common => admin/files}/ssh/notriddle.pub (100%) rename {common => admin/files}/ssh/nox.pub (100%) rename {common => admin/files}/ssh/pcwalton.pub (100%) rename {common => admin/files}/ssh/simonsapin.pub (100%) create mode 100644 admin/map.jinja diff --git a/common/files/hosts b/admin/files/hosts similarity index 100% rename from common/files/hosts rename to admin/files/hosts diff --git a/common/ssh/Ms2ger.pub b/admin/files/ssh/Ms2ger.pub similarity index 100% rename from common/ssh/Ms2ger.pub rename to admin/files/ssh/Ms2ger.pub diff --git a/common/ssh/ajeffrey.pub b/admin/files/ssh/ajeffrey.pub similarity index 100% rename from common/ssh/ajeffrey.pub rename to admin/files/ssh/ajeffrey.pub diff --git a/common/ssh/aneeshusa.pub b/admin/files/ssh/aneeshusa.pub similarity index 100% rename from common/ssh/aneeshusa.pub rename to admin/files/ssh/aneeshusa.pub diff --git a/common/ssh/avadacatavra.pub b/admin/files/ssh/avadacatavra.pub similarity index 100% rename from common/ssh/avadacatavra.pub rename to admin/files/ssh/avadacatavra.pub diff --git a/common/ssh/edunham.pub b/admin/files/ssh/edunham.pub similarity index 100% rename from common/ssh/edunham.pub rename to admin/files/ssh/edunham.pub diff --git a/common/ssh/emilio.pub b/admin/files/ssh/emilio.pub similarity index 100% rename from common/ssh/emilio.pub rename to admin/files/ssh/emilio.pub diff --git a/common/ssh/gw.pub b/admin/files/ssh/gw.pub similarity index 100% rename from common/ssh/gw.pub rename to admin/files/ssh/gw.pub diff --git a/common/ssh/jack.pub b/admin/files/ssh/jack.pub similarity index 100% rename from common/ssh/jack.pub rename to admin/files/ssh/jack.pub diff --git a/common/ssh/jdm.pub b/admin/files/ssh/jdm.pub similarity index 100% rename from common/ssh/jdm.pub rename to admin/files/ssh/jdm.pub diff --git a/common/ssh/larsberg.pub b/admin/files/ssh/larsberg.pub similarity index 100% rename from common/ssh/larsberg.pub rename to admin/files/ssh/larsberg.pub diff --git a/common/ssh/manishearth.pub b/admin/files/ssh/manishearth.pub similarity index 100% rename from common/ssh/manishearth.pub rename to admin/files/ssh/manishearth.pub diff --git a/common/ssh/mbrubeck.pub b/admin/files/ssh/mbrubeck.pub similarity index 100% rename from common/ssh/mbrubeck.pub rename to admin/files/ssh/mbrubeck.pub diff --git a/common/ssh/mwu.pub b/admin/files/ssh/mwu.pub similarity index 100% rename from common/ssh/mwu.pub rename to admin/files/ssh/mwu.pub diff --git a/common/ssh/notriddle.pub b/admin/files/ssh/notriddle.pub similarity index 100% rename from common/ssh/notriddle.pub rename to admin/files/ssh/notriddle.pub diff --git a/common/ssh/nox.pub b/admin/files/ssh/nox.pub similarity index 100% rename from common/ssh/nox.pub rename to admin/files/ssh/nox.pub diff --git a/common/ssh/pcwalton.pub b/admin/files/ssh/pcwalton.pub similarity index 100% rename from common/ssh/pcwalton.pub rename to admin/files/ssh/pcwalton.pub diff --git a/common/ssh/simonsapin.pub b/admin/files/ssh/simonsapin.pub similarity index 100% rename from common/ssh/simonsapin.pub rename to admin/files/ssh/simonsapin.pub diff --git a/admin/init.sls b/admin/init.sls index f11cf433..e9a8f5a4 100644 --- a/admin/init.sls +++ b/admin/init.sls @@ -1,3 +1,5 @@ +{% from tpldir ~ '/map.jinja' import admin %} + admin-packages: pkg.installed: - pkgs: @@ -8,3 +10,26 @@ admin-packages: {% else %} - mobile-shell {% endif %} + +{% if grains['os'] != 'MacOS' %} +UTC: + timezone.system +{% endif %} + +/etc/hosts: + file.managed: + - user: root + {% if grains['os'] == 'MacOS' %} + - group: wheel + {% elif grains['os'] == 'Ubuntu' %} + - group: root + {% endif %} + - mode: 644 + - source: salt://{{ tpldir }}/files/hosts + +{% for ssh_user in admin.ssh_users %} +sshkey-{{ ssh_user }}: + ssh_auth.present: + - user: root + - source: salt://{{ tpldir }}/files/ssh/{{ ssh_user }}.pub +{% endfor %} diff --git a/admin/map.jinja b/admin/map.jinja new file mode 100644 index 00000000..3d1ea2e4 --- /dev/null +++ b/admin/map.jinja @@ -0,0 +1,22 @@ +{% + set admin = { + 'ssh_users': [ + 'ajeffrey', + 'aneeshusa', + 'avadacatavra', + 'edunham', + 'emilio', + 'gw', + 'jack', + 'jdm', + 'larsberg', + 'manishearth', + 'mbrubeck', + 'Ms2ger', + 'mwu', + 'notriddle', + 'pcwalton', + 'simonsapin' + ] + } +%} diff --git a/common/init.sls b/common/init.sls index 0eb4ed17..db62b1c9 100644 --- a/common/init.sls +++ b/common/init.sls @@ -44,26 +44,3 @@ servo: - fullname: Tom Servo - shell: /bin/bash - home: {{ common.servo_home }} - -{% if grains['os'] != 'MacOS' %} -UTC: - timezone.system -{% endif %} - -/etc/hosts: - file.managed: - - user: root - {% if grains['os'] == 'MacOS' %} - - group: wheel - {% elif grains['os'] == 'Ubuntu' %} - - group: root - {% endif %} - - mode: 644 - - source: salt://{{ tpldir }}/files/hosts - -{% for ssh_user in common.ssh_users %} -sshkey-{{ ssh_user }}: - ssh_auth.present: - - user: root - - source: salt://{{ tpldir }}/ssh/{{ ssh_user }}.pub -{% endfor %} diff --git a/common/map.jinja b/common/map.jinja index 804243e4..7addfe9b 100644 --- a/common/map.jinja +++ b/common/map.jinja @@ -2,30 +2,12 @@ set common = salt['grains.filter_by']({ 'defaults': { 'servo_home': '/home/servo', - 'ssh_users': [ - 'ajeffrey', - 'aneeshusa', - 'avadacatavra', - 'edunham', - 'emilio', - 'gw', - 'jack', - 'jdm', - 'larsberg', - 'manishearth', - 'mbrubeck', - 'Ms2ger', - 'mwu', - 'notriddle', - 'pcwalton', - 'simonsapin' - ] }, - 'Darwin': { + 'MacOS': { 'servo_home': '/Users/servo' } }, base='defaults', - grain='kernel' + grain='os' ) %} From 524a112d2662afa798e21b79c42bf0e54396420c Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 11 Jan 2017 19:53:20 -0500 Subject: [PATCH 4/6] Factor out python states, include sls as needed Move the Python (2 and 3) related states into a separate SLS file, so they can be reused without pulling in the servo user state, which is in common. Instead of adding this new SLS file to the top file, include it from other SLS files that depend on a Python installation. This makes it possible to `state.apply servo-build-dependencies` independently, with is needed for `mach bootstrap`. Note: No effort was made to separate Python 2 and Python 3, as the virtualenv pip package will create executables for both. --- buildbot/master/init.sls | 3 +++ buildbot/slave/init.sls | 3 +++ common/init.sls | 39 ---------------------------- homu/init.sls | 3 +++ intermittent-tracker/init.sls | 3 +++ python/init.sls | 38 +++++++++++++++++++++++++++ servo-build-dependencies/android.sls | 3 +++ servo-build-dependencies/init.sls | 3 +++ 8 files changed, 56 insertions(+), 39 deletions(-) create mode 100644 python/init.sls diff --git a/buildbot/master/init.sls b/buildbot/master/init.sls index 97a6b6e4..19285a9e 100644 --- a/buildbot/master/init.sls +++ b/buildbot/master/init.sls @@ -1,5 +1,8 @@ {% from 'common/map.jinja' import common %} +include: + - python + buildbot-master: pip.installed: - pkgs: diff --git a/buildbot/slave/init.sls b/buildbot/slave/init.sls index 6faf2e69..2263d6d9 100644 --- a/buildbot/slave/init.sls +++ b/buildbot/slave/init.sls @@ -1,5 +1,8 @@ {% from 'common/map.jinja' import common %} +include: + - python + buildbot-slave-dependencies: pip.installed: - pkgs: diff --git a/common/init.sls b/common/init.sls index db62b1c9..fccc2436 100644 --- a/common/init.sls +++ b/common/init.sls @@ -1,44 +1,5 @@ {% from tpldir ~ '/map.jinja' import common %} -python2: - pkg.installed: - - pkgs: - - python - {% if grains['os'] == 'MacOS' %} - - refresh: True - {% endif %} - -python3: - pkg.installed: - - pkgs: - - python3 - -{% if grains['os'] == 'Ubuntu' %} -python2-dev: - pkg.installed: - - pkgs: - - python-dev -{% endif %} - -pip: - pkg.installed: - - pkgs: - {% if grains['os'] == 'Ubuntu' %} - - python-pip - {% elif grains['os'] == 'MacOS' %} - - python # pip is included with python in homebrew - {% endif %} - - reload_modules: True - -# virtualenv == 14.0.6 package creates virtualenv and virtualenv-3.5 executables -# note that the version of the second may change between virtualenv versions -virtualenv: - pip.installed: - - pkgs: - - virtualenv == 14.0.6 - - require: - - pkg: pip - servo: user.present: - fullname: Tom Servo diff --git a/homu/init.sls b/homu/init.sls index e77720fb..d4369597 100644 --- a/homu/init.sls +++ b/homu/init.sls @@ -1,5 +1,8 @@ {% from tpldir ~ '/map.jinja' import homu %} +include: + - python + homu-debugging-packages: pkg.installed: - pkgs: diff --git a/intermittent-tracker/init.sls b/intermittent-tracker/init.sls index 2fa56c3d..77b45dd0 100644 --- a/intermittent-tracker/init.sls +++ b/intermittent-tracker/init.sls @@ -1,5 +1,8 @@ {% from tpldir ~ '/map.jinja' import tracker %} +include: + - python + tracker-debugging-packages: pip.installed: - pkgs: diff --git a/python/init.sls b/python/init.sls new file mode 100644 index 00000000..929f4b2f --- /dev/null +++ b/python/init.sls @@ -0,0 +1,38 @@ +python2: + pkg.installed: + - pkgs: + - python + {% if grains['os'] == 'MacOS' %} + - refresh: True + {% endif %} + +python3: + pkg.installed: + - pkgs: + - python3 + +{% if grains['os'] == 'Ubuntu' %} +python2-dev: + pkg.installed: + - pkgs: + - python-dev +{% endif %} + +pip: + pkg.installed: + - pkgs: + {% if grains['os'] == 'Ubuntu' %} + - python-pip + {% elif grains['os'] == 'MacOS' %} + - python # pip is included with python in homebrew + {% endif %} + - reload_modules: True + +# virtualenv == 14.0.6 package creates virtualenv and virtualenv-3.5 executables +# note that the version of the second may change between virtualenv versions +virtualenv: + pip.installed: + - pkgs: + - virtualenv == 14.0.6 + - require: + - pkg: pip diff --git a/servo-build-dependencies/android.sls b/servo-build-dependencies/android.sls index 1c855714..75bf9010 100644 --- a/servo-build-dependencies/android.sls +++ b/servo-build-dependencies/android.sls @@ -1,6 +1,9 @@ {% from 'common/map.jinja' import common %} {% from tpldir ~ '/map.jinja' import android %} +include: + - python + {% if '64' in grains['cpuarch'] %} enable-i386-architecture: file.managed: diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index b3cabfd4..fc70ba68 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -1,3 +1,6 @@ +include: + - python + servo-dependencies: pkg.installed: - pkgs: From 636fc0e9a07623bb103634a14e28ac91baf9609b Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 11 Jan 2017 20:44:18 -0500 Subject: [PATCH 5/6] Only reference sources.list.d on fully managed nodes The reference to /etc/apt/sources.list.d allows cleaning that directory of extraneous sources files. This should only be done on fully managed nodes, so guard this requisite with a pillar check. The check defaults to True for backwards compatibility (i.e. not having to change the production pillars). --- servo-build-dependencies/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index fc70ba68..873c2afd 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -55,8 +55,10 @@ multiverse: file.exists: - require: - pkgrepo: multiverse + {% if salt['pillar.get']('fully_managed', True) %} - require_in: - file: /etc/apt/sources.list.d + {% endif %} ttf-mscorefonts-installer: debconf.set: From ef029a88c02149ee9968bc04206e5e5e43f4ad4c Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Thu, 12 Jan 2017 14:31:20 -0500 Subject: [PATCH 6/6] Sort Servo build dependency packages This will make it easier to maintain these lists. Additionally, be more strict about matching on OS type to make it easier to support more OSes in the future. --- servo-build-dependencies/init.sls | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index 873c2afd..1404dd3e 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -4,36 +4,36 @@ include: servo-dependencies: pkg.installed: - pkgs: + - ccache - cmake - git - - ccache - {% if grains['kernel'] == 'Darwin' %} + {% if grains['os'] == 'MacOS' %} - autoconf@2.13 - automake - - pkg-config - - openssl - - freetype - ffmpeg + - freetype + - openssl + - pkg-config - yasm - {% else %} - - libglib2.0-dev - - libgl1-mesa-dri - - libgles2-mesa-dev + {% elif grains['os'] == 'Ubuntu' %} + - autoconf2.13 - freeglut3-dev + - gperf + - libavcodec-dev + - libavformat-dev + - libavutil-dev + - libbz2-dev + - libdbus-glib-1-dev - libfreetype6-dev - - xorg-dev + - libgl1-mesa-dri + - libglib2.0-dev + - libgles2-mesa-dev + - libosmesa6-dev - libssl-dev - - libbz2-dev + - xorg-dev + - xpra - xserver-xorg-input-void - xserver-xorg-video-dummy - - xpra - - libosmesa6-dev - - gperf - - autoconf2.13 - - libdbus-glib-1-dev - - libavformat-dev - - libavcodec-dev - - libavutil-dev {% endif %} pip.installed: - pkgs: