From ed6fb33f45a6e71c6ec45745c7a5429734da6586 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Thu, 22 Sep 2016 15:21:41 -0400 Subject: [PATCH 1/9] Add homebrew_analytics module for robust disabling Originally, a raw call to the `git.config` state was used to disable Homebrew Analytics. However, this is fragile and recently broke with the Homebrew 1.0.0 release, which migrated the Homebrew repository location. Now that we have support for custom modules, add a `homebrew` execution module to allow running arbitrary Homebrew commands, and a `homebrew_analytics` module on top which allows robustly disabling Homebrew Analytics by using the builtin `brew analytics` command. The execution module is named `homebrew` because the built-in module [`mac_brew.py`] is not available (hidden behind the `pkg` __virtualname__), so its methods cannot be called. --- _modules/homebrew.py | 89 ++++++++++++++++++++++++ _states/homebrew_analytics.py | 126 ++++++++++++++++++++++++++++++++++ osx/init.sls | 12 +--- 3 files changed, 217 insertions(+), 10 deletions(-) create mode 100644 _modules/homebrew.py create mode 100644 _states/homebrew_analytics.py diff --git a/_modules/homebrew.py b/_modules/homebrew.py new file mode 100644 index 00000000..a7ed7295 --- /dev/null +++ b/_modules/homebrew.py @@ -0,0 +1,89 @@ +# This module is mainly comprised of new code, but also contains modified +# versions of functions from the salt/modules/mac_brew.py module from Salt +# develop (at git revision 3e5218daea73f3f24b82a3078764ccb82c2a1ec9). +# Functions taken/modified from Salt are marked, all others are original. +# +# The original copyright and licensing notice for the methods from the +# mac_brew.py module is reproduced below in the double-# comment block: +# +## Salt - Remote execution system +## +## Copyright 2014-2015 SaltStack Team +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +''' +Module for the management of homebrew +''' +from __future__ import absolute_import + +# Import python libs +import logging +import os + +# Import salt libs +from salt.exceptions import CommandExecutionError +import salt.utils + +# Set up logging +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only work if Homebrew is installed + ''' + if salt.utils.which('brew'): + return True + return ( + False, + 'The homebrew execution module could not be loaded: brew not found' + ) + + +def _homebrew_bin(): + ''' + Returns the full path to the homebrew binary in the PATH. + Taken from mac_brew.py with modifications. + ''' + homebrew_dir = __salt__['cmd.run']( + 'brew --prefix', + output_loglevel='trace' + ) + return os.path.join(homebrew_dir, 'bin', 'brew') + + +def cmd_all(args): + ''' + Calls brew with the specified arguments and as the correct user. + Taken from mac_brew.py with modifications. + + args: + Should be a list of arguments to pass to the `brew` binary. + ''' + user = __salt__['file.get_user'](_homebrew_bin()) + runas = user if user != __opts__['user'] else None + ret = __salt__['cmd.run_all']( + ['brew'] + args, + runas=runas, + output_loglevel='trace', + python_shell=False, + redirect_stderr=False, + ) + if ret['retcode'] != 0: + raise CommandExecutionError( + 'stdout: {stdout}\n' + 'stderr: {stderr}\n' + 'retcode: {retcode}\n'.format(**ret) + ) + return ret diff --git a/_states/homebrew_analytics.py b/_states/homebrew_analytics.py new file mode 100644 index 00000000..476ce111 --- /dev/null +++ b/_states/homebrew_analytics.py @@ -0,0 +1,126 @@ +# This module is mainly comprised of new code, but also contains modified +# versions of functions from the salt/modules/mac_brew.py module from Salt +# develop (at git revision 3e5218daea73f3f24b82a3078764ccb82c2a1ec9). +# Functions taken/modified from Salt are marked, all others are original. +# +# The original copyright and licensing notice for the methods from the +# mac_brew.py module is reproduced below in the double-# comment block: +# +## Salt - Remote execution system +## +## Copyright 2014-2015 SaltStack Team +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +''' +Module for the management of homebrew +''' +from __future__ import absolute_import + +# Import python libs +import logging + +# Import salt libs +from salt.exceptions import CommandExecutionError +import salt.utils + +# Set up logging +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only work if Homebrew is installed + ''' + if salt.utils.which('brew'): + return True + return ( + False, + 'The homebrew_analytics state mod could not be loaded: brew not found' + ) + + +def _check_analytics_status(): + out = __salt__['homebrew.cmd_all'](['analytics', 'state'])['stdout'] + if len(out) < 1: + raise CommandExecutionError('Failed to parse brew analytics state') + status_line = out.splitlines()[0] + if 'enabled' in status_line: + return True + elif 'disabled' in status_line: + return False + raise CommandExecutionError('Failed to parse brew analytics state') + + +def managed(name, **kwargs): + ''' + Manage Homebrew analytics state (either enabled or disabled). + + name + Either 'enabled' or 'disabled' + ''' + ret = { + 'name': name, + 'changes': {}, + 'result': None, + 'comment': '', + } + + # Var must be called 'name' due to design of Salt + wanted = None + if name == 'enabled': + wanted = True + elif name == 'disabled': + wanted = False + else: + ret['result'] = False + ret['comment'] = '`name` parameter must be `enabled` or `disabled`' + return ret + wanted_v = name[:-1] # Verb form + + try: + current = _check_analytics_status() + if current == wanted: + ret['result'] = True + ret['comment'] = 'Homebrew analytics are already {}'.format(name) + return ret + + if __opts__['test']: + ret['comment'] = 'Homebrew analytics need to be {}'.format(name) + return ret + + state_arg = 'on' if wanted else 'off' + # Exception bubbles, so we can ignore the return value + __salt__['homebrew.cmd_all'](['analytics', state_arg]) + + new = _check_analytics_status() + if new == wanted: + ret['changes']['homebrew_analytics'] = { + 'old': 'enabled' if current else 'disabled', + 'new': name, + } + ret['result'] = True + ret['comment'] = 'Homebrew analytics was {}'.format(name) + return ret + else: + ret['result'] = False + ret['comment'] = 'Failed to {} Homebrew analytics'.format(wanted_v) + return ret + + except CommandExecutionError as err: + ret['result'] = False + ret['comment'] = 'Failed to {} Homebrew analytics: {}'.format( + wanted_v, + err + ) + return ret diff --git a/osx/init.sls b/osx/init.sls index 9533bc04..7a2d6ccb 100644 --- a/osx/init.sls +++ b/osx/init.sls @@ -12,14 +12,6 @@ - mode: 644 - source: salt://{{ tpldir }}/files/profile -# Disable Homebrew Analytics -# TODO: wrap this up into a proper state that uses the `brew analytics` command -# instead of directly changing the git configuration -# (requires either upstreaming this state + updating Salt, -# or Salting the Salt master) -# TODO: also ensure the `homebrew.analyticsuuid` setting is unset disable-homebrew-analytics: - git.config: - - name: 'homebrew.analyticsdisabled' - - value: 'true' - - repo: /usr/local/Homebrew + homebrew_analytics.managed: + - name: disabled From 09a0e554d5cba0923d9af8ccb8e73bbcfe3bc0e5 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 23 Sep 2016 05:14:41 -0400 Subject: [PATCH 2/9] Be robust to upstream changes when building old revs When running builds that are not from scratch, it is possible that upstream changes have caused old revisions on master to stop building. Because homu maintains the invariant that each merge to master is passing, and we are only building the old revision to create a baseline to test the new revision, it's safe to ignore failures in the old rev. --- .travis/dispatch.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis/dispatch.sh b/.travis/dispatch.sh index f5b98361..cbfddc85 100755 --- a/.travis/dispatch.sh +++ b/.travis/dispatch.sh @@ -50,7 +50,11 @@ else else git fetch origin master:master git checkout master + # Upstream changes could cause the old rev to fail, so disable errexit + # (homu will maintain the invariant that each rev on master is passing) + set +o errexit run_salt 'old' + set -o errexit git checkout "${TRAVIS_COMMIT}" run_salt 'upgrade' From a040219a871e70c54efcc3f81752b523829251cf Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 23 Sep 2016 15:30:44 -0400 Subject: [PATCH 3/9] Use XCode8 (and OS X 10.11) Installing git via Homebrew now requires XCode 8. Additionally, this configures an OS X 10.11 image on Travis, which matches what we have deployment in production (the default is an OS X 10.9.5 image on Travis). With OS X 10.11, the previous launchctl.py module that we had backported has been superseded by a new mac_service.py module in 2016.3.0. Hence, backport the new mac_service.py module as well. saltfs-migration: Ensure that our OS X builders are all running OS X 10.11 and have XCode 8 installed. --- .travis.yml | 2 + _modules/launchctl.py | 357 ------------------------ _modules/mac_service.py | 601 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 603 insertions(+), 357 deletions(-) delete mode 100644 _modules/launchctl.py create mode 100644 _modules/mac_service.py diff --git a/.travis.yml b/.travis.yml index b2510867..2aa81189 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ matrix: - SALT_NODE_ID=servo-mac1 - SALT_FROM_SCRATCH=true os: osx + osx_image: xcode8 - env: - SALT_NODE_ID=servo-linux1 - SALT_FROM_SCRATCH=true @@ -43,6 +44,7 @@ matrix: - SALT_NODE_ID=servo-mac1 - SALT_FROM_SCRATCH=false os: osx + osx_image: xcode8 - env: - SALT_NODE_ID=servo-linux1 - SALT_FROM_SCRATCH=false diff --git a/_modules/launchctl.py b/_modules/launchctl.py deleted file mode 100644 index 7e5ae0d0..00000000 --- a/_modules/launchctl.py +++ /dev/null @@ -1,357 +0,0 @@ -# This module is a backported copy of the salt/states/pip_state.py module from -# Salt 2015.5.8 (at git revision ad4f204cb25c4856de59319a0c0692bfeb5243de), -# without any other changes applied. -# -# The original copyright and licensing notice for this module is reproduced -# below in the double-# comment block: -# -## Salt - Remote execution system -## -## Copyright 2014-2015 SaltStack Team -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. - -# -*- coding: utf-8 -*- -''' -Module for the management of MacOS systems that use launchd/launchctl - -.. important:: - If you feel that Salt should be using this module to manage services on a - minion, and it is using a different module (or gives an error similar to - *'service.start' is not available*), see :ref:`here - `. - -:depends: - plistlib Python module -''' -from __future__ import absolute_import -from distutils.version import LooseVersion - -# Import python libs -import logging -import os -import plistlib -import re - -# Import salt libs -import salt.utils -import salt.utils.decorators as decorators -import salt.ext.six as six - -# Set up logging -log = logging.getLogger(__name__) - -# Define the module's virtual name -__virtualname__ = 'service' - -BEFORE_YOSEMITE = True - - -def __virtual__(): - ''' - Only work on MacOS - ''' - if not salt.utils.is_darwin(): - return (False, 'Failed to load the mac_service module:\n' - 'Only available on Mac OS X systems.') - - if not os.path.exists('/bin/launchctl'): - return (False, 'Failed to load the mac_service module:\n' - 'Required binary not found: "/bin/launchctl"') - - if LooseVersion(__grains__['osrelease']) >= LooseVersion('10.11'): - return (False, 'Failed to load the mac_service module:\n' - 'Not available on El Capitan, uses mac_service.py') - - if LooseVersion(__grains__['osrelease']) >= LooseVersion('10.10'): - global BEFORE_YOSEMITE - BEFORE_YOSEMITE = False - - return __virtualname__ - - -def _launchd_paths(): - ''' - Paths where launchd services can be found - ''' - return [ - '/Library/LaunchAgents', - '/Library/LaunchDaemons', - '/System/Library/LaunchAgents', - '/System/Library/LaunchDaemons', - ] - - -@decorators.memoize -def _available_services(): - ''' - Return a dictionary of all available services on the system - ''' - available_services = dict() - for launch_dir in _launchd_paths(): - for root, dirs, files in os.walk(launch_dir): - for filename in files: - file_path = os.path.join(root, filename) - # Follow symbolic links of files in _launchd_paths - true_path = os.path.realpath(file_path) - # ignore broken symlinks - if not os.path.exists(true_path): - continue - - try: - # This assumes most of the plist files - # will be already in XML format - with salt.utils.fopen(file_path): - plist = plistlib.readPlist(true_path) - - except Exception: - # If plistlib is unable to read the file we'll need to use - # the system provided plutil program to do the conversion - cmd = '/usr/bin/plutil -convert xml1 -o - -- "{0}"'.format( - true_path) - plist_xml = __salt__['cmd.run_all']( - cmd, python_shell=False)['stdout'] - if six.PY2: - plist = plistlib.readPlistFromString(plist_xml) - else: - plist = plistlib.readPlistFromBytes( - salt.utils.to_bytes(plist_xml)) - - available_services[plist.Label.lower()] = { - 'filename': filename, - 'file_path': true_path, - 'plist': plist, - } - - return available_services - - -def _service_by_name(name): - ''' - Return the service info for a service by label, filename or path - ''' - services = _available_services() - name = name.lower() - - if name in services: - # Match on label - return services[name] - - for service in six.itervalues(services): - if service['file_path'].lower() == name: - # Match on full path - return service - basename, ext = os.path.splitext(service['filename']) - if basename.lower() == name: - # Match on basename - return service - - return False - - -def get_all(): - ''' - Return all installed services - - CLI Example: - - .. code-block:: bash - - salt '*' service.get_all - ''' - cmd = 'launchctl list' - - service_lines = [ - line for line in __salt__['cmd.run'](cmd).splitlines() - if not line.startswith('PID') - ] - - service_labels_from_list = [ - line.split("\t")[2] for line in service_lines - ] - service_labels_from_services = list(_available_services().keys()) - - return sorted(set(service_labels_from_list + service_labels_from_services)) - - -def _get_launchctl_data(job_label, runas=None): - if BEFORE_YOSEMITE: - cmd = 'launchctl list -x {0}'.format(job_label) - else: - cmd = 'launchctl list {0}'.format(job_label) - - launchctl_data = __salt__['cmd.run_all'](cmd, - python_shell=False, - runas=runas) - - if launchctl_data['stderr']: - # The service is not loaded, further, it might not even exist - # in either case we didn't get XML to parse, so return an empty - # dict - return None - - return launchctl_data['stdout'] - - -def available(job_label): - ''' - Check that the given service is available. - - CLI Example: - - .. code-block:: bash - - salt '*' service.available com.openssh.sshd - ''' - return True if _service_by_name(job_label) else False - - -def missing(job_label): - ''' - The inverse of service.available - Check that the given service is not available. - - CLI Example: - - .. code-block:: bash - - salt '*' service.missing com.openssh.sshd - ''' - return False if _service_by_name(job_label) else True - - -def status(job_label, runas=None): - ''' - Return the status for a service, returns a bool whether the service is - running. - - CLI Example: - - .. code-block:: bash - - salt '*' service.status - ''' - service = _service_by_name(job_label) - - lookup_name = service['plist']['Label'] if service else job_label - launchctl_data = _get_launchctl_data(lookup_name, runas=runas) - - if launchctl_data: - if BEFORE_YOSEMITE: - if six.PY3: - return 'PID' in plistlib.loads(launchctl_data) - else: - return 'PID' in dict(plistlib.readPlistFromString(launchctl_data)) - else: - pattern = '"PID" = [0-9]+;' - return True if re.search(pattern, launchctl_data) else False - else: - return False - - -def stop(job_label, runas=None): - ''' - Stop the specified service - - CLI Example: - - .. code-block:: bash - - salt '*' service.stop - salt '*' service.stop org.ntp.ntpd - salt '*' service.stop /System/Library/LaunchDaemons/org.ntp.ntpd.plist - ''' - service = _service_by_name(job_label) - if service: - cmd = 'launchctl unload -w {0}'.format(service['file_path'], - runas=runas) - return not __salt__['cmd.retcode'](cmd, runas=runas, python_shell=False) - - return False - - -def start(job_label, runas=None): - ''' - Start the specified service - - CLI Example: - - .. code-block:: bash - - salt '*' service.start - salt '*' service.start org.ntp.ntpd - salt '*' service.start /System/Library/LaunchDaemons/org.ntp.ntpd.plist - ''' - service = _service_by_name(job_label) - if service: - cmd = 'launchctl load -w {0}'.format(service['file_path'], runas=runas) - return not __salt__['cmd.retcode'](cmd, runas=runas, python_shell=False) - - return False - - -def restart(job_label, runas=None): - ''' - Restart the named service - - CLI Example: - - .. code-block:: bash - - salt '*' service.restart - ''' - stop(job_label, runas=runas) - return start(job_label, runas=runas) - - -def enabled(job_label, runas=None): - ''' - Return True if the named service is enabled, false otherwise - - CLI Example: - - .. code-block:: bash - - salt '*' service.enabled - ''' - overrides_data = dict(plistlib.readPlist( - '/var/db/launchd.db/com.apple.launchd/overrides.plist' - )) - if overrides_data.get(job_label, False): - if overrides_data[job_label]['Disabled']: - return False - else: - return True - else: - return False - - -def disabled(job_label, runas=None): - ''' - Return True if the named service is disabled, false otherwise - - CLI Example: - - .. code-block:: bash - - salt '*' service.disabled - ''' - overrides_data = dict(plistlib.readPlist( - '/var/db/launchd.db/com.apple.launchd/overrides.plist' - )) - if overrides_data.get(job_label, False): - if overrides_data[job_label]['Disabled']: - return True - else: - return False - else: - return True diff --git a/_modules/mac_service.py b/_modules/mac_service.py new file mode 100644 index 00000000..01aa11e4 --- /dev/null +++ b/_modules/mac_service.py @@ -0,0 +1,601 @@ +# This module is a backported copy of the salt/modules/mac_service.py module +# from Salt 2016.3.0 (at git revision 3e5218daea73f3f24b82a3078764ccb82c2a1ec9) +# without any other changes applied. +# +# The original copyright and licensing notice for this module is reproduced +# below in the double-# comment block: +# +## Salt - Remote execution system +## +## Copyright 2014-2015 SaltStack Team +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +# -*- coding: utf-8 -*- +''' +The service module for Mac OS X +.. versionadded:: 2016.3.0 +''' +from __future__ import absolute_import + +# Import python libs +import os +import re +import plistlib +from distutils.version import LooseVersion + +# Import salt libs +import salt.utils +import salt.utils.decorators as decorators +from salt.exceptions import CommandExecutionError + +# Import 3rd party libs +import salt.ext.six as six + +# Define the module's virtual name +__virtualname__ = 'service' + +__func_alias__ = { + 'list_': 'list', +} + + +def __virtual__(): + ''' + Only for Mac OS X with launchctl + ''' + if not salt.utils.is_darwin(): + return (False, 'Failed to load the mac_service module:\n' + 'Only available on Mac OS X systems.') + + if not salt.utils.which('launchctl'): + return (False, 'Failed to load the mac_service module:\n' + 'Required binary not found: "launchctl"') + + if not salt.utils.which('plutil'): + return (False, 'Failed to load the mac_service module:\n' + 'Required binary not found: "plutil"') + + if LooseVersion(__grains__['osrelease']) < LooseVersion('10.11'): + return (False, 'Failed to load the mac_service module:\n' + 'Requires OS X 10.11 or newer') + + return __virtualname__ + + +def _launchd_paths(): + ''' + Paths where launchd services can be found + ''' + return [ + '/Library/LaunchAgents', + '/Library/LaunchDaemons', + '/System/Library/LaunchAgents', + '/System/Library/LaunchDaemons', + ] + + +@decorators.memoize +def _available_services(): + ''' + Return a dictionary of all available services on the system + ''' + available_services = dict() + for launch_dir in _launchd_paths(): + for root, dirs, files in os.walk(launch_dir): + for file_name in files: + + # Must be a plist file + if not file_name.endswith('.plist'): + continue + + # Follow symbolic links of files in _launchd_paths + file_path = os.path.join(root, file_name) + true_path = os.path.realpath(file_path) + + # ignore broken symlinks + if not os.path.exists(true_path): + continue + + try: + # This assumes most of the plist files + # will be already in XML format + with salt.utils.fopen(file_path): + plist = plistlib.readPlist(true_path) + + except Exception: + # If plistlib is unable to read the file we'll need to use + # the system provided plutil program to do the conversion + cmd = '/usr/bin/plutil -convert xml1 -o - -- "{0}"'.format( + true_path) + plist_xml = __salt__['cmd.run'](cmd, output_loglevel='quiet') + if six.PY2: + plist = plistlib.readPlistFromString(plist_xml) + else: + plist = plistlib.readPlistFromBytes( + salt.utils.to_bytes(plist_xml)) + + try: + available_services[plist.Label.lower()] = { + 'file_name': file_name, + 'file_path': true_path, + 'plist': plist} + except AttributeError: + # Handle malformed plist files + available_services[os.path.basename(file_name).lower()] = { + 'file_name': file_name, + 'file_path': true_path, + 'plist': plist} + + return available_services + + +def _get_service(name): + ''' + Get information about a service. If the service is not found, raise an + error + + :param str name: Service label, file name, or full path + + :return: The service information for the service, otherwise an Error + :rtype: dict + ''' + services = _available_services() + name = name.lower() + + if name in services: + # Match on label + return services[name] + + for service in six.itervalues(services): + if service['file_path'].lower() == name: + # Match on full path + return service + basename, ext = os.path.splitext(service['file_name']) + if basename.lower() == name: + # Match on basename + return service + + # Could not find service + raise CommandExecutionError('Service not found: {0}'.format(name)) + + +def show(name): + ''' + Show properties of a launchctl service + + :param str name: Service label, file name, or full path + + :return: The service information if the service is found + :rtype: dict + + CLI Example: + + .. code-block:: bash + + salt '*' service.show org.cups.cupsd # service label + salt '*' service.show org.cups.cupsd.plist # file name + salt '*' service.show /System/Library/LaunchDaemons/org.cups.cupsd.plist # full path + ''' + return _get_service(name) + + +def launchctl(sub_cmd, *args, **kwargs): + ''' + Run a launchctl command and raise an error if it fails + + :param str sub_cmd: Sub command supplied to launchctl + + :param tuple args: Tuple containing additional arguments to pass to + launchctl + + :param dict kwargs: Dictionary containing arguments to pass to + ``cmd.run_all`` + + :param bool return_stdout: A keyword argument. If true return the stdout + of the launchctl command + + :return: ``True`` if successful, raise ``CommandExecutionError`` if not, or + the stdout of the launchctl command if requested + :rtype: bool, str + + CLI Example: + + .. code-block:: bash + + salt '*' service.launchctl debug org.cups.cupsd + ''' + # Get return type + return_stdout = kwargs.pop('return_stdout', False) + + # Construct command + cmd = ['launchctl', sub_cmd] + cmd.extend(args) + + # Run command + kwargs['python_shell'] = False + ret = __salt__['cmd.run_all'](cmd, **kwargs) + + # Raise an error or return successful result + if ret['retcode']: + out = 'Failed to {0} service:\n'.format(sub_cmd) + out += 'stdout: {0}\n'.format(ret['stdout']) + out += 'stderr: {0}\n'.format(ret['stderr']) + out += 'retcode: {0}\n'.format(ret['retcode']) + raise CommandExecutionError(out) + else: + return ret['stdout'] if return_stdout else True + + +def list_(name=None, runas=None): + ''' + Run launchctl list and return the output + + :param str name: The name of the service to list + + :param str runas: User to run launchctl commands + + :return: If a name is passed returns information about the named service, + otherwise returns a list of all services and pids + :rtype: str + + CLI Example: + + .. code-block:: bash + + salt '*' service.list + salt '*' service.list org.cups.cupsd + ''' + if name: + # Get service information and label + service = _get_service(name) + label = service['plist']['Label'] + + # Collect information on service: will raise an error if it fails + return launchctl('list', + label, + return_stdout=True, + output_loglevel='trace', + runas=runas) + + # Collect information on all services: will raise an error if it fails + return launchctl('list', + return_stdout=True, + output_loglevel='trace', + runas=runas) + + +def enable(name, runas=None): + ''' + Enable a launchd service. Raises an error if the service fails to be enabled + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful or if the service is already enabled + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.enable org.cups.cupsd + ''' + # Get service information and label + service = _get_service(name) + label = service['plist']['Label'] + + # Enable the service: will raise an error if it fails + return launchctl('enable', 'system/{0}'.format(label), runas=runas) + + +def disable(name, runas=None): + ''' + Disable a launchd service. Raises an error if the service fails to be + disabled + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful or if the service is already disabled + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.disable org.cups.cupsd + ''' + # Get service information and label + service = _get_service(name) + label = service['plist']['Label'] + + # disable the service: will raise an error if it fails + return launchctl('disable', 'system/{0}'.format(label), runas=runas) + + +def start(name, runas=None): + ''' + Start a launchd service. Raises an error if the service fails to start + + .. note:: + To start a service in Mac OS X the service must be enabled first. Use + ``service.enable`` to enable the service. + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful or if the service is already running + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.start org.cups.cupsd + ''' + # Get service information and file path + service = _get_service(name) + path = service['file_path'] + + # Load the service: will raise an error if it fails + return launchctl('load', path, runas=runas) + + +def stop(name, runas=None): + ''' + Stop a launchd service. Raises an error if the service fails to stop + + .. note:: + Though ``service.stop`` will unload a service in Mac OS X, the service + will start on next boot unless it is disabled. Use ``service.disable`` + to disable the service + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful or if the service is already stopped + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.stop org.cups.cupsd + ''' + # Get service information and file path + service = _get_service(name) + path = service['file_path'] + + # Disable the Launch Daemon: will raise an error if it fails + return launchctl('unload', path, runas=runas) + + +def restart(name, runas=None): + ''' + Unloads and reloads a launchd service. Raises an error if the service + fails to reload + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.restart org.cups.cupsd + ''' + # Restart the service: will raise an error if it fails + if enabled(name): + stop(name, runas=runas) + start(name, runas=runas) + + return True + + +def status(name, sig=None, runas=None): + ''' + Return the status for a service. + + :param str name: Used to find the service from launchctl. Can be any part + of the service name or a regex expression. + + :param str sig: Find the service with status.pid instead. Note that + ``name`` must still be provided. + + :param str runas: User to run launchctl commands + + :return: The PID for the service if it is running, otherwise an empty string + :rtype: str + + CLI Example: + + .. code-block:: bash + + salt '*' service.status cups + ''' + # Find service with ps + if sig: + return __salt__['status.pid'](sig) + + output = list_(runas=runas) + + # Used a string here instead of a list because that's what the linux version + # of this module does + pids = '' + for line in output.splitlines(): + if 'PID' in line: + continue + if re.search(name, line): + if line.split()[0].isdigit(): + if pids: + pids += '\n' + pids += line.split()[0] + + return pids + + +def available(name): + ''' + Check that the given service is available. + + :param str name: The name of the service + + :return: True if the service is available, otherwise False + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.available com.openssh.sshd + ''' + try: + _get_service(name) + return True + except CommandExecutionError: + return False + + +def missing(name): + ''' + The inverse of service.available + Check that the given service is not available. + + :param str name: The name of the service + + :return: True if the service is not available, otherwise False + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.missing com.openssh.sshd + ''' + return not available(name) + + +def enabled(name, runas=None): + ''' + Check if the specified service is enabled + + :param str name: The name of the service to look up + + :param str runas: User to run launchctl commands + + :return: True if the specified service enabled, otherwise False + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.enabled org.cups.cupsd + ''' + # Try to list the service. If it can't be listed, it's not enabled + try: + list_(name=name, runas=runas) + return True + except CommandExecutionError: + return False + + +def disabled(name, runas=None): + ''' + Check if the specified service is not enabled. This is the opposite of + ``service.enabled`` + + :param str name: The name to look up + + :param str runas: User to run launchctl commands + + :return: True if the specified service is NOT enabled, otherwise False + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.disabled org.cups.cupsd + ''' + # A service is disabled if it is not enabled + return not enabled(name, runas=runas) + + +def get_all(runas=None): + ''' + Return a list of services that are enabled or available. Can be used to + find the name of a service. + + :param str runas: User to run launchctl commands + + :return: A list of all the services available or enabled + :rtype: list + + CLI Example: + + .. code-block:: bash + + salt '*' service.get_all + ''' + # Get list of enabled services + enabled = get_enabled(runas=runas) + + # Get list of all services + available = list(_available_services().keys()) + + # Return composite list + return sorted(set(enabled + available)) + + +def get_enabled(runas=None): + ''' + Return a list of all services that are enabled. Can be used to find the + name of a service. + + :param str runas: User to run launchctl commands + + :return: A list of all the services enabled on the system + :rtype: list + + CLI Example: + + .. code-block:: bash + + salt '*' service.get_enabled + salt '*' service.get_enabled running=True + ''' + # Collect list of enabled services + stdout = list_(runas=runas) + service_lines = [line for line in stdout.splitlines()] + + # Construct list of enabled services + enabled = [] + for line in service_lines: + # Skip header line + if line.startswith('PID'): + continue + + pid, status, label = line.split('\t') + enabled.append(label) + + return sorted(set(enabled)) From 9fb6628bfc60e75a5687f121ae0b24636cd7c427 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 23 Sep 2016 16:58:15 -0400 Subject: [PATCH 4/9] Invalidate Salt cache in-between runs For some reason it seems Salt is not invalidating the `_modules` cache, causing build failures on upgrade builds (i.e. the second build on a not-from-scratch run). Invalidate the cache manually. Note that `nullglob` is set for safety, but we always expect `rm` to find cache files to invalidate. In case it does not find any files, the glob will expand to nothing, which is considered an error, causing failure via `errexit`. That is, not ignoring the result of rm is intentional. (`nullglob` is nicer than `failglob` because it is friendlier to loops over globs.) --- .travis/dispatch.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis/dispatch.sh b/.travis/dispatch.sh index cbfddc85..4fec9fc5 100755 --- a/.travis/dispatch.sh +++ b/.travis/dispatch.sh @@ -4,6 +4,8 @@ set -o errexit set -o nounset set -o pipefail +shopt -s nullglob + salt_call() { sudo salt-call \ --id="${SALT_NODE_ID}" \ @@ -58,6 +60,10 @@ else git checkout "${TRAVIS_COMMIT}" run_salt 'upgrade' + + # Invalidate the Salt cache + rm -rf /var/cache/salt/minion/files/base/* + salt_call 'saltutil.sync_all' fi # Only run tests against the new configuration From 87d97417bf54ed6745867e1711d41c83e0ab4c91 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 23 Sep 2016 19:59:12 -0400 Subject: [PATCH 5/9] Bump Salt to 2016.3.3 Homebrew recently changed their packaging of Salt and Python, using a virtualenvs for Salt. Because we were pinning an old version of Salt, but not the rest of Homebrew, this caused problems where pip was not being resolved properly. Update Salt to the latest version to fix this. As part of the update, also update some states to avoid deprecated parameters in favor of newer ones: - archive.extracted: archive_user -> user, group - cmd.run: user, group -> runas Note that due to https://github.com/saltstack/salt/pull/36552, `archive.extracted` states still have the archive_user argument to ensure parent directories are made with the correct permissions. saltfs-migration: Upgrade Salt on all machines. Steps: - Stop all salt-master and salt-minion services. Because we're upgrading Salt by two major versions at once, there are no guarantees that the masters and minions with different versions will be able to communicate. - Use the install script to re-install Salt on each machine, the master first then the minion. - Restart the salt-master and salt-minion services; pay attention to the system logs in case of startup failure. - Do NOT run a highstate until all minions and masters have been updated to Salt 2016.3.0. --- .travis/install_salt.sh | 8 ++++---- README.md | 2 +- salt/map.jinja | 4 ++-- servo-build-dependencies/android.sls | 9 ++++++--- servo-build-dependencies/arm.sls | 3 +++ servo-build-dependencies/init.sls | 4 ++-- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.travis/install_salt.sh b/.travis/install_salt.sh index 870e85a5..ccbd0709 100755 --- a/.travis/install_salt.sh +++ b/.travis/install_salt.sh @@ -11,14 +11,14 @@ install_salt () { # Use Trusty (Ubuntu 14.04) on Travis # Don't autostart services printf '#!/bin/sh\nexit 101\n' | sudo install -m 755 /dev/stdin /usr/sbin/policy-rc.d - curl https://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2015.5.8/SALTSTACK-GPG-KEY.pub | sudo apt-key add - - printf 'deb http://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2015.5.8 trusty main\n' | sudo tee /etc/apt/sources.list.d/saltstack.list >/dev/null + curl https://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2016.3.3/SALTSTACK-GPG-KEY.pub | sudo apt-key add - + printf 'deb http://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2016.3.3 trusty main\n' | sudo tee /etc/apt/sources.list.d/saltstack.list >/dev/null sudo apt-get -y update - sudo apt-get -y install salt-minion=2015.5.8+ds-1 + sudo apt-get -y install salt-minion=2016.3.3+ds-1 elif [[ "${OS_NAME}" == "osx" ]]; then printf "$0: installing salt for Mac OS X\n" brew update - brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/3461c9c74b2f3aba9a6fbd7165823c81dc2b4792/Formula/saltstack.rb + brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/9e3a66b6b7ca978bfea86897dcc3391c37f9f0ef/Formula/saltstack.rb else printf >&2 "$0: unknown operating system ${OS_NAME}\n" exit 1 diff --git a/README.md b/README.md index 9b201143..28d05887 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## What's going on? Salt is a configuration management tool that we use to automate Servo's -infrastructure. See [the tutorials](https://docs.saltstack.com/en/2015.5/topics/tutorials/index.html) to get started. +infrastructure. See [the tutorials](https://docs.saltstack.com/en/2016.3/topics/tutorials/index.html) to get started. ## Contributing diff --git a/salt/map.jinja b/salt/map.jinja index b31d988c..b3811699 100644 --- a/salt/map.jinja +++ b/salt/map.jinja @@ -1,7 +1,7 @@ {% set salt_ = salt %} {% set salt = { - 'version': '2015.5.8', + 'version': '2016.3.3', 'master': salt['grains.filter_by']({ 'defaults': { 'config': { @@ -19,7 +19,7 @@ 'Ubuntu': { 'pkg': { 'name': 'salt-master', - 'version': '2015.5.8+ds-1' + 'version': '2016.3.3+ds-1' } } }, diff --git a/servo-build-dependencies/android.sls b/servo-build-dependencies/android.sls index c43eb8f2..53ccddfd 100644 --- a/servo-build-dependencies/android.sls +++ b/servo-build-dependencies/android.sls @@ -42,7 +42,10 @@ android-sdk: - source: https://dl.google.com/android/android-sdk_{{ android.sdk.version }}-linux.tgz - source_hash: sha512={{ android.sdk.sha512 }} - archive_format: tar + # Workaround for https://github.com/saltstack/salt/pull/36552 - archive_user: servo + - user: servo + - group: servo - if_missing: {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux - require: - user: servo @@ -56,7 +59,7 @@ android-sdk: eof } ' - - user: servo + - runas: servo - creates: - {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux/platform-tools - {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux/platforms/android-{{ android.platform }} @@ -90,7 +93,7 @@ android-ndk: cmd.run: # Need to filter log output to avoid hitting log limits on Travis CI - name: '{{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}-linux-x86_64.bin | grep -v Extracting' - - user: servo + - runas: servo - cwd: {{ common.servo_home }}/android/ndk/{{ android.ndk.version }} - creates: {{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }} - require: @@ -99,7 +102,7 @@ android-ndk: android-toolchain: cmd.run: - name: bash {{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}/build/tools/make-standalone-toolchain.sh --platform=android-{{ android.platform }} --toolchain=arm-linux-androideabi-4.8 --install-dir='{{ common.servo_home }}/android/toolchain/{{ android.ndk.version }}/android-toolchain' --ndk-dir='{{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}' - - user: servo + - runas: servo - creates: {{ common.servo_home }}/android/toolchain/{{ android.ndk.version }}/android-toolchain - require: - cmd: android-ndk diff --git a/servo-build-dependencies/arm.sls b/servo-build-dependencies/arm.sls index e5ffeae9..09211fc4 100644 --- a/servo-build-dependencies/arm.sls +++ b/servo-build-dependencies/arm.sls @@ -39,7 +39,10 @@ libs-{{ target.name }}: - source: https://servo-rust.s3.amazonaws.com/ARM/{{ target.download_name }}/{{ target.version }}/{{ target.download_name }}-{{ target.version }}.tgz - source_hash: sha512={{ target.sha512 }} - archive_format: tar + # Workaround for https://github.com/saltstack/salt/pull/36552 - archive_user: servo + - user: servo + - group: servo {% for binary in binaries %} {{ common.servo_home }}/bin/{{ target.symlink_name }}-{{ binary }}: diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index dce3569d..19f21533 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -57,7 +57,7 @@ servo-darwin-homebrew-versions-dependencies: homebrew-link-autoconf: cmd.run: - name: 'brew link --overwrite autoconf' - - user: {{ homebrew.user }} + - runas: {{ homebrew.user }} - creates: /usr/local/Library/LinkedKegs/autoconf - require: - pkg: servo-dependencies @@ -66,7 +66,7 @@ homebrew-link-autoconf: homebrew-link-openssl: cmd.run: - name: 'brew link --force openssl' - - user: {{ homebrew.user }} + - runas: {{ homebrew.user }} - creates: /usr/local/Library/LinkedKegs/openssl - require: - pkg: servo-dependencies From 04d4e61761ae67cc75e21077db4b745381b47b7d Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 23 Sep 2016 21:52:56 -0400 Subject: [PATCH 6/9] Use existing config files when reinstalling Salt The install_salt script may be called again to reinstall or update Salt. Because we are Salting our configuration files, force apt-get to use any existing configuration files and ignore updates in the Salt packages. --- .travis/install_salt.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis/install_salt.sh b/.travis/install_salt.sh index ccbd0709..99b5209e 100755 --- a/.travis/install_salt.sh +++ b/.travis/install_salt.sh @@ -14,7 +14,11 @@ install_salt () { curl https://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2016.3.3/SALTSTACK-GPG-KEY.pub | sudo apt-key add - printf 'deb http://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2016.3.3 trusty main\n' | sudo tee /etc/apt/sources.list.d/saltstack.list >/dev/null sudo apt-get -y update - sudo apt-get -y install salt-minion=2016.3.3+ds-1 + # Use existing config file if it exists (if reinstalling) + sudo apt-get -y \ + -o Dpkg::Options::="--force-confold" \ + -o Dpkg::Options::="--force-confdef" \ + install salt-minion=2016.3.3+ds-1 elif [[ "${OS_NAME}" == "osx" ]]; then printf "$0: installing salt for Mac OS X\n" brew update From 4289ca7de7af6ef706199ac2cdfc47879678c112 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Fri, 23 Sep 2016 22:17:18 -0400 Subject: [PATCH 7/9] Don't link Homebrew OpenSSL, use env vars OpenSSL is a "keg-only crate", which Homebrew 1.0.0 won't link because ``` we may end up linking against the insecure, deprecated system OpenSSL while using the headers from Homebrew's openssl. ``` Instead, use the `OPENSSL_INCLUDE_DIR` and `OPENSSL_LIB_DIR` to explicitly pass the paths to the Homebrew OpenSSL. Note that these paths are currently hardcoded because they are unlikely to change. For some reason, Homebrew was reporting this error on its stderr but Salt was not picking it up as and failing the state - possibly because the exit status seems to have been 0. saltfs-migration: Run `brew unlink --dry-run openssl` on the Mac builders, then `brew unlink openssl` once confirming the output is as expected. This will unlink openssl on the existing builders. --- buildbot/master/files/config/environments.py | 2 ++ servo-build-dependencies/init.sls | 8 -------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/buildbot/master/files/config/environments.py b/buildbot/master/files/config/environments.py index d014a90f..917c54d8 100644 --- a/buildbot/master/files/config/environments.py +++ b/buildbot/master/files/config/environments.py @@ -64,6 +64,8 @@ def without(self, to_unset): 'CARGO_HOME': '/Users/servo/.cargo', 'CCACHE': '/usr/local/bin/ccache', 'SERVO_CACHE_DIR': '/Users/servo/.servo', + 'OPENSSL_INCLUDE_DIR': '/usr/local/opt/openssl/include', + 'OPENSSL_LIB_DIR': '/usr/local/opt/openssl/lib', }) diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index 19f21533..54581ac8 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -62,14 +62,6 @@ homebrew-link-autoconf: - require: - pkg: servo-dependencies - module: servo-darwin-homebrew-versions-dependencies - -homebrew-link-openssl: - cmd.run: - - name: 'brew link --force openssl' - - runas: {{ homebrew.user }} - - creates: /usr/local/Library/LinkedKegs/openssl - - require: - - pkg: servo-dependencies {% else %} multiverse: pkgrepo.managed: From c9a4710c188135d7e66ea4dafd5e8be94e32777d Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 24 Sep 2016 00:23:43 -0400 Subject: [PATCH 8/9] Unlink existing Salt in Homebrew before installing If a different version of Salt is already installed via Homebrew, it will not allow installing a different version without previously unlinking the old one. Ideally there would be an atomic operation to do both of these in one command, but it appears Homebrew does not have one. Thus, users of this script should be careful to watch the output in case the script is interrupted and no Salt installation is left linked. --- .travis/install_salt.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis/install_salt.sh b/.travis/install_salt.sh index 99b5209e..abdc0b5e 100755 --- a/.travis/install_salt.sh +++ b/.travis/install_salt.sh @@ -22,6 +22,11 @@ install_salt () { elif [[ "${OS_NAME}" == "osx" ]]; then printf "$0: installing salt for Mac OS X\n" brew update + # Unlink allows switching versions, + # I wish Homebrew had an atomic operation for pinned upgrades + if brew list | grep 'saltstack' >/dev/null; then + brew unlink saltstack + fi brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/9e3a66b6b7ca978bfea86897dcc3391c37f9f0ef/Formula/saltstack.rb else printf >&2 "$0: unknown operating system ${OS_NAME}\n" From 4c67e6584ee2b576ed2ec3e733c815dc8c881c2c Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Sat, 24 Sep 2016 01:09:53 -0400 Subject: [PATCH 9/9] Fix autoconf/autoconf213 installation via Homebrew Use an idempotent cmd.run to tap `homebrew/versions` only if necessary. Homebrew 1.0.0 apparently tightened its return codes and will exit with a non-zero status if linking fails during installation; there is no way to install without linking or install and link --overwrite in one operation. This makes it difficult to install multiple versions that have conflicting links from the CLI, so add a custom script to handle ensuring that autoconf and autoconf213 are installed and linked properly, with autoconf's links over autoconf213's links, in an idempotent way and doing as little as possible. --- .../files/install-homebrew-autoconf213.sh | 112 ++++++++++++++++++ servo-build-dependencies/init.sls | 27 ++--- 2 files changed, 123 insertions(+), 16 deletions(-) create mode 100644 servo-build-dependencies/files/install-homebrew-autoconf213.sh diff --git a/servo-build-dependencies/files/install-homebrew-autoconf213.sh b/servo-build-dependencies/files/install-homebrew-autoconf213.sh new file mode 100644 index 00000000..c1437b0e --- /dev/null +++ b/servo-build-dependencies/files/install-homebrew-autoconf213.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# Homebrew insists on being stateful, non-idempotent, lacking useful command +# line flags, and in general hard to administer, so this is a custom script for +# the simple yet impossible via Homebrew CLI task of installing two versions of +# the same package, without an error. +# Specifically, autoconf213 and autoconf need to be installed, with autoconf's +# links taking precedence. + +set -o errexit +set -o nounset +set -o pipefail + + +# Helper methods because brew doesn't like conflicting links during install, +# and there is no way to install w/o linking from the CLI or ignore that error, +# only via editing the Formula to add `keg_only`. +# Hence, just ignore errors for now, and double-check everything at the end. +brew_install() { set +o errexit; brew install "$@"; set -o errexit; } +brew_link() { set +o errexit; brew link "$@"; set -o errexit; } + +# Use "yes"/"no" for conditionals (not "true"/"false") +# to avoid confusion with the true/false commands +autoconf_installed="no" +autoconf_fully_linked="no" +autoconf213_installed="no" +autoconf213_linked="no" + + +set_autoconf_vars() { + if brew list | grep 'autoconf' >/dev/null; then + autoconf_installed="yes" + if readlink '/usr/local/share/info/autoconf.info' \ + | grep 'Cellar/autoconf/' >/dev/null; then + autoconf_fully_linked="yes" + fi + fi +} + + +set_autoconf213_vars() { + if brew list | grep 'autoconf213' >/dev/null; then + autoconf213_installed="yes" + if readlink '/usr/local/bin/autoconf213' \ + | grep 'Cellar/autoconf213/' >/dev/null; then + autoconf213_linked="yes" + fi + fi +} + + +check() { + local verbose="no" + if [[ "$#" -ge 1 && "${1}" == 'verbose' ]]; then + declare -r verbose="yes" + fi + set_autoconf_vars + set_autoconf213_vars + + if [[ "${autoconf213_installed}" == "yes" + && "${autoconf_installed}" == "yes" + && "${autoconf213_linked}" == "yes" + && "${autoconf_fully_linked}" == "yes" ]]; then + return 0 + else + if [[ "${verbose}" == "yes" ]]; then + printf "%s\n" "autoconf/autoconf213 check failed:" + printf "%s %s\n" "autoconf 213 installed?" \ + "${autoconf213_installed}" + printf "%s %s\n" "autoconf installed?" \ + "${autoconf_installed}" + printf "%s %s\n" "autoconf213 linked?" \ + "${autoconf213_linked}" + printf "%s %s\n" "autoconf fully linked?" \ + "${autoconf_fully_linked}" + fi + + return 1 + fi +} + + +main() { + if check; then + return 0 + fi + + # autoconf213 is first so autoconf can override + set_autoconf213_vars + if [[ "${autoconf213_installed}" == "no" ]]; then + brew_install autoconf213 + fi + set_autoconf213_vars + if [[ "${autoconf213_linked}" == "no" ]]; then + brew_link --overwrite autoconf213 + fi + + set_autoconf_vars + if [[ "${autoconf_installed}" == "no" ]]; then + brew_install autoconf + fi + set_autoconf_vars + if [[ "${autoconf_fully_linked}" == "no" ]]; then + brew_link --overwrite autoconf + fi + + check 'verbose' # errexit will handle return in failure case + return 0 +} + + +main "$@" diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index 54581ac8..3c60f1be 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -42,26 +42,21 @@ servo-dependencies: {% if grains['kernel'] == 'Darwin' %} # Workaround for https://github.com/saltstack/salt/issues/26414 -servo-darwin-homebrew-versions-dependencies: - module.run: - - name: pkg.install - - pkgs: - - autoconf213 - - taps: - - homebrew/versions - -# Warning: These states that manually run brew link only check that some -# version of the Homebrew package is linked, not necessarily the version -# linked above. Whether this handles updates properly is an open question. -# These should be replaced by a custom Salt state. -homebrew-link-autoconf: +servo-darwin-tap-homebrew-versions: cmd.run: - - name: 'brew link --overwrite autoconf' + - name: 'brew tap homebrew/versions' + - runas: {{ homebrew.user }} + - unless: 'brew tap | grep homebrew/versions' + - require: + - pkg: servo-dependencies + +# This should be replaced by a custom Salt state. +servo-darwin-install-autoconf213-and-fix-links: + cmd.script: + - source: salt://{{ tpldir }}/files/install-homebrew-autoconf213.sh - runas: {{ homebrew.user }} - - creates: /usr/local/Library/LinkedKegs/autoconf - require: - pkg: servo-dependencies - - module: servo-darwin-homebrew-versions-dependencies {% else %} multiverse: pkgrepo.managed: