diff --git a/.travis.yml b/.travis.yml index b2510867..2aa81189 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ matrix: - SALT_NODE_ID=servo-mac1 - SALT_FROM_SCRATCH=true os: osx + osx_image: xcode8 - env: - SALT_NODE_ID=servo-linux1 - SALT_FROM_SCRATCH=true @@ -43,6 +44,7 @@ matrix: - SALT_NODE_ID=servo-mac1 - SALT_FROM_SCRATCH=false os: osx + osx_image: xcode8 - env: - SALT_NODE_ID=servo-linux1 - SALT_FROM_SCRATCH=false diff --git a/.travis/dispatch.sh b/.travis/dispatch.sh index f5b98361..4fec9fc5 100755 --- a/.travis/dispatch.sh +++ b/.travis/dispatch.sh @@ -4,6 +4,8 @@ set -o errexit set -o nounset set -o pipefail +shopt -s nullglob + salt_call() { sudo salt-call \ --id="${SALT_NODE_ID}" \ @@ -50,10 +52,18 @@ else else git fetch origin master:master git checkout master + # Upstream changes could cause the old rev to fail, so disable errexit + # (homu will maintain the invariant that each rev on master is passing) + set +o errexit run_salt 'old' + set -o errexit git checkout "${TRAVIS_COMMIT}" run_salt 'upgrade' + + # Invalidate the Salt cache + rm -rf /var/cache/salt/minion/files/base/* + salt_call 'saltutil.sync_all' fi # Only run tests against the new configuration diff --git a/.travis/install_salt.sh b/.travis/install_salt.sh index 870e85a5..abdc0b5e 100755 --- a/.travis/install_salt.sh +++ b/.travis/install_salt.sh @@ -11,14 +11,23 @@ install_salt () { # Use Trusty (Ubuntu 14.04) on Travis # Don't autostart services printf '#!/bin/sh\nexit 101\n' | sudo install -m 755 /dev/stdin /usr/sbin/policy-rc.d - curl https://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2015.5.8/SALTSTACK-GPG-KEY.pub | sudo apt-key add - - printf 'deb http://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2015.5.8 trusty main\n' | sudo tee /etc/apt/sources.list.d/saltstack.list >/dev/null + curl https://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2016.3.3/SALTSTACK-GPG-KEY.pub | sudo apt-key add - + printf 'deb http://repo.saltstack.com/apt/ubuntu/14.04/amd64/archive/2016.3.3 trusty main\n' | sudo tee /etc/apt/sources.list.d/saltstack.list >/dev/null sudo apt-get -y update - sudo apt-get -y install salt-minion=2015.5.8+ds-1 + # Use existing config file if it exists (if reinstalling) + sudo apt-get -y \ + -o Dpkg::Options::="--force-confold" \ + -o Dpkg::Options::="--force-confdef" \ + install salt-minion=2016.3.3+ds-1 elif [[ "${OS_NAME}" == "osx" ]]; then printf "$0: installing salt for Mac OS X\n" brew update - brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/3461c9c74b2f3aba9a6fbd7165823c81dc2b4792/Formula/saltstack.rb + # Unlink allows switching versions, + # I wish Homebrew had an atomic operation for pinned upgrades + if brew list | grep 'saltstack' >/dev/null; then + brew unlink saltstack + fi + brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/9e3a66b6b7ca978bfea86897dcc3391c37f9f0ef/Formula/saltstack.rb else printf >&2 "$0: unknown operating system ${OS_NAME}\n" exit 1 diff --git a/README.md b/README.md index 9b201143..28d05887 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## What's going on? Salt is a configuration management tool that we use to automate Servo's -infrastructure. See [the tutorials](https://docs.saltstack.com/en/2015.5/topics/tutorials/index.html) to get started. +infrastructure. See [the tutorials](https://docs.saltstack.com/en/2016.3/topics/tutorials/index.html) to get started. ## Contributing diff --git a/_modules/homebrew.py b/_modules/homebrew.py new file mode 100644 index 00000000..a7ed7295 --- /dev/null +++ b/_modules/homebrew.py @@ -0,0 +1,89 @@ +# This module is mainly comprised of new code, but also contains modified +# versions of functions from the salt/modules/mac_brew.py module from Salt +# develop (at git revision 3e5218daea73f3f24b82a3078764ccb82c2a1ec9). +# Functions taken/modified from Salt are marked, all others are original. +# +# The original copyright and licensing notice for the methods from the +# mac_brew.py module is reproduced below in the double-# comment block: +# +## Salt - Remote execution system +## +## Copyright 2014-2015 SaltStack Team +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +''' +Module for the management of homebrew +''' +from __future__ import absolute_import + +# Import python libs +import logging +import os + +# Import salt libs +from salt.exceptions import CommandExecutionError +import salt.utils + +# Set up logging +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only work if Homebrew is installed + ''' + if salt.utils.which('brew'): + return True + return ( + False, + 'The homebrew execution module could not be loaded: brew not found' + ) + + +def _homebrew_bin(): + ''' + Returns the full path to the homebrew binary in the PATH. + Taken from mac_brew.py with modifications. + ''' + homebrew_dir = __salt__['cmd.run']( + 'brew --prefix', + output_loglevel='trace' + ) + return os.path.join(homebrew_dir, 'bin', 'brew') + + +def cmd_all(args): + ''' + Calls brew with the specified arguments and as the correct user. + Taken from mac_brew.py with modifications. + + args: + Should be a list of arguments to pass to the `brew` binary. + ''' + user = __salt__['file.get_user'](_homebrew_bin()) + runas = user if user != __opts__['user'] else None + ret = __salt__['cmd.run_all']( + ['brew'] + args, + runas=runas, + output_loglevel='trace', + python_shell=False, + redirect_stderr=False, + ) + if ret['retcode'] != 0: + raise CommandExecutionError( + 'stdout: {stdout}\n' + 'stderr: {stderr}\n' + 'retcode: {retcode}\n'.format(**ret) + ) + return ret diff --git a/_modules/launchctl.py b/_modules/launchctl.py deleted file mode 100644 index 7e5ae0d0..00000000 --- a/_modules/launchctl.py +++ /dev/null @@ -1,357 +0,0 @@ -# This module is a backported copy of the salt/states/pip_state.py module from -# Salt 2015.5.8 (at git revision ad4f204cb25c4856de59319a0c0692bfeb5243de), -# without any other changes applied. -# -# The original copyright and licensing notice for this module is reproduced -# below in the double-# comment block: -# -## Salt - Remote execution system -## -## Copyright 2014-2015 SaltStack Team -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. - -# -*- coding: utf-8 -*- -''' -Module for the management of MacOS systems that use launchd/launchctl - -.. important:: - If you feel that Salt should be using this module to manage services on a - minion, and it is using a different module (or gives an error similar to - *'service.start' is not available*), see :ref:`here - `. - -:depends: - plistlib Python module -''' -from __future__ import absolute_import -from distutils.version import LooseVersion - -# Import python libs -import logging -import os -import plistlib -import re - -# Import salt libs -import salt.utils -import salt.utils.decorators as decorators -import salt.ext.six as six - -# Set up logging -log = logging.getLogger(__name__) - -# Define the module's virtual name -__virtualname__ = 'service' - -BEFORE_YOSEMITE = True - - -def __virtual__(): - ''' - Only work on MacOS - ''' - if not salt.utils.is_darwin(): - return (False, 'Failed to load the mac_service module:\n' - 'Only available on Mac OS X systems.') - - if not os.path.exists('/bin/launchctl'): - return (False, 'Failed to load the mac_service module:\n' - 'Required binary not found: "/bin/launchctl"') - - if LooseVersion(__grains__['osrelease']) >= LooseVersion('10.11'): - return (False, 'Failed to load the mac_service module:\n' - 'Not available on El Capitan, uses mac_service.py') - - if LooseVersion(__grains__['osrelease']) >= LooseVersion('10.10'): - global BEFORE_YOSEMITE - BEFORE_YOSEMITE = False - - return __virtualname__ - - -def _launchd_paths(): - ''' - Paths where launchd services can be found - ''' - return [ - '/Library/LaunchAgents', - '/Library/LaunchDaemons', - '/System/Library/LaunchAgents', - '/System/Library/LaunchDaemons', - ] - - -@decorators.memoize -def _available_services(): - ''' - Return a dictionary of all available services on the system - ''' - available_services = dict() - for launch_dir in _launchd_paths(): - for root, dirs, files in os.walk(launch_dir): - for filename in files: - file_path = os.path.join(root, filename) - # Follow symbolic links of files in _launchd_paths - true_path = os.path.realpath(file_path) - # ignore broken symlinks - if not os.path.exists(true_path): - continue - - try: - # This assumes most of the plist files - # will be already in XML format - with salt.utils.fopen(file_path): - plist = plistlib.readPlist(true_path) - - except Exception: - # If plistlib is unable to read the file we'll need to use - # the system provided plutil program to do the conversion - cmd = '/usr/bin/plutil -convert xml1 -o - -- "{0}"'.format( - true_path) - plist_xml = __salt__['cmd.run_all']( - cmd, python_shell=False)['stdout'] - if six.PY2: - plist = plistlib.readPlistFromString(plist_xml) - else: - plist = plistlib.readPlistFromBytes( - salt.utils.to_bytes(plist_xml)) - - available_services[plist.Label.lower()] = { - 'filename': filename, - 'file_path': true_path, - 'plist': plist, - } - - return available_services - - -def _service_by_name(name): - ''' - Return the service info for a service by label, filename or path - ''' - services = _available_services() - name = name.lower() - - if name in services: - # Match on label - return services[name] - - for service in six.itervalues(services): - if service['file_path'].lower() == name: - # Match on full path - return service - basename, ext = os.path.splitext(service['filename']) - if basename.lower() == name: - # Match on basename - return service - - return False - - -def get_all(): - ''' - Return all installed services - - CLI Example: - - .. code-block:: bash - - salt '*' service.get_all - ''' - cmd = 'launchctl list' - - service_lines = [ - line for line in __salt__['cmd.run'](cmd).splitlines() - if not line.startswith('PID') - ] - - service_labels_from_list = [ - line.split("\t")[2] for line in service_lines - ] - service_labels_from_services = list(_available_services().keys()) - - return sorted(set(service_labels_from_list + service_labels_from_services)) - - -def _get_launchctl_data(job_label, runas=None): - if BEFORE_YOSEMITE: - cmd = 'launchctl list -x {0}'.format(job_label) - else: - cmd = 'launchctl list {0}'.format(job_label) - - launchctl_data = __salt__['cmd.run_all'](cmd, - python_shell=False, - runas=runas) - - if launchctl_data['stderr']: - # The service is not loaded, further, it might not even exist - # in either case we didn't get XML to parse, so return an empty - # dict - return None - - return launchctl_data['stdout'] - - -def available(job_label): - ''' - Check that the given service is available. - - CLI Example: - - .. code-block:: bash - - salt '*' service.available com.openssh.sshd - ''' - return True if _service_by_name(job_label) else False - - -def missing(job_label): - ''' - The inverse of service.available - Check that the given service is not available. - - CLI Example: - - .. code-block:: bash - - salt '*' service.missing com.openssh.sshd - ''' - return False if _service_by_name(job_label) else True - - -def status(job_label, runas=None): - ''' - Return the status for a service, returns a bool whether the service is - running. - - CLI Example: - - .. code-block:: bash - - salt '*' service.status - ''' - service = _service_by_name(job_label) - - lookup_name = service['plist']['Label'] if service else job_label - launchctl_data = _get_launchctl_data(lookup_name, runas=runas) - - if launchctl_data: - if BEFORE_YOSEMITE: - if six.PY3: - return 'PID' in plistlib.loads(launchctl_data) - else: - return 'PID' in dict(plistlib.readPlistFromString(launchctl_data)) - else: - pattern = '"PID" = [0-9]+;' - return True if re.search(pattern, launchctl_data) else False - else: - return False - - -def stop(job_label, runas=None): - ''' - Stop the specified service - - CLI Example: - - .. code-block:: bash - - salt '*' service.stop - salt '*' service.stop org.ntp.ntpd - salt '*' service.stop /System/Library/LaunchDaemons/org.ntp.ntpd.plist - ''' - service = _service_by_name(job_label) - if service: - cmd = 'launchctl unload -w {0}'.format(service['file_path'], - runas=runas) - return not __salt__['cmd.retcode'](cmd, runas=runas, python_shell=False) - - return False - - -def start(job_label, runas=None): - ''' - Start the specified service - - CLI Example: - - .. code-block:: bash - - salt '*' service.start - salt '*' service.start org.ntp.ntpd - salt '*' service.start /System/Library/LaunchDaemons/org.ntp.ntpd.plist - ''' - service = _service_by_name(job_label) - if service: - cmd = 'launchctl load -w {0}'.format(service['file_path'], runas=runas) - return not __salt__['cmd.retcode'](cmd, runas=runas, python_shell=False) - - return False - - -def restart(job_label, runas=None): - ''' - Restart the named service - - CLI Example: - - .. code-block:: bash - - salt '*' service.restart - ''' - stop(job_label, runas=runas) - return start(job_label, runas=runas) - - -def enabled(job_label, runas=None): - ''' - Return True if the named service is enabled, false otherwise - - CLI Example: - - .. code-block:: bash - - salt '*' service.enabled - ''' - overrides_data = dict(plistlib.readPlist( - '/var/db/launchd.db/com.apple.launchd/overrides.plist' - )) - if overrides_data.get(job_label, False): - if overrides_data[job_label]['Disabled']: - return False - else: - return True - else: - return False - - -def disabled(job_label, runas=None): - ''' - Return True if the named service is disabled, false otherwise - - CLI Example: - - .. code-block:: bash - - salt '*' service.disabled - ''' - overrides_data = dict(plistlib.readPlist( - '/var/db/launchd.db/com.apple.launchd/overrides.plist' - )) - if overrides_data.get(job_label, False): - if overrides_data[job_label]['Disabled']: - return True - else: - return False - else: - return True diff --git a/_modules/mac_service.py b/_modules/mac_service.py new file mode 100644 index 00000000..01aa11e4 --- /dev/null +++ b/_modules/mac_service.py @@ -0,0 +1,601 @@ +# This module is a backported copy of the salt/modules/mac_service.py module +# from Salt 2016.3.0 (at git revision 3e5218daea73f3f24b82a3078764ccb82c2a1ec9) +# without any other changes applied. +# +# The original copyright and licensing notice for this module is reproduced +# below in the double-# comment block: +# +## Salt - Remote execution system +## +## Copyright 2014-2015 SaltStack Team +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +# -*- coding: utf-8 -*- +''' +The service module for Mac OS X +.. versionadded:: 2016.3.0 +''' +from __future__ import absolute_import + +# Import python libs +import os +import re +import plistlib +from distutils.version import LooseVersion + +# Import salt libs +import salt.utils +import salt.utils.decorators as decorators +from salt.exceptions import CommandExecutionError + +# Import 3rd party libs +import salt.ext.six as six + +# Define the module's virtual name +__virtualname__ = 'service' + +__func_alias__ = { + 'list_': 'list', +} + + +def __virtual__(): + ''' + Only for Mac OS X with launchctl + ''' + if not salt.utils.is_darwin(): + return (False, 'Failed to load the mac_service module:\n' + 'Only available on Mac OS X systems.') + + if not salt.utils.which('launchctl'): + return (False, 'Failed to load the mac_service module:\n' + 'Required binary not found: "launchctl"') + + if not salt.utils.which('plutil'): + return (False, 'Failed to load the mac_service module:\n' + 'Required binary not found: "plutil"') + + if LooseVersion(__grains__['osrelease']) < LooseVersion('10.11'): + return (False, 'Failed to load the mac_service module:\n' + 'Requires OS X 10.11 or newer') + + return __virtualname__ + + +def _launchd_paths(): + ''' + Paths where launchd services can be found + ''' + return [ + '/Library/LaunchAgents', + '/Library/LaunchDaemons', + '/System/Library/LaunchAgents', + '/System/Library/LaunchDaemons', + ] + + +@decorators.memoize +def _available_services(): + ''' + Return a dictionary of all available services on the system + ''' + available_services = dict() + for launch_dir in _launchd_paths(): + for root, dirs, files in os.walk(launch_dir): + for file_name in files: + + # Must be a plist file + if not file_name.endswith('.plist'): + continue + + # Follow symbolic links of files in _launchd_paths + file_path = os.path.join(root, file_name) + true_path = os.path.realpath(file_path) + + # ignore broken symlinks + if not os.path.exists(true_path): + continue + + try: + # This assumes most of the plist files + # will be already in XML format + with salt.utils.fopen(file_path): + plist = plistlib.readPlist(true_path) + + except Exception: + # If plistlib is unable to read the file we'll need to use + # the system provided plutil program to do the conversion + cmd = '/usr/bin/plutil -convert xml1 -o - -- "{0}"'.format( + true_path) + plist_xml = __salt__['cmd.run'](cmd, output_loglevel='quiet') + if six.PY2: + plist = plistlib.readPlistFromString(plist_xml) + else: + plist = plistlib.readPlistFromBytes( + salt.utils.to_bytes(plist_xml)) + + try: + available_services[plist.Label.lower()] = { + 'file_name': file_name, + 'file_path': true_path, + 'plist': plist} + except AttributeError: + # Handle malformed plist files + available_services[os.path.basename(file_name).lower()] = { + 'file_name': file_name, + 'file_path': true_path, + 'plist': plist} + + return available_services + + +def _get_service(name): + ''' + Get information about a service. If the service is not found, raise an + error + + :param str name: Service label, file name, or full path + + :return: The service information for the service, otherwise an Error + :rtype: dict + ''' + services = _available_services() + name = name.lower() + + if name in services: + # Match on label + return services[name] + + for service in six.itervalues(services): + if service['file_path'].lower() == name: + # Match on full path + return service + basename, ext = os.path.splitext(service['file_name']) + if basename.lower() == name: + # Match on basename + return service + + # Could not find service + raise CommandExecutionError('Service not found: {0}'.format(name)) + + +def show(name): + ''' + Show properties of a launchctl service + + :param str name: Service label, file name, or full path + + :return: The service information if the service is found + :rtype: dict + + CLI Example: + + .. code-block:: bash + + salt '*' service.show org.cups.cupsd # service label + salt '*' service.show org.cups.cupsd.plist # file name + salt '*' service.show /System/Library/LaunchDaemons/org.cups.cupsd.plist # full path + ''' + return _get_service(name) + + +def launchctl(sub_cmd, *args, **kwargs): + ''' + Run a launchctl command and raise an error if it fails + + :param str sub_cmd: Sub command supplied to launchctl + + :param tuple args: Tuple containing additional arguments to pass to + launchctl + + :param dict kwargs: Dictionary containing arguments to pass to + ``cmd.run_all`` + + :param bool return_stdout: A keyword argument. If true return the stdout + of the launchctl command + + :return: ``True`` if successful, raise ``CommandExecutionError`` if not, or + the stdout of the launchctl command if requested + :rtype: bool, str + + CLI Example: + + .. code-block:: bash + + salt '*' service.launchctl debug org.cups.cupsd + ''' + # Get return type + return_stdout = kwargs.pop('return_stdout', False) + + # Construct command + cmd = ['launchctl', sub_cmd] + cmd.extend(args) + + # Run command + kwargs['python_shell'] = False + ret = __salt__['cmd.run_all'](cmd, **kwargs) + + # Raise an error or return successful result + if ret['retcode']: + out = 'Failed to {0} service:\n'.format(sub_cmd) + out += 'stdout: {0}\n'.format(ret['stdout']) + out += 'stderr: {0}\n'.format(ret['stderr']) + out += 'retcode: {0}\n'.format(ret['retcode']) + raise CommandExecutionError(out) + else: + return ret['stdout'] if return_stdout else True + + +def list_(name=None, runas=None): + ''' + Run launchctl list and return the output + + :param str name: The name of the service to list + + :param str runas: User to run launchctl commands + + :return: If a name is passed returns information about the named service, + otherwise returns a list of all services and pids + :rtype: str + + CLI Example: + + .. code-block:: bash + + salt '*' service.list + salt '*' service.list org.cups.cupsd + ''' + if name: + # Get service information and label + service = _get_service(name) + label = service['plist']['Label'] + + # Collect information on service: will raise an error if it fails + return launchctl('list', + label, + return_stdout=True, + output_loglevel='trace', + runas=runas) + + # Collect information on all services: will raise an error if it fails + return launchctl('list', + return_stdout=True, + output_loglevel='trace', + runas=runas) + + +def enable(name, runas=None): + ''' + Enable a launchd service. Raises an error if the service fails to be enabled + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful or if the service is already enabled + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.enable org.cups.cupsd + ''' + # Get service information and label + service = _get_service(name) + label = service['plist']['Label'] + + # Enable the service: will raise an error if it fails + return launchctl('enable', 'system/{0}'.format(label), runas=runas) + + +def disable(name, runas=None): + ''' + Disable a launchd service. Raises an error if the service fails to be + disabled + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful or if the service is already disabled + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.disable org.cups.cupsd + ''' + # Get service information and label + service = _get_service(name) + label = service['plist']['Label'] + + # disable the service: will raise an error if it fails + return launchctl('disable', 'system/{0}'.format(label), runas=runas) + + +def start(name, runas=None): + ''' + Start a launchd service. Raises an error if the service fails to start + + .. note:: + To start a service in Mac OS X the service must be enabled first. Use + ``service.enable`` to enable the service. + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful or if the service is already running + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.start org.cups.cupsd + ''' + # Get service information and file path + service = _get_service(name) + path = service['file_path'] + + # Load the service: will raise an error if it fails + return launchctl('load', path, runas=runas) + + +def stop(name, runas=None): + ''' + Stop a launchd service. Raises an error if the service fails to stop + + .. note:: + Though ``service.stop`` will unload a service in Mac OS X, the service + will start on next boot unless it is disabled. Use ``service.disable`` + to disable the service + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful or if the service is already stopped + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.stop org.cups.cupsd + ''' + # Get service information and file path + service = _get_service(name) + path = service['file_path'] + + # Disable the Launch Daemon: will raise an error if it fails + return launchctl('unload', path, runas=runas) + + +def restart(name, runas=None): + ''' + Unloads and reloads a launchd service. Raises an error if the service + fails to reload + + :param str name: Service label, file name, or full path + + :param str runas: User to run launchctl commands + + :return: ``True`` if successful + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.restart org.cups.cupsd + ''' + # Restart the service: will raise an error if it fails + if enabled(name): + stop(name, runas=runas) + start(name, runas=runas) + + return True + + +def status(name, sig=None, runas=None): + ''' + Return the status for a service. + + :param str name: Used to find the service from launchctl. Can be any part + of the service name or a regex expression. + + :param str sig: Find the service with status.pid instead. Note that + ``name`` must still be provided. + + :param str runas: User to run launchctl commands + + :return: The PID for the service if it is running, otherwise an empty string + :rtype: str + + CLI Example: + + .. code-block:: bash + + salt '*' service.status cups + ''' + # Find service with ps + if sig: + return __salt__['status.pid'](sig) + + output = list_(runas=runas) + + # Used a string here instead of a list because that's what the linux version + # of this module does + pids = '' + for line in output.splitlines(): + if 'PID' in line: + continue + if re.search(name, line): + if line.split()[0].isdigit(): + if pids: + pids += '\n' + pids += line.split()[0] + + return pids + + +def available(name): + ''' + Check that the given service is available. + + :param str name: The name of the service + + :return: True if the service is available, otherwise False + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.available com.openssh.sshd + ''' + try: + _get_service(name) + return True + except CommandExecutionError: + return False + + +def missing(name): + ''' + The inverse of service.available + Check that the given service is not available. + + :param str name: The name of the service + + :return: True if the service is not available, otherwise False + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.missing com.openssh.sshd + ''' + return not available(name) + + +def enabled(name, runas=None): + ''' + Check if the specified service is enabled + + :param str name: The name of the service to look up + + :param str runas: User to run launchctl commands + + :return: True if the specified service enabled, otherwise False + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.enabled org.cups.cupsd + ''' + # Try to list the service. If it can't be listed, it's not enabled + try: + list_(name=name, runas=runas) + return True + except CommandExecutionError: + return False + + +def disabled(name, runas=None): + ''' + Check if the specified service is not enabled. This is the opposite of + ``service.enabled`` + + :param str name: The name to look up + + :param str runas: User to run launchctl commands + + :return: True if the specified service is NOT enabled, otherwise False + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' service.disabled org.cups.cupsd + ''' + # A service is disabled if it is not enabled + return not enabled(name, runas=runas) + + +def get_all(runas=None): + ''' + Return a list of services that are enabled or available. Can be used to + find the name of a service. + + :param str runas: User to run launchctl commands + + :return: A list of all the services available or enabled + :rtype: list + + CLI Example: + + .. code-block:: bash + + salt '*' service.get_all + ''' + # Get list of enabled services + enabled = get_enabled(runas=runas) + + # Get list of all services + available = list(_available_services().keys()) + + # Return composite list + return sorted(set(enabled + available)) + + +def get_enabled(runas=None): + ''' + Return a list of all services that are enabled. Can be used to find the + name of a service. + + :param str runas: User to run launchctl commands + + :return: A list of all the services enabled on the system + :rtype: list + + CLI Example: + + .. code-block:: bash + + salt '*' service.get_enabled + salt '*' service.get_enabled running=True + ''' + # Collect list of enabled services + stdout = list_(runas=runas) + service_lines = [line for line in stdout.splitlines()] + + # Construct list of enabled services + enabled = [] + for line in service_lines: + # Skip header line + if line.startswith('PID'): + continue + + pid, status, label = line.split('\t') + enabled.append(label) + + return sorted(set(enabled)) diff --git a/_states/homebrew_analytics.py b/_states/homebrew_analytics.py new file mode 100644 index 00000000..476ce111 --- /dev/null +++ b/_states/homebrew_analytics.py @@ -0,0 +1,126 @@ +# This module is mainly comprised of new code, but also contains modified +# versions of functions from the salt/modules/mac_brew.py module from Salt +# develop (at git revision 3e5218daea73f3f24b82a3078764ccb82c2a1ec9). +# Functions taken/modified from Salt are marked, all others are original. +# +# The original copyright and licensing notice for the methods from the +# mac_brew.py module is reproduced below in the double-# comment block: +# +## Salt - Remote execution system +## +## Copyright 2014-2015 SaltStack Team +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +''' +Module for the management of homebrew +''' +from __future__ import absolute_import + +# Import python libs +import logging + +# Import salt libs +from salt.exceptions import CommandExecutionError +import salt.utils + +# Set up logging +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only work if Homebrew is installed + ''' + if salt.utils.which('brew'): + return True + return ( + False, + 'The homebrew_analytics state mod could not be loaded: brew not found' + ) + + +def _check_analytics_status(): + out = __salt__['homebrew.cmd_all'](['analytics', 'state'])['stdout'] + if len(out) < 1: + raise CommandExecutionError('Failed to parse brew analytics state') + status_line = out.splitlines()[0] + if 'enabled' in status_line: + return True + elif 'disabled' in status_line: + return False + raise CommandExecutionError('Failed to parse brew analytics state') + + +def managed(name, **kwargs): + ''' + Manage Homebrew analytics state (either enabled or disabled). + + name + Either 'enabled' or 'disabled' + ''' + ret = { + 'name': name, + 'changes': {}, + 'result': None, + 'comment': '', + } + + # Var must be called 'name' due to design of Salt + wanted = None + if name == 'enabled': + wanted = True + elif name == 'disabled': + wanted = False + else: + ret['result'] = False + ret['comment'] = '`name` parameter must be `enabled` or `disabled`' + return ret + wanted_v = name[:-1] # Verb form + + try: + current = _check_analytics_status() + if current == wanted: + ret['result'] = True + ret['comment'] = 'Homebrew analytics are already {}'.format(name) + return ret + + if __opts__['test']: + ret['comment'] = 'Homebrew analytics need to be {}'.format(name) + return ret + + state_arg = 'on' if wanted else 'off' + # Exception bubbles, so we can ignore the return value + __salt__['homebrew.cmd_all'](['analytics', state_arg]) + + new = _check_analytics_status() + if new == wanted: + ret['changes']['homebrew_analytics'] = { + 'old': 'enabled' if current else 'disabled', + 'new': name, + } + ret['result'] = True + ret['comment'] = 'Homebrew analytics was {}'.format(name) + return ret + else: + ret['result'] = False + ret['comment'] = 'Failed to {} Homebrew analytics'.format(wanted_v) + return ret + + except CommandExecutionError as err: + ret['result'] = False + ret['comment'] = 'Failed to {} Homebrew analytics: {}'.format( + wanted_v, + err + ) + return ret diff --git a/buildbot/master/files/config/environments.py b/buildbot/master/files/config/environments.py index d014a90f..917c54d8 100644 --- a/buildbot/master/files/config/environments.py +++ b/buildbot/master/files/config/environments.py @@ -64,6 +64,8 @@ def without(self, to_unset): 'CARGO_HOME': '/Users/servo/.cargo', 'CCACHE': '/usr/local/bin/ccache', 'SERVO_CACHE_DIR': '/Users/servo/.servo', + 'OPENSSL_INCLUDE_DIR': '/usr/local/opt/openssl/include', + 'OPENSSL_LIB_DIR': '/usr/local/opt/openssl/lib', }) diff --git a/osx/init.sls b/osx/init.sls index 9533bc04..7a2d6ccb 100644 --- a/osx/init.sls +++ b/osx/init.sls @@ -12,14 +12,6 @@ - mode: 644 - source: salt://{{ tpldir }}/files/profile -# Disable Homebrew Analytics -# TODO: wrap this up into a proper state that uses the `brew analytics` command -# instead of directly changing the git configuration -# (requires either upstreaming this state + updating Salt, -# or Salting the Salt master) -# TODO: also ensure the `homebrew.analyticsuuid` setting is unset disable-homebrew-analytics: - git.config: - - name: 'homebrew.analyticsdisabled' - - value: 'true' - - repo: /usr/local/Homebrew + homebrew_analytics.managed: + - name: disabled diff --git a/salt/map.jinja b/salt/map.jinja index b31d988c..b3811699 100644 --- a/salt/map.jinja +++ b/salt/map.jinja @@ -1,7 +1,7 @@ {% set salt_ = salt %} {% set salt = { - 'version': '2015.5.8', + 'version': '2016.3.3', 'master': salt['grains.filter_by']({ 'defaults': { 'config': { @@ -19,7 +19,7 @@ 'Ubuntu': { 'pkg': { 'name': 'salt-master', - 'version': '2015.5.8+ds-1' + 'version': '2016.3.3+ds-1' } } }, diff --git a/servo-build-dependencies/android.sls b/servo-build-dependencies/android.sls index c43eb8f2..53ccddfd 100644 --- a/servo-build-dependencies/android.sls +++ b/servo-build-dependencies/android.sls @@ -42,7 +42,10 @@ android-sdk: - source: https://dl.google.com/android/android-sdk_{{ android.sdk.version }}-linux.tgz - source_hash: sha512={{ android.sdk.sha512 }} - archive_format: tar + # Workaround for https://github.com/saltstack/salt/pull/36552 - archive_user: servo + - user: servo + - group: servo - if_missing: {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux - require: - user: servo @@ -56,7 +59,7 @@ android-sdk: eof } ' - - user: servo + - runas: servo - creates: - {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux/platform-tools - {{ common.servo_home }}/android/sdk/{{ android.sdk.version }}/android-sdk-linux/platforms/android-{{ android.platform }} @@ -90,7 +93,7 @@ android-ndk: cmd.run: # Need to filter log output to avoid hitting log limits on Travis CI - name: '{{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}-linux-x86_64.bin | grep -v Extracting' - - user: servo + - runas: servo - cwd: {{ common.servo_home }}/android/ndk/{{ android.ndk.version }} - creates: {{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }} - require: @@ -99,7 +102,7 @@ android-ndk: android-toolchain: cmd.run: - name: bash {{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}/build/tools/make-standalone-toolchain.sh --platform=android-{{ android.platform }} --toolchain=arm-linux-androideabi-4.8 --install-dir='{{ common.servo_home }}/android/toolchain/{{ android.ndk.version }}/android-toolchain' --ndk-dir='{{ common.servo_home }}/android/ndk/{{ android.ndk.version }}/android-ndk-{{ android.ndk.version }}' - - user: servo + - runas: servo - creates: {{ common.servo_home }}/android/toolchain/{{ android.ndk.version }}/android-toolchain - require: - cmd: android-ndk diff --git a/servo-build-dependencies/arm.sls b/servo-build-dependencies/arm.sls index e5ffeae9..09211fc4 100644 --- a/servo-build-dependencies/arm.sls +++ b/servo-build-dependencies/arm.sls @@ -39,7 +39,10 @@ libs-{{ target.name }}: - source: https://servo-rust.s3.amazonaws.com/ARM/{{ target.download_name }}/{{ target.version }}/{{ target.download_name }}-{{ target.version }}.tgz - source_hash: sha512={{ target.sha512 }} - archive_format: tar + # Workaround for https://github.com/saltstack/salt/pull/36552 - archive_user: servo + - user: servo + - group: servo {% for binary in binaries %} {{ common.servo_home }}/bin/{{ target.symlink_name }}-{{ binary }}: diff --git a/servo-build-dependencies/files/install-homebrew-autoconf213.sh b/servo-build-dependencies/files/install-homebrew-autoconf213.sh new file mode 100644 index 00000000..c1437b0e --- /dev/null +++ b/servo-build-dependencies/files/install-homebrew-autoconf213.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# Homebrew insists on being stateful, non-idempotent, lacking useful command +# line flags, and in general hard to administer, so this is a custom script for +# the simple yet impossible via Homebrew CLI task of installing two versions of +# the same package, without an error. +# Specifically, autoconf213 and autoconf need to be installed, with autoconf's +# links taking precedence. + +set -o errexit +set -o nounset +set -o pipefail + + +# Helper methods because brew doesn't like conflicting links during install, +# and there is no way to install w/o linking from the CLI or ignore that error, +# only via editing the Formula to add `keg_only`. +# Hence, just ignore errors for now, and double-check everything at the end. +brew_install() { set +o errexit; brew install "$@"; set -o errexit; } +brew_link() { set +o errexit; brew link "$@"; set -o errexit; } + +# Use "yes"/"no" for conditionals (not "true"/"false") +# to avoid confusion with the true/false commands +autoconf_installed="no" +autoconf_fully_linked="no" +autoconf213_installed="no" +autoconf213_linked="no" + + +set_autoconf_vars() { + if brew list | grep 'autoconf' >/dev/null; then + autoconf_installed="yes" + if readlink '/usr/local/share/info/autoconf.info' \ + | grep 'Cellar/autoconf/' >/dev/null; then + autoconf_fully_linked="yes" + fi + fi +} + + +set_autoconf213_vars() { + if brew list | grep 'autoconf213' >/dev/null; then + autoconf213_installed="yes" + if readlink '/usr/local/bin/autoconf213' \ + | grep 'Cellar/autoconf213/' >/dev/null; then + autoconf213_linked="yes" + fi + fi +} + + +check() { + local verbose="no" + if [[ "$#" -ge 1 && "${1}" == 'verbose' ]]; then + declare -r verbose="yes" + fi + set_autoconf_vars + set_autoconf213_vars + + if [[ "${autoconf213_installed}" == "yes" + && "${autoconf_installed}" == "yes" + && "${autoconf213_linked}" == "yes" + && "${autoconf_fully_linked}" == "yes" ]]; then + return 0 + else + if [[ "${verbose}" == "yes" ]]; then + printf "%s\n" "autoconf/autoconf213 check failed:" + printf "%s %s\n" "autoconf 213 installed?" \ + "${autoconf213_installed}" + printf "%s %s\n" "autoconf installed?" \ + "${autoconf_installed}" + printf "%s %s\n" "autoconf213 linked?" \ + "${autoconf213_linked}" + printf "%s %s\n" "autoconf fully linked?" \ + "${autoconf_fully_linked}" + fi + + return 1 + fi +} + + +main() { + if check; then + return 0 + fi + + # autoconf213 is first so autoconf can override + set_autoconf213_vars + if [[ "${autoconf213_installed}" == "no" ]]; then + brew_install autoconf213 + fi + set_autoconf213_vars + if [[ "${autoconf213_linked}" == "no" ]]; then + brew_link --overwrite autoconf213 + fi + + set_autoconf_vars + if [[ "${autoconf_installed}" == "no" ]]; then + brew_install autoconf + fi + set_autoconf_vars + if [[ "${autoconf_fully_linked}" == "no" ]]; then + brew_link --overwrite autoconf + fi + + check 'verbose' # errexit will handle return in failure case + return 0 +} + + +main "$@" diff --git a/servo-build-dependencies/init.sls b/servo-build-dependencies/init.sls index dce3569d..3c60f1be 100644 --- a/servo-build-dependencies/init.sls +++ b/servo-build-dependencies/init.sls @@ -42,32 +42,19 @@ servo-dependencies: {% if grains['kernel'] == 'Darwin' %} # Workaround for https://github.com/saltstack/salt/issues/26414 -servo-darwin-homebrew-versions-dependencies: - module.run: - - name: pkg.install - - pkgs: - - autoconf213 - - taps: - - homebrew/versions - -# Warning: These states that manually run brew link only check that some -# version of the Homebrew package is linked, not necessarily the version -# linked above. Whether this handles updates properly is an open question. -# These should be replaced by a custom Salt state. -homebrew-link-autoconf: +servo-darwin-tap-homebrew-versions: cmd.run: - - name: 'brew link --overwrite autoconf' - - user: {{ homebrew.user }} - - creates: /usr/local/Library/LinkedKegs/autoconf + - name: 'brew tap homebrew/versions' + - runas: {{ homebrew.user }} + - unless: 'brew tap | grep homebrew/versions' - require: - pkg: servo-dependencies - - module: servo-darwin-homebrew-versions-dependencies -homebrew-link-openssl: - cmd.run: - - name: 'brew link --force openssl' - - user: {{ homebrew.user }} - - creates: /usr/local/Library/LinkedKegs/openssl +# This should be replaced by a custom Salt state. +servo-darwin-install-autoconf213-and-fix-links: + cmd.script: + - source: salt://{{ tpldir }}/files/install-homebrew-autoconf213.sh + - runas: {{ homebrew.user }} - require: - pkg: servo-dependencies {% else %}