Source code for watchmaker

# -*- coding: utf-8 -*-
"""Watchmaker module."""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals, with_statement)

import collections
import datetime
import logging
import os
import platform
import re
import subprocess

import oschmod
import pkg_resources
import setuptools
import yaml
from compatibleversion import check_version

import watchmaker.utils
from watchmaker import static
from watchmaker.exceptions import InvalidValueError, WatchmakerError
from watchmaker.logger import log_system_details
from watchmaker.managers.worker_manager import (LinuxWorkersManager,
                                                WindowsWorkersManager)
from watchmaker.utils import urllib


def _extract_version(package_name):
    try:
        return pkg_resources.get_distribution(package_name).version
    except pkg_resources.DistributionNotFound:
        _conf = setuptools.config.read_configuration(
            os.path.join(
                os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
                'setup.cfg'
            )
        )
        return _conf['metadata']['version']


def _version_info(app_name, version):
    return '{0}/{1} Python/{2} {3}/{4}'.format(
        app_name,
        version,
        platform.python_version(),
        platform.system(),
        platform.release())


__version__ = _extract_version('watchmaker')
VERSION_INFO = _version_info('Watchmaker', __version__)


[docs]class Arguments(dict): """ Create an arguments object for the :class:`watchmaker.Client`. Args: config_path: (:obj:`str`) Path or URL to the Watchmaker configuration file. If ``None``, the default config.yaml file is used. (*Default*: ``None``) log_dir: (:obj:`str`) Path to a directory. If set, Watchmaker logs to a file named ``watchmaker.log`` in the specified directory. Both the directory and the file will be created if necessary. If the file already exists, Watchmaker appends to it rather than overwriting it. If this argument evaluates to ``False``, then logging to a file is disabled. Watchmaker will always output to stdout/stderr. Additionaly, Watchmaker workers may use this directory to keep other log files. (*Default*: ``None``) no_reboot: (:obj:`bool`) Switch to control whether to reboot the system upon a successful execution of :func:`watchmaker.Client.install`. When this parameter is set, Watchmaker will suppress the reboot. Watchmaker automatically suppresses the reboot if it encounters an error. (*Default*: ``False``) log_level: (:obj:`str`) Level to log at. Case-insensitive. Valid options include, from least to most verbose: - ``critical`` - ``error`` - ``warning`` - ``info`` - ``debug`` .. important:: For all **Keyword Arguments**, below, the default value of ``None`` means Watchmaker will get the value from the configuration file. Be aware that ``None`` and ``'None'`` are two different values, with different meanings and effects. Keyword Arguments: admin_groups: (:obj:`str`) Set a salt grain that specifies the domain _groups_ that should have root privileges on Linux or admin privileges on Windows. Value must be a colon-separated string. On Linux, use the ``^`` to denote spaces in the group name. (*Default*: ``None``) .. code-block:: python admin_groups = "group1:group2" # (Linux only) The group names must be lowercased. Also, if # there are spaces in a group name, replace the spaces with a # '^'. admin_groups = "space^out" # (Windows only) No special capitalization nor syntax # requirements. admin_groups = "Space Out" admin_users: (:obj:`str`) Set a salt grain that specifies the domain _users_ that should have root privileges on Linux or admin privileges on Windows. Value must be a colon-separated string. (*Default*: ``None``) .. code-block:: python admin_users = "user1:user2" computer_name: (:obj:`str`) Set a salt grain that specifies the computername to apply to the system. (*Default*: ``None``) environment: (:obj:`str`) Set a salt grain that specifies the environment in which the system is being built. For example: ``dev``, ``test``, or ``prod``. (*Default*: ``None``) salt_states: (:obj:`str`) Comma-separated string of salt states to apply. A value of ``None`` will not apply any salt states. A value of ``'Highstate'`` will apply the salt highstate. (*Default*: ``None``) ou_path: (:obj:`str`) Set a salt grain that specifies the full DN of the OU where the computer account will be created when joining a domain. (*Default*: ``None``) .. code-block:: python ou_path="OU=Super Cool App,DC=example,DC=com" extra_arguments: (:obj:`list`) A list of extra arguments to be merged into the worker configurations. The list must be formed as pairs of named arguments and values. Any leading hypens in the argument name are stripped. (*Default*: ``[]``) .. code-block:: python extra_arguments=['--arg1', 'value1', '--arg2', 'value2'] # This list would be converted to the following dict and merged # into the parameters passed to the worker configurations: {'arg1': 'value1', 'arg2': 'value2'} """ DEFAULT_VALUE = 'WAM_NONE' def __init__( self, config_path=None, log_dir=None, no_reboot=False, log_level=None, *args, **kwargs ): super(Arguments, self).__init__(*args, **kwargs) self.config_path = config_path self.log_dir = log_dir self.no_reboot = no_reboot self.log_level = log_level self.admin_groups = watchmaker.utils.clean_none( kwargs.pop('admin_groups', None) or Arguments.DEFAULT_VALUE) self.admin_users = watchmaker.utils.clean_none( kwargs.pop('admin_users', None) or Arguments.DEFAULT_VALUE) self.computer_name = watchmaker.utils.clean_none( kwargs.pop('computer_name', None) or Arguments.DEFAULT_VALUE) self.environment = watchmaker.utils.clean_none( kwargs.pop('environment', None) or Arguments.DEFAULT_VALUE) self.salt_states = watchmaker.utils.clean_none( kwargs.pop('salt_states', None) or Arguments.DEFAULT_VALUE) self.ou_path = watchmaker.utils.clean_none( kwargs.pop('ou_path', None) or Arguments.DEFAULT_VALUE) # Parse extra_arguments passed as `--argument=value` into separate # tokens, ['--argument', 'value']. This way the `=` as the separator is # treated the same as the ` ` as the separator, e.g. `--argument value` extra_arguments = [] for item in kwargs.pop('extra_arguments', None) or []: match = re.match('^(?P<arg>-+.*?)=(?P<val>.*)', item) if match: # item is using '=' to separate argument name and value extra_arguments.extend([ match.group('arg'), watchmaker.utils.clean_none( match.group('val') or Arguments.DEFAULT_VALUE ) ]) elif item.startswith('-'): # item is the argument name extra_arguments.extend([item]) else: # item is the argument value extra_arguments.extend([ watchmaker.utils.clean_none( item or Arguments.DEFAULT_VALUE ) ]) self.extra_arguments = extra_arguments def __getattr__(self, attr): """Support attr-notation for getting dict contents.""" return super(Arguments, self).__getitem__(attr) def __setattr__(self, attr, value): """Support attr-notation for setting dict contents.""" super(Arguments, self).__setitem__(attr, value)
[docs]class Client(object): """ Prepare a system for setup and installation. Keyword Arguments: arguments: (:obj:`Arguments`) A dictionary of arguments. See :class:`watchmaker.Arguments`. """ def __init__(self, arguments): self.log = logging.getLogger( '{0}.{1}'.format(__name__, self.__class__.__name__) ) # Pop extra_arguments now so we can log it separately extra_arguments = arguments.pop('extra_arguments', []) header = ' WATCHMAKER RUN ' header = header.rjust((40 + len(header) // 2), '#').ljust(80, '#') self.log.info(header) self.log.debug('Watchmaker Version: %s', __version__) self.log.debug('Parameters: %s', arguments) self.log.debug('Extra Parameters: %s', extra_arguments) # Pop remaining arguments used by watchmaker.Client itself self.default_config = os.path.join(static.__path__[0], 'config.yaml') self.no_reboot = arguments.pop('no_reboot', False) self.config_path = arguments.pop('config_path') self.log_dir = arguments.pop('log_dir') self.log_level = arguments.pop('log_level') log_system_details(self.log) # Get the system params self.system = platform.system().lower() self._set_system_params() self.log.debug('System Parameters: %s', self.system_params) # All remaining arguments are worker_args worker_args = arguments # Convert extra_arguments to a dict and merge it with worker_args. # Leading hypens are removed, and other hyphens are converted to # underscores worker_args.update(dict( (k.lstrip('-').replace('-', '_'), v) for k, v in zip( *[iter(extra_arguments)] * 2) )) try: # Set self.worker_args, removing `None` values from worker_args self.worker_args = dict( (k, yaml.safe_load('null' if v is None else v)) for k, v in worker_args.items() if v != Arguments.DEFAULT_VALUE ) except yaml.YAMLError as exc: if hasattr(exc, "problem_mark"): msg = ( "Failed to parse argument value as YAML. Check the format " "and/or properly quote the value when using the CLI to " "account for shell interpolation. YAML error: {0}" ).format(str(exc)) self.log.critical(msg) raise InvalidValueError(msg) raise self.config = self._get_config() def _get_config(self): """ Read and validate configuration data for installation. Returns: :obj:`collections.OrderedDict`: Returns the data from the the YAML configuration file, scoped to the value of ``self.system`` and merged with the value of the ``"All"`` key. """ if not self.config_path: self.log.warning( 'User did not supply a config. Using the default config.' ) self.config_path = self.default_config else: self.log.info('User supplied config being used.') # Convert a local config path to a URI self.config_path = watchmaker.utils.uri_from_filepath(self.config_path) # Get the raw config data data = '' try: data = watchmaker.utils.urlopen_retry(self.config_path).read() except (ValueError, urllib.error.URLError): msg = ( 'Could not read config file from the provided value "{0}"! ' 'Check that the config is available.'.format(self.config_path) ) self.log.critical(msg) raise config_full = yaml.safe_load(data) try: config_all = config_full.get('all', []) config_system = config_full.get(self.system, []) config_version_specifier = config_full.get( 'watchmaker_version', None) except AttributeError: msg = 'Malformed config file. Must be a dictionary.' self.log.critical(msg) raise # If both config and config_system are empty, raise if not config_system and not config_all: msg = 'Malformed config file. No workers for this system.' self.log.critical(msg) raise WatchmakerError(msg) if config_version_specifier and not check_version( watchmaker.__version__, config_version_specifier): msg = ( 'Watchmaker version {} is not compatible with the config ' 'file (watchmaker_version = {})').format( watchmaker.__version__, config_version_specifier) self.log.critical(msg) raise WatchmakerError(msg) # Merge the config data, preserving the listed order of workers. # The worker order from config_system has precedence over config_all. # This is managed by adding config_system to the config first, using # the loop order, e.g. config_system + config_all. In the loop, if the # worker is already in the config, it is always the worker from # config_system. # To also preserve precedence of worker options from config_system, the # worker_config from config_all is updated with the config from # config_system, then the config is replaced with the worker_config. config = collections.OrderedDict() for worker in config_system + config_all: try: # worker is a single-key dict, where the key is the name of the # worker and the value is the worker parameters. we need to # test if the worker is already in the config, but a dict is # is not hashable so cannot be tested directly with # `if worker not in config`. this bit of ugliness extracts the # key and its value so we can use them directly. worker_name, worker_config = list(worker.items())[0] if worker_name not in config: # Add worker to config config[worker_name] = {'config': worker_config} self.log.debug('%s config: %s', worker_name, worker_config) else: # Worker is present in both config_system and config_all, # config[worker_name]['config'] is from config_system, # worker_config is from config_all worker_config.update(config[worker_name]['config']) config[worker_name]['config'] = worker_config self.log.debug( '%s extra config: %s', worker_name, worker_config ) # Need to (re)merge cli worker args so they override config[worker_name]['__merged'] = False if not config[worker_name].get('__merged'): # Merge worker_args into config params config[worker_name]['config'].update(self.worker_args) config[worker_name]['__merged'] = True except Exception: msg = ( 'Failed to merge worker config; worker={0}' .format(worker) ) self.log.critical(msg) raise self.log.debug( 'Command-line arguments merged into worker configs: %s', self.worker_args ) return config def _get_linux_system_params(self): """Set ``self.system_params`` attribute for Linux systems.""" params = {} params['prepdir'] = os.path.join( '{0}'.format(self.system_drive), 'usr', 'tmp', 'watchmaker') params['readyfile'] = os.path.join( '{0}'.format(self.system_drive), 'var', 'run', 'system-is-ready') params['logdir'] = os.path.join( '{0}'.format(self.system_drive), 'var', 'log') params['workingdir'] = os.path.join( '{0}'.format(params['prepdir']), 'workingfiles') params['restart'] = 'shutdown -r +1 &' return params def _get_windows_system_params(self): """Set ``self.system_params`` attribute for Windows systems.""" params = {} # os.path.join does not produce path as expected when first string # ends in colon; so using a join on the sep character. params['prepdir'] = os.path.sep.join([self.system_drive, 'Watchmaker']) params['readyfile'] = os.path.join( '{0}'.format(params['prepdir']), 'system-is-ready') params['logdir'] = os.path.join( '{0}'.format(params['prepdir']), 'Logs') params['workingdir'] = os.path.join( '{0}'.format(params['prepdir']), 'WorkingFiles') params['shutdown_path'] = os.path.join( '{0}'.format(os.environ['SYSTEMROOT']), 'system32', 'shutdown.exe') params['restart'] = params["shutdown_path"] + \ ' /r /t 30 /d p:2:4 /c ' + \ '"Watchmaker complete. Rebooting computer."' return params def _set_system_params(self): """Set OS-specific attributes.""" if 'linux' in self.system: self.system_drive = '/' self.workers_manager = LinuxWorkersManager self.system_params = self._get_linux_system_params() os.umask(0o077) elif 'windows' in self.system: self.system_drive = os.environ['SYSTEMDRIVE'] self.workers_manager = WindowsWorkersManager self.system_params = self._get_windows_system_params() else: msg = 'System, {0}, is not recognized?'.format(self.system) self.log.critical(msg) raise WatchmakerError(msg) if self.log_dir: self.system_params['logdir'] = self.log_dir
[docs] def install(self): """ Execute the watchmaker workers against the system. Upon successful execution, the system will be properly provisioned, according to the defined configuration and workers. """ self.log.info('Start time: %s', datetime.datetime.now()) self.log.info('Workers to execute: %s', self.config.keys()) # Create watchmaker directories try: os.makedirs(self.system_params['workingdir']) oschmod.set_mode(self.system_params['prepdir'], 0o700) except OSError: if not os.path.exists(self.system_params['workingdir']): msg = ( 'Unable to create directory - {0}' .format(self.system_params['workingdir']) ) self.log.critical(msg) raise workers_manager = self.workers_manager( system_params=self.system_params, workers=self.config ) try: workers_manager.worker_cadence() except Exception: msg = 'Execution of the workers cadence has failed.' self.log.critical(msg) raise if self.no_reboot: self.log.info( 'Detected `no-reboot` switch. System will not be ' 'rebooted.' ) else: self.log.info( 'Reboot scheduled. System will reboot after the script ' 'exits.' ) subprocess.call(self.system_params['restart'], shell=True) self.log.info('Stop time: %s', datetime.datetime.now())