Global config object (#633)

* Add global `pyinfra.config` pseudo module.

* Implement `__setattr__` for the pseudo modules.

* Move config defaults into a dictionary rather than class attributes.

* Simplify config extraction from files, update for new defaults.

* Log a warning when loading config variables directly from files.

* Implement config state locking.

This provides two guarantees:

+ When executing multiple deploy files via the CLI, any config changes
  in them does not affect the others (this has not existed previously).
+ When including files, any config changes in them are only held during
  the execution of the file, and removed after. This matches the current
  include config logic.

* Linting fixes.

Author: Fizzadar

pyinfra/api/config.py

@@ -1,66 +1,87 @@
 import six
 
 
-class Config(object):
-    '''
-    The default/base configuration options for a pyinfra deploy.
-    '''
-
-    state = None
-
+config_defaults = {
     # % of hosts which have to fail for all operations to stop
-    FAIL_PERCENT = None
+    'FAIL_PERCENT': None,
 
     # Seconds to timeout SSH connections
-    CONNECT_TIMEOUT = 10
+    'CONNECT_TIMEOUT': 10,
 
     # Temporary directory (on the remote side) to use for caching any files/downloads
-    TEMP_DIR = '/tmp'
+    'TEMP_DIR': '/tmp',
 
     # Gevent pool size (defaults to #of target hosts)
-    PARALLEL = None
+    'PARALLEL': None,
 
     # Specify the required pyinfra version (using PEP 440 setuptools specifier)
-    REQUIRE_PYINFRA_VERSION = None
+    'REQUIRE_PYINFRA_VERSION': None,
     # Specify any required packages (either using PEP 440 or a requirements file)
     # Note: this can also include pyinfra, potentially replacing REQUIRE_PYINFRA_VERSION
-    REQUIRE_PACKAGES = None
+    'REQUIRE_PACKAGES': None,
 
     # COMPAT w/<1.1
     # TODO: remove this in favour of above at v2
     # Specify a minimum required pyinfra version for a deploy
-    MIN_PYINFRA_VERSION = None
+    'MIN_PYINFRA_VERSION': None,
 
     # All these can be overridden inside individual operation calls:
 
     # Switch to this user (from ssh_user) using su before executing operations
-    SU_USER = None
-    USE_SU_LOGIN = False
-    SU_SHELL = None
-    PRESERVE_SU_ENV = False
+    'SU_USER': None,
+    'USE_SU_LOGIN': False,
+    'SU_SHELL': None,
+    'PRESERVE_SU_ENV': False,
 
     # Use sudo and optional user
-    SUDO = False
-    SUDO_USER = None
-    PRESERVE_SUDO_ENV = False
-    USE_SUDO_LOGIN = False
-    USE_SUDO_PASSWORD = False
+    'SUDO': False,
+    'SUDO_USER': None,
+    'PRESERVE_SUDO_ENV': False,
+    'USE_SUDO_LOGIN': False,
+    'USE_SUDO_PASSWORD': False,
 
     # Use doas and optional user
-    DOAS = False
-    DOAS_USER = None
+    'DOAS': False,
+    'DOAS_USER': None,
 
     # Only show errors, but don't count as failure
-    IGNORE_ERRORS = False
+    'IGNORE_ERRORS': False,
 
     # Shell to use to execute commands
-    SHELL = None
+    'SHELL': None,
+}
+
+
+class Config(object):
+    '''
+    The default/base configuration options for a pyinfra deploy.
+    '''
+
+    state = None
 
     def __init__(self, **kwargs):
         # Always apply some env
         env = kwargs.pop('ENV', {})
         self.ENV = env
 
-        # Apply kwargs
-        for key, value in six.iteritems(kwargs):
+        config = config_defaults.copy()
+        config.update(kwargs)
+
+        for key, value in six.iteritems(config):
             setattr(self, key, value)
+
+    def get_current_state(self):
+        return [
+            (key, getattr(self, key))
+            for key in config_defaults.keys()
+        ]
+
+    def set_current_state(self, config_state):
+        for key, value in config_state:
+            setattr(self, key, value)
+
+    def lock_current_sate(self):
+        self._locked_config = self.get_current_state()
+
+    def reset_locked_state(self):
+        self.set_current_state(self._locked_config)

pyinfra/api/connectors/util.py

@@ -12,7 +12,7 @@
 from six.moves import shlex_quote
 
 from pyinfra import logger
-from pyinfra.api import Config, MaskString, QuoteString, StringCommand
+from pyinfra.api import MaskString, QuoteString, StringCommand
 from pyinfra.api.util import memoize
 
 SUDO_ASKPASS_ENV_VAR = 'PYINFRA_SUDO_PASSWORD'
@@ -214,21 +214,21 @@ def make_unix_command(
     command,
     env=None,
     chdir=None,
-    shell_executable=Config.SHELL,
+    shell_executable=None,
     # Su config
-    su_user=Config.SU_USER,
-    use_su_login=Config.USE_SU_LOGIN,
-    su_shell=Config.SU_SHELL,
-    preserve_su_env=Config.PRESERVE_SU_ENV,
+    su_user=None,
+    use_su_login=False,
+    su_shell=None,
+    preserve_su_env=False,
     # Sudo config
-    sudo=Config.SUDO,
-    sudo_user=Config.SUDO_USER,
-    use_sudo_login=Config.USE_SUDO_LOGIN,
-    use_sudo_password=Config.USE_SUDO_PASSWORD,
-    preserve_sudo_env=Config.PRESERVE_SUDO_ENV,
+    sudo=False,
+    sudo_user=None,
+    use_sudo_login=False,
+    use_sudo_password=False,
+    preserve_sudo_env=False,
     # Doas config
-    doas=Config.DOAS,
-    doas_user=Config.DOAS_USER,
+    doas=False,
+    doas_user=None,
     # Optional state object, used to decide if we print invalid auth arg warnings
     state=None,
 ):

pyinfra/api/connectors/winrm.py

@@ -6,7 +6,6 @@
 import click
 
 from pyinfra import logger
-from pyinfra.api import Config
 from pyinfra.api.exceptions import ConnectError, PyinfraError
 from pyinfra.api.util import get_file_io, memoize, sha1_hash
 
@@ -108,7 +107,7 @@ def run_shell_command(
     print_output=False,
     print_input=False,
     return_combined_output=False,
-    shell_executable=Config.SHELL,
+    shell_executable=None,
     **ignored_command_kwargs
 ):
     '''

pyinfra/local.py

@@ -7,7 +7,7 @@
 
 import pyinfra
 
-from . import logger, pseudo_state
+from . import logger, pseudo_config, pseudo_state
 from .api.connectors.util import run_local_process, split_combined_output
 from .api.exceptions import PyinfraError
 
@@ -28,6 +28,8 @@ def include(filename):
 
     logger.debug('Including local file: {0}'.format(filename))
 
+    config_state = pseudo_config.get_current_state()
+
     try:
         # Fixes a circular import because `pyinfra.local` is really a CLI
         # only thing (so should be `pyinfra_cli.local`). It is kept here
@@ -38,6 +40,7 @@ def include(filename):
         from pyinfra_cli.util import exec_file
 
         # Load any config defined in the file and setup like a @deploy
+        # TODO: remove this in v2
         config_data = extract_file_config(filename)
         kwargs = {
             key.lower(): value
@@ -58,6 +61,9 @@ def include(filename):
             'Could not include local file: {0}:\n{1}'.format(filename, e),
         )
 
+    finally:
+        pseudo_config.set_current_state(config_state)
+
 
 def shell(commands, splitlines=False, ignore_errors=False, print_output=False, print_input=False):
     '''

pyinfra/pseudo_modules.py

@@ -35,6 +35,15 @@ def __getattr__(self, key):
             return getattr(self._base_module, key)
         return getattr(self._module, key)
 
+    def __setattr__(self, key, value):
+        if key in ('_module', '_base_module'):
+            return super(PseudoModule, self).__setattr__(key, value)
+
+        if self._module is None:
+            raise TypeError('Cannot assign to pseudo base module')
+
+        return setattr(self._module, key, value)
+
     def __iter__(self):
         return iter(self._module)
 
@@ -63,6 +72,12 @@ def isset(self):
     pyinfra.pseudo_state = pyinfra.state = \
     PseudoModule()
 
+# The current deploy config
+pseudo_config = \
+    sys.modules['pyinfra.pseudo_config'] = sys.modules['pyinfra.config'] = \
+    pyinfra.pseudo_config = pyinfra.config = \
+    PseudoModule()
+
 # The current deploy inventory
 pseudo_inventory = \
     sys.modules['pyinfra.pseudo_inventory'] = sys.modules['pyinfra.inventory'] = \
@@ -77,8 +92,9 @@ def isset(self):
 
 
 def init_base_classes():
-    from pyinfra.api import Host, Inventory, State
+    from pyinfra.api import Config, Host, Inventory, State
 
+    pseudo_config.set_base(Config)
     pseudo_host.set_base(Host)
     pseudo_inventory.set_base(Inventory)
     pseudo_state.set_base(State)

pyinfra_cli/config.py

@@ -1,12 +1,9 @@
 import ast
 
-from os import path
-
 import six
 
-from pyinfra.api import Config
-
-from .util import exec_file
+from pyinfra import logger
+from pyinfra.api.config import config_defaults
 
 
 def extract_file_config(filename, config=None):
@@ -48,7 +45,13 @@ def extract_file_config(filename, config=None):
         # If one of the assignments matches a config variable (e.g. SUDO = True)
         # then assign it to the config object!
         for target in node.targets:
-            if target.id.isupper() and hasattr(Config, target.id):
+            if not isinstance(target, ast.Name):
+                continue
+            if target.id.isupper() and target.id in config_defaults:
+                logger.warning((
+                    'file: {0}\n\tDefining config variables directly is deprecated, '
+                    'please use `config.{1} = {2}`.'
+                ).format(filename, target.id, repr(value)))
                 config_data[target.id] = value
 
     # If we have a config, update and exit
@@ -58,37 +61,3 @@ def extract_file_config(filename, config=None):
         return
 
     return config_data
-
-
-def load_config(deploy_dir):
-    '''
-    Loads any local config.py file.
-    '''
-
-    config = Config()
-    config_filename = path.join(deploy_dir, 'config.py')
-
-    if path.exists(config_filename):
-        extract_file_config(config_filename, config)
-
-        # Now execute the file to trigger loading of any hooks
-        exec_file(config_filename)
-
-    return config
-
-
-def load_deploy_config(deploy_filename, config=None):
-    '''
-    Loads any local config overrides in the deploy file.
-    '''
-
-    if not config:
-        config = Config()
-
-    if not deploy_filename:
-        return
-
-    if path.exists(deploy_filename):
-        extract_file_config(deploy_filename, config)
-
-    return config

pyinfra_cli/main.py

@@ -12,10 +12,11 @@
 from pyinfra import (
     __version__,
     logger,
+    pseudo_config,
     pseudo_inventory,
     pseudo_state,
 )
-from pyinfra.api import State
+from pyinfra.api import Config, State
 from pyinfra.api.connect import connect_all, disconnect_all
 from pyinfra.api.exceptions import NoGroupError, PyinfraError
 from pyinfra.api.facts import (
@@ -29,7 +30,7 @@
 from pyinfra.api.util import get_kwargs_str
 from pyinfra.operations import server
 
-from .config import load_config, load_deploy_config
+from .config import extract_file_config
 from .exceptions import (
     CliError,
     UnexpectedExternalError,
@@ -49,6 +50,7 @@
     print_support_info,
 )
 from .util import (
+    exec_file,
     get_facts_and_args,
     get_operation_and_args,
     list_dirs_above_file,
@@ -346,8 +348,16 @@ def _main(
     if not quiet:
         click.echo('--> Loading config...', err=True)
 
+    config = Config()
+    pseudo_config.set(config)
+
     # Load up any config.py from the filesystem
-    config = load_config(deploy_dir)
+    config_filename = path.join(deploy_dir, 'config.py')
+    if path.exists(config_filename):
+        extract_file_config(config_filename, config)  # TODO: remove this
+        exec_file(config_filename)
+
+    # TODO: lock the config here, moving up from below when possible (v2)
 
     # Make a copy before we overwrite
     original_operations = operations
@@ -409,9 +419,14 @@ def _main(
     pyinfra INVENTORY exec -- echo "hello world"
     pyinfra INVENTORY fact os [users]...'''.format(operations))
 
-    # Load any hooks/config from the deploy file
+    # TODO: remove this - legacy load of any config variables from the top of
+    # the first deploy file.
     if command == 'deploy':
-        load_deploy_config(operations[0], config)
+        extract_file_config(operations[0], config)
+
+    # Lock the current config, this allows us to restore this version after
+    # executing deploy files that may alter them.
+    config.lock_current_sate()
 
     # Arg based config overrides
     if sudo:
@@ -577,6 +592,8 @@ def _main(
         for i, filename in enumerate(operations):
             logger.info('Loading: {0}'.format(click.style(filename, bold=True)))
             load_deploy_file(state, filename)
+            # Remove any config changes introduced by the deploy file & any includes
+            config.reset_locked_state()
 
     # Operation w/optional args
     elif command == 'op':