Merge branch 'master' into schedule_modify_no_function_fix

Author: dwoz

salt/engines/stalekey.py

@@ -4,13 +4,16 @@
 have been recently connected and remove their keys if they have not been
 connected for a certain period of time.
 
-Requires that the minion_data_cache option be enabled.
+Requires that the :conf_master:`minion_data_cache` option be enabled.
 
 .. versionadded: 2017.7.0
 
 :configuration:
 
-    Example configuration
+    Example configuration:
+
+    .. code-block:: yaml
+
         engines:
           - stalekey:
               interval: 3600

salt/modules/disk.py

@@ -268,24 +268,34 @@ def percent(args=None):
 
 
 @salt.utils.decorators.path.which('blkid')
-def blkid(device=None):
+def blkid(device=None, token=None):
     '''
     Return block device attributes: UUID, LABEL, etc. This function only works
     on systems where blkid is available.
 
+    device
+        Device name from the system
+
+    token
+        Any valid token used for the search
+
     CLI Example:
 
     .. code-block:: bash
 
         salt '*' disk.blkid
         salt '*' disk.blkid /dev/sda
+        salt '*' disk.blkid token='UUID=6a38ee5-7235-44e7-8b22-816a403bad5d'
+        salt '*' disk.blkid token='TYPE=ext4'
     '''
-    args = ""
+    cmd = ['blkid']
     if device:
-        args = " " + device
+        cmd.append(device)
+    elif token:
+        cmd.extend(['-t', token])
 
     ret = {}
-    blkid_result = __salt__['cmd.run_all']('blkid' + args, python_shell=False)
+    blkid_result = __salt__['cmd.run_all'](cmd, python_shell=False)
 
     if blkid_result['retcode'] > 0:
         return ret

salt/modules/virt.py

@@ -112,6 +112,7 @@
 from salt.exceptions import CommandExecutionError, SaltInvocationError
 from salt.ext import six
 from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-builtin
+from salt._compat import ipaddress
 
 log = logging.getLogger(__name__)
 
@@ -663,7 +664,8 @@ def _gen_net_xml(name,
                  bridge,
                  forward,
                  vport,
-                 tag=None):
+                 tag=None,
+                 ip_configs=None):
     '''
     Generate the XML string to define a libvirt network
     '''
@@ -673,6 +675,10 @@ def _gen_net_xml(name,
         'forward': forward,
         'vport': vport,
         'tag': tag,
+        'ip_configs': [{
+            'address': ipaddress.ip_network(config['cidr']),
+            'dhcp_ranges': config.get('dhcp_ranges', []),
+        } for config in ip_configs or []],
     }
     fn_ = 'libvirt_network.jinja'
     try:
@@ -4402,7 +4408,12 @@ def cpu_baseline(full=False, migratable=False, out='libvirt', **kwargs):
     return cpu.toxml()
 
 
-def network_define(name, bridge, forward, **kwargs):
+def network_define(name,
+                   bridge,
+                   forward,
+                   ipv4_config=None,
+                   ipv6_config=None,
+                   **kwargs):
     '''
     Create libvirt network.
 
@@ -4413,10 +4424,38 @@ def network_define(name, bridge, forward, **kwargs):
     :param tag: Vlan tag
     :param autostart: Network autostart (default True)
     :param start: Network start (default True)
+    :param ipv4_config: IP v4 configuration
+        Dictionary describing the IP v4 setup like IP range and
+        a possible DHCP configuration. The structure is documented
+        in net-define-ip_.
+
+        ..versionadded:: Neon
+    :type ipv4_config: dict or None
+
+    :param ipv6_config: IP v6 configuration
+        Dictionary describing the IP v6 setup like IP range and
+        a possible DHCP configuration. The structure is documented
+        in net-define-ip_.
+
+        ..versionadded:: Neon
+    :type ipv6_config: dict or None
+
     :param connection: libvirt connection URI, overriding defaults
     :param username: username to connect with, overriding defaults
     :param password: password to connect with, overriding defaults
 
+    .. _net-define-ip:
+
+    ** IP configuration definition
+
+    Both the IPv4 and IPv6 configuration dictionaries can contain the following properties:
+
+    cidr
+        CIDR notation for the network. For example '192.168.124.0/24'
+
+    dhcp_ranges
+        A list of dictionary with ``'start'`` and ``'end'`` properties.
+
     CLI Example:
 
     .. code-block:: bash
@@ -4430,12 +4469,14 @@ def network_define(name, bridge, forward, **kwargs):
     tag = kwargs.get('tag', None)
     autostart = kwargs.get('autostart', True)
     starting = kwargs.get('start', True)
+
     net_xml = _gen_net_xml(
         name,
         bridge,
         forward,
         vport,
-        tag,
+        tag=tag,
+        ip_configs=[config for config in [ipv4_config, ipv6_config] if config],
     )
     try:
         conn.networkDefineXML(net_xml)

salt/modules/zfs.py

@@ -1145,8 +1145,8 @@ def set(*dataset, **kwargs):
     res = __salt__['cmd.run_all'](
         __utils__['zfs.zfs_command'](
             command='set',
-            property_name=filesystem_properties.keys(),
-            property_value=filesystem_properties.values(),
+            property_name=list(filesystem_properties.keys()),
+            property_value=list(filesystem_properties.values()),
             target=list(dataset),
         ),
         python_shell=False,

salt/runners/cloud.py

@@ -179,8 +179,8 @@ def create(provider, instances, opts=None, **kwargs):
 
     .. code-block:: bash
 
-        salt-run cloud.create my-ec2-config myinstance \
-            image=ami-1624987f size='t1.micro' ssh_username=ec2-user \
+        salt-run cloud.create my-ec2-config myinstance \\
+            image=ami-1624987f size='t1.micro' ssh_username=ec2-user \\
             securitygroup=default delvol_on_destroy=True
     '''
     client = _get_client()

salt/states/network.py

@@ -164,7 +164,7 @@
         - max_bonds: 1
         - updelay: 0
         - use_carrier: on
-        - xmit_hash_policy: layer2
+        - hashing-algorithm: layer2
         - mtu: 9000
         - autoneg: on
         - speed: 1000

salt/states/virt.py

@@ -658,13 +658,34 @@ def network_running(name,
                     forward,
                     vport=None,
                     tag=None,
+                    ipv4_config=None,
+                    ipv6_config=None,
                     autostart=True,
                     connection=None,
                     username=None,
                     password=None):
     '''
     Defines and starts a new network with specified arguments.
 
+    :param bridge: Bridge name
+    :param forward: Forward mode(bridge, router, nat)
+    :param vport: Virtualport type (Default: ``'None'``)
+    :param tag: Vlan tag (Default: ``'None'``)
+    :param ipv4_config:
+        IPv4 network configuration. See the :py:func`virt.network_define
+        <salt.modules.virt.network_define>` function corresponding parameter documentation
+        for more details on this dictionary.
+        (Default: None).
+
+        .. versionadded:: neon
+    :param ipv6_config:
+        IPv6 network configuration. See the :py:func`virt.network_define
+        <salt.modules.virt.network_define>` function corresponding parameter documentation
+        for more details on this dictionary.
+        (Default: None).
+
+        .. versionadded:: neon
+    :param autostart: Network autostart (default ``'True'``)
     :param connection: libvirt connection URI, overriding defaults
 
         .. versionadded:: 2019.2.0
@@ -690,6 +711,21 @@ def network_running(name,
             - tag: 180
             - autostart: True
 
+    .. code-block:: yaml
+
+        network_name:
+          virt.network_define:
+            - bridge: natted
+            - forward: nat
+            - ipv4_config:
+                cidr: 192.168.42.0/24
+                dhcp_ranges:
+                  - start: 192.168.42.10
+                    end: 192.168.42.25
+                  - start: 192.168.42.100
+                    end: 192.168.42.150
+            - autostart: True
+
     '''
     ret = {'name': name,
            'changes': {},
@@ -712,6 +748,8 @@ def network_running(name,
                                             forward,
                                             vport=vport,
                                             tag=tag,
+                                            ipv4_config=ipv4_config,
+                                            ipv6_config=ipv6_config,
                                             autostart=autostart,
                                             start=True,
                                             connection=connection,

salt/templates/virt/libvirt_network.jinja

@@ -6,4 +6,15 @@
   <vlan>
     <tag id='{{ tag }}'/>
   </vlan>{% endif %}
-</network>
+  {% for ip_config in ip_configs %}
+  <ip family='ipv{{ ip_config.address.version }}'
+      address='{{ ip_config.address.network_address }}'
+      prefix='{{ ip_config.address.prefixlen }}'>
+    <dhcp>
+      {% for range in ip_config.dhcp_ranges %}
+      <range start='{{ range.start }}' end='{{ range.end }}' />
+      {% endfor %}
+    </dhcp>
+  </ip>
+  {% endfor %}
+</network>

salt/utils/roster_matcher.py

@@ -5,10 +5,11 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 # Import python libs
+import copy
 import fnmatch
+import functools
 import logging
 import re
-import copy
 
 # Try to import range from https://github.com/ytoolshed/range
 HAS_RANGE = False
@@ -31,6 +32,18 @@ def targets(conditioned_raw, tgt, tgt_type, ipv='ipv4'):
     return rmatcher.targets()
 
 
+def _tgt_set(tgt):
+    '''
+    Return the tgt as a set of literal names
+    '''
+    try:
+        # A comma-delimited string
+        return set(tgt.split(','))
+    except AttributeError:
+        # Assume tgt is already a non-string iterable.
+        return set(tgt)
+
+
 class RosterMatcher(object):
     '''
     Matcher for the roster data structure
@@ -50,59 +63,47 @@ def targets(self):
         except AttributeError:
             return {}
 
-    def ret_glob_minions(self):
+    def _ret_minions(self, filter_):
         '''
-        Return minions that match via glob
+        Filter minions by a generic filter.
         '''
         minions = {}
-        for minion in self.raw:
-            if fnmatch.fnmatch(minion, self.tgt):
-                data = self.get_data(minion)
-                if data:
-                    minions[minion] = data.copy()
+        for minion in filter_(self.raw):
+            data = self.get_data(minion)
+            if data:
+                minions[minion] = data.copy()
         return minions
 
+    def ret_glob_minions(self):
+        '''
+        Return minions that match via glob
+        '''
+        fnfilter = functools.partial(fnmatch.filter, pat=self.tgt)
+        return self._ret_minions(fnfilter)
+
     def ret_pcre_minions(self):
         '''
         Return minions that match via pcre
         '''
-        minions = {}
-        for minion in self.raw:
-            if re.match(self.tgt, minion):
-                data = self.get_data(minion)
-                if data:
-                    minions[minion] = data.copy()
-        return minions
+        tgt = re.compile(self.tgt)
+        refilter = functools.partial(filter, tgt.match)
+        return self._ret_minions(refilter)
 
     def ret_list_minions(self):
         '''
         Return minions that match via list
         '''
-        minions = {}
-        if not isinstance(self.tgt, list):
-            self.tgt = self.tgt.split(',')
-        for minion in self.raw:
-            if minion in self.tgt:
-                data = self.get_data(minion)
-                if data:
-                    minions[minion] = data.copy()
-        return minions
+        tgt = _tgt_set(self.tgt)
+        return self._ret_minions(tgt.intersection)
 
     def ret_nodegroup_minions(self):
         '''
         Return minions which match the special list-only groups defined by
         ssh_list_nodegroups
         '''
-        minions = {}
         nodegroup = __opts__.get('ssh_list_nodegroups', {}).get(self.tgt, [])
-        if not isinstance(nodegroup, list):
-            nodegroup = nodegroup.split(',')
-        for minion in self.raw:
-            if minion in nodegroup:
-                data = self.get_data(minion)
-                if data:
-                    minions[minion] = data.copy()
-        return minions
+        nodegroup = _tgt_set(nodegroup)
+        return self._ret_minions(nodegroup.intersection)
 
     def ret_range_minions(self):
         '''
@@ -113,13 +114,7 @@ def ret_range_minions(self):
 
         minions = {}
         range_hosts = _convert_range_to_list(self.tgt, __opts__['range_server'])
-
-        for minion in self.raw:
-            if minion in range_hosts:
-                data = self.get_data(minion)
-                if data:
-                    minions[minion] = data.copy()
-        return minions
+        return self._ret_minions(range_hosts.__contains__)
 
     def get_data(self, minion):
         '''