T7789: T7661: VPP prevent failing to set XDP driver on clouds

Author: natali-rs1985

python/vyos/ethtool.py

@@ -59,6 +59,7 @@ class Ethtool:
     _ring_buffer = None
     _driver_name = None
     _flow_control = None
+    _channels = ''
 
     def __init__(self, ifname):
         # Get driver used for interface
@@ -111,6 +112,11 @@ def __init__(self, ifname):
         if not bool(err):
             self._flow_control = loads(out)[0]
 
+        # Get information about NIC channels
+        out, err = popen(f'ethtool --show-channels {ifname}')
+        if not bool(err):
+            self._channels = out.lower()
+
     def check_auto_negotiation_supported(self):
         """ Check if the NIC supports changing auto-negotiation """
         return self._base_settings['supports-auto-negotiation']
@@ -199,3 +205,20 @@ def get_flow_control(self):
                              'flow-control settings!')
 
         return 'on' if bool(self._flow_control['autonegotiate']) else 'off'
+
+    def get_channels(self, rx_tx_comb):
+        """
+        Get both the pre-set maximum and current value for a given channel type.
+
+        Args:
+            rx_tx_comb (str): Channel type, one of "rx", "tx", or "combined".
+
+        Returns:
+            list[int]: [maximum, current] values for the channel,
+                       or an empty list if not supported.
+        """
+        if rx_tx_comb not in ['rx', 'tx', 'combined']:
+            raise ValueError('Channel type must be either "rx", "tx" or "combined"')
+        matches = re.findall(rf'{rx_tx_comb}:\s+(\d+)', self._channels)
+
+        return [int(value) for value in matches]

python/vyos/ifconfig/ethernet.py

@@ -455,6 +455,23 @@ def set_ring_buffer(self, rx_tx, size):
             print(f'could not set "{rx_tx}" ring-buffer for {ifname}')
         return output
 
+    def set_channels(self, rx_tx_comb, queues):
+        """
+        Example:
+        >>> from vyos.ifconfig import EthernetIf
+        >>> i = EthernetIf('eth0')
+        >>> i.set_channels('rx', 2)
+        """
+        ifname = self.config['ifname']
+        cmd = f'ethtool --set-channels {ifname} {rx_tx_comb} {queues}'
+        output, code = self._popen(cmd)
+        # ethtool error codes:
+        #  80 - value already setted
+        #  81 - does not possible to set value
+        if code and code != 80:
+            print(f'could not set "{rx_tx_comb}" channel for {ifname}')
+        return output
+
     def set_switchdev(self, enable):
         ifname = self.config['ifname']
         addr, code = self._popen(

python/vyos/vpp/control_host.py

@@ -23,6 +23,8 @@
 
 from pyroute2 import IPRoute
 
+from vyos.ethtool import Ethtool
+from vyos.ifconfig import EthernetIf
 from vyos.vpp.utils import EthtoolGDrvinfo
 
 
@@ -373,3 +375,40 @@ def flush_ip(iface_name: str) -> None:
     """
     iproute = IPRoute()
     iproute.flush_addr(label=iface_name)
+
+
+def get_eth_channels(iface_name: str) -> dict:
+    """
+    Get the current hardware queue counts for channels of an interface using ethtool.
+
+    Args:
+        iface_name (str): name of an interface
+
+    Returns:
+        dict: Mapping of channel types to their current values:
+              - 'rx' (int | None): RX channel count.
+              - 'tx' (int | None): TX channel count.
+              - 'combined' (int | None): Combined channel count.
+              Returns None if the channel type is not supported.
+    """
+    ethtool = Ethtool(iface_name)
+
+    channels = {}
+    for channel in ['rx', 'tx', 'combined']:
+        queues_list = ethtool.get_channels(channel)
+        channels[channel] = queues_list[-1] if (len(queues_list) == 2) else None
+
+    return channels
+
+
+def set_eth_channels(iface_name: str, channels: dict) -> None:
+    """Configure the number of RX, TX, or combined channels for an interface.
+
+    Args:
+        iface_name (str): name of an interface
+        channels (dict): channels to set
+    """
+    interface = EthernetIf(iface_name)
+    for channel, value in channels.items():
+        if value:
+            interface.set_channels(channel, value)

python/vyos/vpp/control_vpp.py

@@ -351,7 +351,7 @@ def xdp_iface_create(
         self,
         host_if: str,
         name: str,
-        rxq_num: int = 0,
+        rxq_num: int = 65535,
         rxq_size: int = 0,
         txq_size: int = 0,
         mode: Literal['auto', 'copy', 'zero-copy'] = 'auto',

src/conf_mode/vpp.py

@@ -112,6 +112,9 @@
     'virtio_net': ['xdp'],
 }
 
+# drivers that require changing channels (half the maximum number of RX/TX queues)
+ethtool_channels_change_drv: list[str] = ['ena', 'gve']
+
 
 def _load_module(module_name: str):
     """
@@ -171,6 +174,25 @@ def _normalize_buffers(config: dict):
         config['settings']['buffers']['buffers_per_numa'] = str(buffers)
 
 
+def _get_max_xdp_rx_queues(config: dict):
+    """
+    Count max number of RX queues for XDP driver
+    - If the interface driver is in `ethtool_channels_change_drv`
+      only half of the available queues are used (to avoid NIC issues)
+    - For other interface drivers the full number of queues is returned.
+    - If neither `rx` nor `combined` is set, return 1.
+    """
+    for key in ('rx', 'combined'):
+        value = config['channels'].get(key)
+        if value:
+            if config['original_driver'] in ethtool_channels_change_drv:
+                return max(1, int(value) // 2)
+            else:
+                return int(value)
+
+    return 1
+
+
 def get_config(config=None):
     # use persistent config to store interfaces data between executions
     # this is required because some interfaces after they are connected
@@ -284,6 +306,20 @@ def get_config(config=None):
         _normalize_buffers(effective_config)
         config['effective'] = effective_config
 
+    # Save important info about all interfaces that cannot be retrieved later
+    # Add new interfaces (only if they are first time seen in a config)
+    for iface, iface_config in config.get('settings', {}).get('interface', {}).items():
+        if iface not in effective_config.get('settings', {}).get('interface', {}):
+            eth_ifaces_persist[iface] = {
+                'original_driver': EthtoolGDrvinfo(iface).driver,
+            }
+            eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface)
+            eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface)
+            eth_ifaces_persist[iface]['channels'] = control_host.get_eth_channels(iface)
+
+    # Return to config dictionary
+    config['persist_config'] = eth_ifaces_persist
+
     if 'settings' in config:
         if 'interface' in config['settings']:
             for iface, iface_config in config['settings']['interface'].items():
@@ -307,11 +343,11 @@ def get_config(config=None):
                 iface_filter_eth(conf, iface)
                 set_dependents('ethernet', conf, iface)
                 # Interfaces with changed driver should be removed/readded
-                if old_driver and old_driver[0] == 'dpdk':
+                if old_driver:
                     removed_ifaces.append(
                         {
                             'iface_name': iface,
-                            'driver': 'dpdk',
+                            'driver': old_driver[0],
                         }
                     )
 
@@ -344,7 +380,8 @@ def get_config(config=None):
                         'txq_size': int(iface_config['xdp_options']['tx_queue_size']),
                     }
                     if iface_config['xdp_options']['num_rx_queues'] == 'all':
-                        xdp_api_params['rxq_num'] = 0
+                        # 65535 is used as special value to request all available queues
+                        xdp_api_params['rxq_num'] = 65535
                     else:
                         xdp_api_params['rxq_num'] = int(
                             iface_config['xdp_options']['num_rx_queues']
@@ -374,26 +411,11 @@ def get_config(config=None):
                     iface_filter_eth(conf, iface)
                 set_dependents(dependency, conf, iface)
 
-    # Save important info about all interfaces that cannot be retrieved later
-    # Add new interfaces (only if they are first time seen in a config)
-    for iface, iface_config in config.get('settings', {}).get('interface', {}).items():
-        if iface not in effective_config.get('settings', {}).get('interface', {}):
-            eth_ifaces_persist[iface] = {
-                'original_driver': config['settings']['interface'][iface][
-                    'kernel_module'
-                ],
-            }
-            eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface)
-            eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface)
-
     # PPPoE dependency
     if pppoe_map_ifaces:
         config['pppoe_ifaces'] = pppoe_map_ifaces
         set_dependents('pppoe_server', conf)
 
-    # Return to config dictionary
-    config['persist_config'] = eth_ifaces_persist
-
     return config
 
 
@@ -485,6 +507,14 @@ def verify(config):
                 )
         if iface_config['driver'] == 'xdp' and 'xdp_options' in iface_config:
             if iface_config['xdp_options']['num_rx_queues'] != 'all':
+                rx_queues = iface_config['xdp_api_params']['rxq_num']
+                max_rx_queues = _get_max_xdp_rx_queues(config['persist_config'][iface])
+                if rx_queues > max_rx_queues:
+                    raise ConfigError(
+                        f'Maximum supported number of RX queues for interface {iface} is {max_rx_queues}. '
+                        f'Please set "xdp-options num-rx-queues" to {max_rx_queues} or fewer'
+                    )
+
                 Warning(f'Not all RX queues will be connected to VPP for {iface}!')
 
         if iface_config['driver'] == 'xdp' and 'dpdk_options' in iface_config:
@@ -593,8 +623,10 @@ def initialize_interface(iface, driver, iface_config) -> None:
         iface_new_name: str = control_host.get_eth_name(iface_config['dev_id'])
         control_host.rename_iface(iface_new_name, iface)
 
-    # XDP - rename an interface, disable promisc and XDP
+    # XDP - rename an interface, set original channels, disable promisc and XDP
     if driver == 'xdp':
+        if iface_config['original_driver'] in ethtool_channels_change_drv:
+            control_host.set_eth_channels(f'defunct_{iface}', iface_config['channels'])
         control_host.set_promisc(f'defunct_{iface}', 'off')
         control_host.rename_iface(f'defunct_{iface}', iface)
         control_host.xdp_remove(iface)
@@ -691,6 +723,25 @@ def apply(config):
                 # add XDP interfaces
                 if iface_config['driver'] == 'xdp':
                     control_host.rename_iface(iface, f'defunct_{iface}')
+
+                    # Some cloud NICs fail to load XDP if all RX queues are configured. To avoid this,
+                    # we limit the number of queues to half of the maximum supported by the driver.
+                    if (
+                        config['persist_config'][iface]['original_driver']
+                        in ethtool_channels_change_drv
+                    ):
+                        max_rx_queues = _get_max_xdp_rx_queues(
+                            config['persist_config'][iface]
+                        )
+                        channels_orig = config['persist_config'][iface]['channels']
+                        channels = {}
+                        if channels_orig.get('rx'):
+                            channels = {'rx': max_rx_queues, 'tx': max_rx_queues}
+                        if channels_orig.get('combined'):
+                            channels['combined'] = max_rx_queues
+                        if channels:
+                            control_host.set_eth_channels(f'defunct_{iface}', channels)
+
                     vpp_control.xdp_iface_create(
                         host_if=f'defunct_{iface}',
                         name=iface,