neutron: Add connectivity test with DPDK

The test will prepare hypervisor units in virtual machines by
enabling hugepages configuration and rebooting them and
subsequent required runtime changes.  If the hypervisor units
are physical machines the test assumes they already have
received the appropriate configuration from the bare metal
provisioning system.

Launch nested instance using flavor that enables hugepages and
attach it directly to the external network and perform
connectivity tests.

After a successful test the hypervisor units are brought back
to their original state to not influence subsequent tests.

Author: fnordahl

zaza/openstack/charm_tests/neutron/tests.py

@@ -33,6 +33,7 @@
 import zaza.openstack.charm_tests.test_utils as test_utils
 import zaza.openstack.configure.guest as guest
 import zaza.openstack.utilities.openstack as openstack_utils
+import zaza.utilities.machine_os
 
 
 class NeutronPluginApiSharedTests(test_utils.OpenStackBaseTest):
@@ -835,6 +836,10 @@ def setUpClass(cls):
         # the external provider network
         cls.attach_to_external_network = False
 
+        # Override this if you want your test to launch instances with a
+        # specific flavor
+        cls.instance_flavor = None
+
     @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
                     reraise=True, stop=tenacity.stop_after_attempt(8))
     def validate_instance_can_reach_other(self,
@@ -1112,13 +1117,177 @@ class name + run_tearDown key under the `tests_options` key in
         instance_1, instance_2 = self.retrieve_guests()
         if not all([instance_1, instance_2]):
             self.launch_guests(
-                attach_to_external_network=self.attach_to_external_network)
+                attach_to_external_network=self.attach_to_external_network,
+                flavor_name=self.instance_flavor)
             instance_1, instance_2 = self.retrieve_guests()
         self.check_connectivity(instance_1, instance_2)
         self.run_resource_cleanup = self.get_my_tests_options(
             'run_resource_cleanup', True)
 
 
+class DPDKNeutronNetworkingTest(NeutronNetworkingTest):
+    """Ensure that openstack instances have valid networking with DPDK."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running Neutron API Networking tests."""
+        super(DPDKNeutronNetworkingTest, cls).setUpClass()
+
+        # At this point in time the charms do not support configuring overlay
+        # networks with DPDK.  To perform end to end validation we need to
+        # attach instances directly to the provider network and subsequently
+        # DHCP needs to be enabled on that network.
+        #
+        # Note that for instances wired with DPDK the DHCP request/response is
+        # handled as private communication between the ovn-controller and the
+        # instance, and as such there is no risk of rogue DHCP replies escaping
+        # to the surrounding network.
+        cls.attach_to_external_network = True
+        cls.instance_flavor = 'hugepages'
+        cls.external_subnet = cls.neutron_client.find_resource(
+            'subnet',
+            neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_subnet_name'])
+        if ('dhcp_enabled' not in cls.external_subnet or
+                not cls.external_subnet['dhcp_enabled']):
+            logging.info('Enabling DHCP on subnet {}'
+                         .format(cls.external_subnet['name']))
+            openstack_utils.update_subnet_dhcp(
+                cls.neutron_client, cls.external_subnet, True)
+
+        cls.nr_1g_hugepages = 4
+        # Prepare hypervisor units for the test
+        for unit in zaza.model.get_units(
+                zaza.utilities.machine_os.get_hv_application(),
+                model_name=cls.model_name):
+            if not zaza.utilities.machine_os.is_vm(unit.name,
+                                                   model_name=cls.model_name):
+                logging.info('Unit {} is a physical machine, assuming '
+                             'hugepages and IOMMU configuration already '
+                             'performed through kernel command line.')
+                continue
+            logging.info('Checking CPU topology on {}'.format(unit.name))
+            cls._assert_unit_cpu_topology(cls, unit, model_name=cls.model_name)
+            logging.info('Enabling hugepages on {}'.format(unit.name))
+            zaza.utilities.machine_os.enable_hugepages(
+                unit, cls.nr_1g_hugepages, model_name=cls.model_name)
+            logging.info('Enabling unsafe VFIO NOIOMMU mode on {}'
+                         .format(unit.name))
+            zaza.utilities.machine_os.enable_vfio_unsafe_noiommu_mode(
+                unit, model_name=cls.model_name)
+
+    def _assert_unit_cpu_topology(self, unit, model_name=None):
+        r"""Assert unit under test CPU topology.
+
+        When using OpenStack as CI substrate:
+
+        By default, when instance NUMA placement is not specified,
+        a topology of N sockets, each with one core and one thread,
+        is used for an instance, where N corresponds to the number of
+        instance vCPUs requested.
+
+        In this context a socket is a physical socket on the motherboard
+        where a CPU is connected.
+
+        The DPDK Environment Abstraction Layer (EAL) allocates memory per
+        CPU socket, so we want the CPU topology inside the instance to
+        mimic something we would be likely to find in the real world and
+        at the same time not make the test too heavy.
+
+        The charm default is to have Open vSwitch allocate 1GB RAM per
+        CPU socket.
+
+        The following command would set the apropriate CPU topology for a
+        4 VCPU flavor:
+
+            openstack flavor set m1.large \
+                --property hw:cpu_sockets=2 \
+                --property hw:cpu_cores=2 \
+                --property hw:cpu_threads=2
+        """
+        # Get number of sockets
+        cmd = 'lscpu -p|grep -v ^#|cut -f3 -d,|sort|uniq|wc -l'
+        sockets = int(zaza.utilities.juju.remote_run(
+            unit.name, cmd, model_name=model_name, fatal=True).rstrip())
+
+        # Get total memory
+        cmd = 'cat /proc/meminfo |grep ^MemTotal'
+        _, meminfo_value, _ = zaza.utilities.juju.remote_run(
+            unit.name,
+            cmd,
+            model_name=model_name,
+            fatal=True).rstrip().split()
+        mbtotal = int(meminfo_value) * 1024 / 1000 / 1000
+        mbtotalhugepages = self.nr_1g_hugepages * 1024
+
+        # headroom for operating system and daemons in instance
+        mbsystemheadroom = 2048
+        # memory to be consumed by the nested instance
+        mbinstance = 1024
+
+        # the amount of hugepage memory OVS / DPDK EAL will allocate
+        mbovshugepages = sockets * 1024
+        # the amount of hugepage memory available for nested instance
+        mbfreehugepages = mbtotalhugepages - mbovshugepages
+
+        assert (mbtotal - mbtotalhugepages >= mbsystemheadroom and
+                mbfreehugepages >= mbinstance), (
+            'Unit {} is not suitable for test, please adjust instance '
+            'type CPU topology or provide suitable physical machine. '
+            'CPU Sockets: {} '
+            'Available memory: {} MB '
+            'Details:\n{}'
+            .format(unit.name,
+                    sockets,
+                    mbtotal,
+                    self._assert_unit_cpu_topology.__doc__))
+
+    def test_instances_have_networking(self):
+        """Enable DPDK then Validate North/South and East/West networking."""
+        with self.config_change(
+                {
+                    'enable-dpdk': False,
+                    'dpdk-driver': '',
+                },
+                {
+                    'enable-dpdk': True,
+                    'dpdk-driver': 'vfio-pci',
+                },
+                application_name='ovn-chassis'):
+            super().test_instances_have_networking()
+        self.run_resource_cleanup = self.get_my_tests_options(
+            'run_resource_cleanup', True)
+
+    def resource_cleanup(self):
+        """Extend to also revert VFIO NOIOMMU mode on units under test."""
+        super().resource_cleanup()
+        if not self.run_resource_cleanup:
+            return
+
+        if ('dhcp_enabled' not in self.external_subnet or
+                not self.external_subnet['dhcp_enabled']):
+            logging.info('Disabling DHCP on subnet {}'
+                         .format(self.external_subnet['name']))
+            openstack_utils.update_subnet_dhcp(
+                self.neutron_client, self.external_subnet, False)
+
+        for unit in zaza.model.get_units(
+                zaza.utilities.machine_os.get_hv_application(),
+                model_name=self.model_name):
+            if not zaza.utilities.machine_os.is_vm(unit.name,
+                                                   model_name=self.model_name):
+                logging.info('Unit {} is a physical machine, assuming '
+                             'hugepages and IOMMU configuration already '
+                             'performed through kernel command line.')
+                continue
+            logging.info('Disabling hugepages on {}'.format(unit.name))
+            zaza.utilities.machine_os.disable_hugepages(
+                unit, model_name=self.model_name)
+            logging.info('Disabling unsafe VFIO NOIOMMU mode on {}'
+                         .format(unit.name))
+            zaza.utilities.machine_os.disable_vfio_unsafe_noiommu_mode(
+                unit, model_name=self.model_name)
+
+
 class NeutronNetworkingVRRPTests(NeutronNetworkingBase):
     """Check networking when gateways are restarted."""
 

zaza/openstack/charm_tests/nova/utils.py

@@ -65,5 +65,14 @@
             'hw:tpm_model': 'tpm-crb',
         },
     },
+    'hugepages': {
+        'flavorid': 'auto',
+        'ram': 1024,
+        'disk': 20,
+        'vcpus': 1,
+        'extra-specs': {
+            'hw:mem_page_size': 'large',
+        },
+    },
 }
 KEYPAIR_NAME = 'zaza'

zaza/openstack/charm_tests/test_utils.py

@@ -686,7 +686,8 @@ def launch_guest(self, guest_name, userdata=None, use_boot_volume=False,
                     flavor_name=flavor_name,
                     attach_to_external_network=attach_to_external_network)
 
-    def launch_guests(self, userdata=None, attach_to_external_network=False):
+    def launch_guests(self, userdata=None, attach_to_external_network=False,
+                      flavor_name=None):
         """Launch two guests to use in tests.
 
         Note that it is up to the caller to have set the RESOURCE_PREFIX class
@@ -706,7 +707,8 @@ def launch_guests(self, userdata=None, attach_to_external_network=False):
                 self.launch_guest(
                     guest_name='ins-{}'.format(guest_number),
                     userdata=userdata,
-                    attach_to_external_network=attach_to_external_network))
+                    attach_to_external_network=attach_to_external_network,
+                    flavor_name=flavor_name))
         return launched_instances
 
     def retrieve_guest(self, guest_name):