From 3f711ff6dad8952f1e12c5de4022e0e513417b8a Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 13 Apr 2022 10:44:37 +0200 Subject: [PATCH 01/11] openstack: Add update subnet DHCP helper --- .../utilities/test_zaza_utilities_openstack.py | 14 ++++++++++++++ zaza/openstack/utilities/openstack.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index cc5fa68..3cd33a6 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1500,6 +1500,20 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.configure_networking_charms.assert_called_once_with( 'fakenetworkingdata', expect, use_juju_wait=False) + def test_update_subnet_dhcp(self): + neutron_client = mock.MagicMock() + openstack_utils.update_subnet_dhcp( + neutron_client, {'id': 'aId'}, True) + neutron_client.update_subnet.assert_called_once_with( + 'aId', + {'subnet': {'enable_dhcp': True}}) + neutron_client.reset_mock() + openstack_utils.update_subnet_dhcp( + neutron_client, {'id': 'aId'}, False) + neutron_client.update_subnet.assert_called_once_with( + 'aId', + {'subnet': {'enable_dhcp': False}}) + class TestAsyncOpenstackUtils(ut_utils.AioTestCase): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index bc04a1d..fdadcb1 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1328,6 +1328,24 @@ def update_subnet_dns(neutron_client, subnet, dns_servers): neutron_client.update_subnet(subnet['id'], msg) +def update_subnet_dhcp(neutron_client, subnet, enable_dhcp): + """Update subnet DHCP status. + + :param neutron_client: Authenticated neutronclient + :type neutron_client: neutronclient.Client object + :param subnet: Subnet object + :type subnet: dict + :param enable_dhcp: Whether DHCP should be enabled or not + :type enable_dhcp: bool + """ + msg = { + 'subnet': { + 'enable_dhcp': enable_dhcp, + } + } + neutron_client.update_subnet(subnet['id'], msg) + + def create_provider_router(neutron_client, project_id): """Create the provider router. From 3ec64703edfc904b6c1a563c294160d83ae9a880 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 12 Apr 2022 17:23:18 +0200 Subject: [PATCH 02/11] Support launch instance on external provider network Allow launching an instance and attaching it directly to the external provider network. This is useful in its own right and is also a requirement in order to successfully test some configurations. --- zaza/openstack/charm_tests/test_utils.py | 17 +++++++++--- zaza/openstack/configure/guest.py | 34 +++++++++++++++++------- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 9041df7..6a1b567 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -634,7 +634,8 @@ class OpenStackBaseTest(BaseCharmTest): pass def launch_guest(self, guest_name, userdata=None, use_boot_volume=False, - instance_key=None, flavor_name=None): + instance_key=None, flavor_name=None, + attach_to_external_network=False): """Launch one guest to use in tests. Note that it is up to the caller to have set the RESOURCE_PREFIX class @@ -651,6 +652,9 @@ class OpenStackBaseTest(BaseCharmTest): :type use_boot_volume: boolean :param instance_key: Key to collect associated config data with. :type instance_key: Optional[str] + :param attach_to_external_network: Attach instance directly to external + network. + :type attach_to_external_network: bool :returns: Nova instance objects :rtype: Server """ @@ -679,9 +683,10 @@ class OpenStackBaseTest(BaseCharmTest): vm_name=instance_name, use_boot_volume=use_boot_volume, userdata=userdata, - flavor_name=flavor_name) + flavor_name=flavor_name, + attach_to_external_network=attach_to_external_network) - def launch_guests(self, userdata=None): + def launch_guests(self, userdata=None, attach_to_external_network=False): """Launch two guests to use in tests. Note that it is up to the caller to have set the RESOURCE_PREFIX class @@ -689,6 +694,9 @@ class OpenStackBaseTest(BaseCharmTest): :param userdata: Userdata to attach to instance :type userdata: Optional[str] + :param attach_to_external_network: Attach instance directly to external + network. + :type attach_to_external_network: bool :returns: List of launched Nova instance objects :rtype: List[Server] """ @@ -697,7 +705,8 @@ class OpenStackBaseTest(BaseCharmTest): launched_instances.append( self.launch_guest( guest_name='ins-{}'.format(guest_number), - userdata=userdata)) + userdata=userdata, + attach_to_external_network=attach_to_external_network)) return launched_instances def retrieve_guest(self, guest_name): diff --git a/zaza/openstack/configure/guest.py b/zaza/openstack/configure/guest.py index 4e20ef4..d5034b3 100644 --- a/zaza/openstack/configure/guest.py +++ b/zaza/openstack/configure/guest.py @@ -53,7 +53,7 @@ boot_tests = { def launch_instance(instance_key, use_boot_volume=False, vm_name=None, private_network_name=None, image_name=None, flavor_name=None, external_network_name=None, meta=None, - userdata=None): + userdata=None, attach_to_external_network=False): """Launch an instance. :param instance_key: Key to collect associated config data with. @@ -76,6 +76,9 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, :type meta: dict :param userdata: Configuration to use upon launch, used by cloud-init. :type userdata: str + :param attach_to_external_network: Attach instance directly to external + network. + :type attach_to_external_network: bool :returns: the created instance :rtype: novaclient.Server """ @@ -94,12 +97,18 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, flavor = nova_client.flavors.find(name=flavor_name) private_network_name = private_network_name or "private" - net = neutron_client.find_resource("network", private_network_name) - nics = [{'net-id': net.get('id')}] meta = meta or {} external_network_name = external_network_name or "ext_net" + if attach_to_external_network: + instance_network_name = external_network_name + else: + instance_network_name = private_network_name + + net = neutron_client.find_resource("network", instance_network_name) + nics = [{'net-id': net.get('id')}] + if use_boot_volume: bdmv2 = [{ 'boot_index': '0', @@ -143,12 +152,19 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, port = openstack_utils.get_ports_from_device_id( neutron_client, instance.id)[0] - logging.info('Assigning floating ip.') - ip = openstack_utils.create_floating_ip( - neutron_client, - external_network_name, - port=port)['floating_ip_address'] - logging.info('Assigned floating IP {} to {}'.format(ip, vm_name)) + if attach_to_external_network: + logging.info('attach_to_external_network={}, not assigning floating IP' + .format(attach_to_external_network)) + ip = port['fixed_ips'][0]['ip_address'] + logging.info('Using fixed IP {} on network {} for {}' + .format(ip, instance_network_name, vm_name)) + else: + logging.info('Assigning floating ip.') + ip = openstack_utils.create_floating_ip( + neutron_client, + external_network_name, + port=port)['floating_ip_address'] + logging.info('Assigned floating IP {} to {}'.format(ip, vm_name)) try: for attempt in Retrying( stop=stop_after_attempt(8), From ad16b5a7fc3a0971b168b133fbf0b5cbd55115fd Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 21 Apr 2022 15:14:19 +0200 Subject: [PATCH 03/11] Make project network and subnet configurable --- zaza/openstack/charm_tests/neutron/setup.py | 2 ++ zaza/openstack/configure/network.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index 290483a..ba913da 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -43,6 +43,8 @@ OVERCLOUD_NETWORK_CONFIG = { "prefix_len": "24", "subnetpool_name": "pooled_subnets", "subnetpool_prefix": "192.168.0.0/16", + "project_net_name": "private", + "project_subnet_name": "private_subnet", } OVERCLOUD_PROVIDER_VLAN_NETWORK_CONFIG = { diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index c0bbe1c..9bfa5a0 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -164,14 +164,16 @@ def setup_sdn(network_config, keystone_session=None): neutron_client, project_id, shared=False, - network_type=network_config["network_type"]) + network_type=network_config["network_type"], + net_name=network_config["project_net_name"]) project_subnet = openstack_utils.create_project_subnet( neutron_client, project_id, project_network, network_config.get("private_net_cidr"), subnetpool=subnetpool, - ip_version=ip_version) + ip_version=ip_version, + subnet_name=network_config["project_subnet_name"]) openstack_utils.update_subnet_dns( neutron_client, project_subnet, From ba0dc0232d6104e8df455b6dfc423c33f326d40d Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 12 Apr 2022 17:43:13 +0200 Subject: [PATCH 04/11] Support connectivity check without FIPs --- zaza/openstack/charm_tests/neutron/tests.py | 62 +++++++++++++++------ zaza/openstack/configure/network.py | 3 +- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 19bda01..9159b74 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -28,6 +28,7 @@ from neutronclient.common import exceptions as neutronexceptions import yaml import zaza +import zaza.openstack.charm_tests.neutron.setup as neutron_setup import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.guest as guest @@ -805,6 +806,11 @@ class NeutronOvsVsctlTest(NeutronPluginApiSharedTests): self.assertEqual(actual_external_id, expected_external_id) +def router_address_from_subnet(subnet): + """Retrieve router address from subnet.""" + return subnet['gateway_ip'] + + class NeutronNetworkingBase(test_utils.OpenStackBaseTest): """Base for checking openstack instances have valid networking.""" @@ -818,6 +824,17 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) + cls.project_subnet = cls.neutron_client.find_resource( + 'subnet', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['project_subnet_name']) + cls.external_subnet = cls.neutron_client.find_resource( + 'subnet', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_subnet_name']) + + # Override this if you want your test to attach instances directly to + # the external provider network + cls.attach_to_external_network = False + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_other(self, @@ -840,8 +857,10 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): :param mtu: Check that we can send non-fragmented packets of given size :type mtu: Optional[int] """ - floating_1 = floating_ips_from_instance(instance_1)[0] - floating_2 = floating_ips_from_instance(instance_2)[0] + if not self.attach_to_external_network: + floating_1 = floating_ips_from_instance(instance_1)[0] + floating_2 = floating_ips_from_instance(instance_2)[0] + address_1 = fixed_ips_from_instance(instance_1)[0] address_2 = fixed_ips_from_instance(instance_2)[0] username = guest.boot_tests['bionic']['username'] @@ -859,15 +878,21 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): 'ping -M do -s {} -c 1'.format(packetsize)) for cmd in cmds: - openstack_utils.ssh_command( - username, floating_1, 'instance-1', - '{} {}'.format(cmd, address_2), - password=password, privkey=privkey, verify=verify) + if self.attach_to_external_network: + openstack_utils.ssh_command( + username, address_1, 'instance-1', + '{} {}'.format(cmd, address_2), + password=password, privkey=privkey, verify=verify) + else: + openstack_utils.ssh_command( + username, floating_1, 'instance-1', + '{} {}'.format(cmd, address_2), + password=password, privkey=privkey, verify=verify) - openstack_utils.ssh_command( - username, floating_1, 'instance-1', - '{} {}'.format(cmd, floating_2), - password=password, privkey=privkey, verify=verify) + openstack_utils.ssh_command( + username, floating_1, 'instance-1', + '{} {}'.format(cmd, floating_2), + password=password, privkey=privkey, verify=verify) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) @@ -875,11 +900,6 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): """ Validate that an instance can reach it's primary gateway. - We make the assumption that the router's IP is 192.168.0.1 - as that's the network that is setup in - neutron.setup.basic_overcloud_network which is used in all - Zaza Neutron validations. - :param instance: The instance to check networking from :type instance: nova_client.Server @@ -889,7 +909,12 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): :param mtu: Check that we can send non-fragmented packets of given size :type mtu: Optional[int] """ - address = floating_ips_from_instance(instance)[0] + if self.attach_to_external_network: + router = router_address_from_subnet(self.external_subnet) + address = fixed_ips_from_instance(instance)[0] + else: + router = router_address_from_subnet(self.project_subnet) + address = floating_ips_from_instance(instance)[0] username = guest.boot_tests['bionic']['username'] password = guest.boot_tests['bionic'].get('password') @@ -907,7 +932,7 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): for cmd in cmds: openstack_utils.ssh_command( - username, address, 'instance', '{} 192.168.0.1'.format(cmd), + username, address, 'instance', '{} {}'.format(cmd, router), password=password, privkey=privkey, verify=verify) @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), @@ -1086,7 +1111,8 @@ class NeutronNetworkingTest(NeutronNetworkingBase): """ instance_1, instance_2 = self.retrieve_guests() if not all([instance_1, instance_2]): - self.launch_guests() + self.launch_guests( + attach_to_external_network=self.attach_to_external_network) instance_1, instance_2 = self.retrieve_guests() self.check_connectivity(instance_1, instance_2) self.run_resource_cleanup = self.get_my_tests_options( diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 9bfa5a0..7e4e6f0 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -139,7 +139,8 @@ def setup_sdn(network_config, keystone_session=None): network_config["default_gateway"], network_config["external_net_cidr"], network_config["start_floating_ip"], - network_config["end_floating_ip"]) + network_config["end_floating_ip"], + dhcp=True) provider_router = ( openstack_utils.create_provider_router(neutron_client, project_id)) openstack_utils.plug_extnet_into_router( From f38448fee736e22cbf126f54f9416dcf16fcd5d3 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 21 Apr 2022 17:21:56 +0200 Subject: [PATCH 05/11] neutron: Add connectivity test with DPDK The test will prepare hypervisor units in virtual machines by enabling hugepages configuration and rebooting them and subsequent required runtime changes. If the hypervisor units are physical machines the test assumes they already have received the appropriate configuration from the bare metal provisioning system. Launch nested instance using flavor that enables hugepages and attach it directly to the external network and perform connectivity tests. After a successful test the hypervisor units are brought back to their original state to not influence subsequent tests. --- zaza/openstack/charm_tests/neutron/tests.py | 171 +++++++++++++++++++- zaza/openstack/charm_tests/nova/utils.py | 9 ++ zaza/openstack/charm_tests/test_utils.py | 6 +- 3 files changed, 183 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 9159b74..91e77f5 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -33,6 +33,7 @@ import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.guest as guest import zaza.openstack.utilities.openstack as openstack_utils +import zaza.utilities.machine_os class NeutronPluginApiSharedTests(test_utils.OpenStackBaseTest): @@ -835,6 +836,10 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): # the external provider network cls.attach_to_external_network = False + # Override this if you want your test to launch instances with a + # specific flavor + cls.instance_flavor = None + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_other(self, @@ -1112,13 +1117,177 @@ class NeutronNetworkingTest(NeutronNetworkingBase): instance_1, instance_2 = self.retrieve_guests() if not all([instance_1, instance_2]): self.launch_guests( - attach_to_external_network=self.attach_to_external_network) + attach_to_external_network=self.attach_to_external_network, + flavor_name=self.instance_flavor) instance_1, instance_2 = self.retrieve_guests() self.check_connectivity(instance_1, instance_2) self.run_resource_cleanup = self.get_my_tests_options( 'run_resource_cleanup', True) +class DPDKNeutronNetworkingTest(NeutronNetworkingTest): + """Ensure that openstack instances have valid networking with DPDK.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Neutron API Networking tests.""" + super(DPDKNeutronNetworkingTest, cls).setUpClass() + + # At this point in time the charms do not support configuring overlay + # networks with DPDK. To perform end to end validation we need to + # attach instances directly to the provider network and subsequently + # DHCP needs to be enabled on that network. + # + # Note that for instances wired with DPDK the DHCP request/response is + # handled as private communication between the ovn-controller and the + # instance, and as such there is no risk of rogue DHCP replies escaping + # to the surrounding network. + cls.attach_to_external_network = True + cls.instance_flavor = 'hugepages' + cls.external_subnet = cls.neutron_client.find_resource( + 'subnet', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_subnet_name']) + if ('dhcp_enabled' not in cls.external_subnet or + not cls.external_subnet['dhcp_enabled']): + logging.info('Enabling DHCP on subnet {}' + .format(cls.external_subnet['name'])) + openstack_utils.update_subnet_dhcp( + cls.neutron_client, cls.external_subnet, True) + + cls.nr_1g_hugepages = 4 + # Prepare hypervisor units for the test + for unit in zaza.model.get_units( + zaza.utilities.machine_os.get_hv_application(), + model_name=cls.model_name): + if not zaza.utilities.machine_os.is_vm(unit.name, + model_name=cls.model_name): + logging.info('Unit {} is a physical machine, assuming ' + 'hugepages and IOMMU configuration already ' + 'performed through kernel command line.') + continue + logging.info('Checking CPU topology on {}'.format(unit.name)) + cls._assert_unit_cpu_topology(cls, unit, model_name=cls.model_name) + logging.info('Enabling hugepages on {}'.format(unit.name)) + zaza.utilities.machine_os.enable_hugepages( + unit, cls.nr_1g_hugepages, model_name=cls.model_name) + logging.info('Enabling unsafe VFIO NOIOMMU mode on {}' + .format(unit.name)) + zaza.utilities.machine_os.enable_vfio_unsafe_noiommu_mode( + unit, model_name=cls.model_name) + + def _assert_unit_cpu_topology(self, unit, model_name=None): + r"""Assert unit under test CPU topology. + + When using OpenStack as CI substrate: + + By default, when instance NUMA placement is not specified, + a topology of N sockets, each with one core and one thread, + is used for an instance, where N corresponds to the number of + instance vCPUs requested. + + In this context a socket is a physical socket on the motherboard + where a CPU is connected. + + The DPDK Environment Abstraction Layer (EAL) allocates memory per + CPU socket, so we want the CPU topology inside the instance to + mimic something we would be likely to find in the real world and + at the same time not make the test too heavy. + + The charm default is to have Open vSwitch allocate 1GB RAM per + CPU socket. + + The following command would set the apropriate CPU topology for a + 4 VCPU flavor: + + openstack flavor set m1.large \ + --property hw:cpu_sockets=2 \ + --property hw:cpu_cores=2 \ + --property hw:cpu_threads=2 + """ + # Get number of sockets + cmd = 'lscpu -p|grep -v ^#|cut -f3 -d,|sort|uniq|wc -l' + sockets = int(zaza.utilities.juju.remote_run( + unit.name, cmd, model_name=model_name, fatal=True).rstrip()) + + # Get total memory + cmd = 'cat /proc/meminfo |grep ^MemTotal' + _, meminfo_value, _ = zaza.utilities.juju.remote_run( + unit.name, + cmd, + model_name=model_name, + fatal=True).rstrip().split() + mbtotal = int(meminfo_value) * 1024 / 1000 / 1000 + mbtotalhugepages = self.nr_1g_hugepages * 1024 + + # headroom for operating system and daemons in instance + mbsystemheadroom = 2048 + # memory to be consumed by the nested instance + mbinstance = 1024 + + # the amount of hugepage memory OVS / DPDK EAL will allocate + mbovshugepages = sockets * 1024 + # the amount of hugepage memory available for nested instance + mbfreehugepages = mbtotalhugepages - mbovshugepages + + assert (mbtotal - mbtotalhugepages >= mbsystemheadroom and + mbfreehugepages >= mbinstance), ( + 'Unit {} is not suitable for test, please adjust instance ' + 'type CPU topology or provide suitable physical machine. ' + 'CPU Sockets: {} ' + 'Available memory: {} MB ' + 'Details:\n{}' + .format(unit.name, + sockets, + mbtotal, + self._assert_unit_cpu_topology.__doc__)) + + def test_instances_have_networking(self): + """Enable DPDK then Validate North/South and East/West networking.""" + with self.config_change( + { + 'enable-dpdk': False, + 'dpdk-driver': '', + }, + { + 'enable-dpdk': True, + 'dpdk-driver': 'vfio-pci', + }, + application_name='ovn-chassis'): + super().test_instances_have_networking() + self.run_resource_cleanup = self.get_my_tests_options( + 'run_resource_cleanup', True) + + def resource_cleanup(self): + """Extend to also revert VFIO NOIOMMU mode on units under test.""" + super().resource_cleanup() + if not self.run_resource_cleanup: + return + + if ('dhcp_enabled' not in self.external_subnet or + not self.external_subnet['dhcp_enabled']): + logging.info('Disabling DHCP on subnet {}' + .format(self.external_subnet['name'])) + openstack_utils.update_subnet_dhcp( + self.neutron_client, self.external_subnet, False) + + for unit in zaza.model.get_units( + zaza.utilities.machine_os.get_hv_application(), + model_name=self.model_name): + if not zaza.utilities.machine_os.is_vm(unit.name, + model_name=self.model_name): + logging.info('Unit {} is a physical machine, assuming ' + 'hugepages and IOMMU configuration already ' + 'performed through kernel command line.') + continue + logging.info('Disabling hugepages on {}'.format(unit.name)) + zaza.utilities.machine_os.disable_hugepages( + unit, model_name=self.model_name) + logging.info('Disabling unsafe VFIO NOIOMMU mode on {}' + .format(unit.name)) + zaza.utilities.machine_os.disable_vfio_unsafe_noiommu_mode( + unit, model_name=self.model_name) + + class NeutronNetworkingVRRPTests(NeutronNetworkingBase): """Check networking when gateways are restarted.""" diff --git a/zaza/openstack/charm_tests/nova/utils.py b/zaza/openstack/charm_tests/nova/utils.py index f2c8af9..87af7d5 100644 --- a/zaza/openstack/charm_tests/nova/utils.py +++ b/zaza/openstack/charm_tests/nova/utils.py @@ -65,5 +65,14 @@ FLAVORS = { 'hw:tpm_model': 'tpm-crb', }, }, + 'hugepages': { + 'flavorid': 'auto', + 'ram': 1024, + 'disk': 20, + 'vcpus': 1, + 'extra-specs': { + 'hw:mem_page_size': 'large', + }, + }, } KEYPAIR_NAME = 'zaza' diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 6a1b567..7abd331 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -686,7 +686,8 @@ class OpenStackBaseTest(BaseCharmTest): flavor_name=flavor_name, attach_to_external_network=attach_to_external_network) - def launch_guests(self, userdata=None, attach_to_external_network=False): + def launch_guests(self, userdata=None, attach_to_external_network=False, + flavor_name=None): """Launch two guests to use in tests. Note that it is up to the caller to have set the RESOURCE_PREFIX class @@ -706,7 +707,8 @@ class OpenStackBaseTest(BaseCharmTest): self.launch_guest( guest_name='ins-{}'.format(guest_number), userdata=userdata, - attach_to_external_network=attach_to_external_network)) + attach_to_external_network=attach_to_external_network, + flavor_name=flavor_name)) return launched_instances def retrieve_guest(self, guest_name): From c77ac32f5eb6fa94198ef84c8e660a741a7c4908 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 22 Apr 2022 09:26:30 +0200 Subject: [PATCH 06/11] Add *.swp to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 276135e..8dda292 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ dist/ zaza.openstack.egg-info/ .coverage .vscode/ +*.swp # Sphinx doc/build From 16e0fbbfa75e83f63031416b076e84175be0b2fe Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 22 Apr 2022 07:22:22 +0200 Subject: [PATCH 07/11] ovn: Fix ``wrong_bridge_config`` test The test blindly reverts the ``bridge-interface-mappings`` configuration to an empty string, which would be the wrong thing to do if it was previously set up by a test configure step. The test also does not properly populate target_deploy_status and the test will always wait for the next update status to run, which may take several minutes, before completing. --- zaza/openstack/charm_tests/ovn/tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 3b43d65..c99b2b9 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -177,7 +177,8 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): 'target_deploy_status', {}) new_target_deploy_status = stored_target_deploy_status.copy() new_target_deploy_status[self.application_name] = { - 'ovn-chassis': 'blocked', + 'workload-status': 'blocked', + 'workload-status-message': 'Wrong format', } if 'target_deploy_status' in self.test_config: self.test_config['target_deploy_status'].update( @@ -186,7 +187,9 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): self.test_config['target_deploy_status'] = new_target_deploy_status with self.config_change( - {'bridge-interface-mappings': ''}, + self.config_current( + application_name=self.application_name, + keys=['bridge-interface-mappings']), {'bridge-interface-mappings': 'incorrect'}): logging.info('Charm went into blocked state as expected, restore ' 'configuration') From 899f2b7cd22c75e3a487bd02d7f65564c4007f3b Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 22 Apr 2022 07:46:09 +0200 Subject: [PATCH 08/11] Move hugepages/vfio helpers to common test utils Update CPU topology flavor advice. The example currently listed would create a two socket CPU with only one NUMA node, which is a invalid configuration which leads to undefined results. Instead list two examples one with a single socket for simpler end to end tests and a two socket two numa node example for more advanced low level tests. --- zaza/openstack/charm_tests/neutron/tests.py | 105 +----------------- zaza/openstack/charm_tests/test_utils.py | 117 ++++++++++++++++++++ 2 files changed, 119 insertions(+), 103 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 91e77f5..cf925b0 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -1154,95 +1154,9 @@ class DPDKNeutronNetworkingTest(NeutronNetworkingTest): openstack_utils.update_subnet_dhcp( cls.neutron_client, cls.external_subnet, True) - cls.nr_1g_hugepages = 4 - # Prepare hypervisor units for the test - for unit in zaza.model.get_units( - zaza.utilities.machine_os.get_hv_application(), - model_name=cls.model_name): - if not zaza.utilities.machine_os.is_vm(unit.name, - model_name=cls.model_name): - logging.info('Unit {} is a physical machine, assuming ' - 'hugepages and IOMMU configuration already ' - 'performed through kernel command line.') - continue - logging.info('Checking CPU topology on {}'.format(unit.name)) - cls._assert_unit_cpu_topology(cls, unit, model_name=cls.model_name) - logging.info('Enabling hugepages on {}'.format(unit.name)) - zaza.utilities.machine_os.enable_hugepages( - unit, cls.nr_1g_hugepages, model_name=cls.model_name) - logging.info('Enabling unsafe VFIO NOIOMMU mode on {}' - .format(unit.name)) - zaza.utilities.machine_os.enable_vfio_unsafe_noiommu_mode( - unit, model_name=cls.model_name) - - def _assert_unit_cpu_topology(self, unit, model_name=None): - r"""Assert unit under test CPU topology. - - When using OpenStack as CI substrate: - - By default, when instance NUMA placement is not specified, - a topology of N sockets, each with one core and one thread, - is used for an instance, where N corresponds to the number of - instance vCPUs requested. - - In this context a socket is a physical socket on the motherboard - where a CPU is connected. - - The DPDK Environment Abstraction Layer (EAL) allocates memory per - CPU socket, so we want the CPU topology inside the instance to - mimic something we would be likely to find in the real world and - at the same time not make the test too heavy. - - The charm default is to have Open vSwitch allocate 1GB RAM per - CPU socket. - - The following command would set the apropriate CPU topology for a - 4 VCPU flavor: - - openstack flavor set m1.large \ - --property hw:cpu_sockets=2 \ - --property hw:cpu_cores=2 \ - --property hw:cpu_threads=2 - """ - # Get number of sockets - cmd = 'lscpu -p|grep -v ^#|cut -f3 -d,|sort|uniq|wc -l' - sockets = int(zaza.utilities.juju.remote_run( - unit.name, cmd, model_name=model_name, fatal=True).rstrip()) - - # Get total memory - cmd = 'cat /proc/meminfo |grep ^MemTotal' - _, meminfo_value, _ = zaza.utilities.juju.remote_run( - unit.name, - cmd, - model_name=model_name, - fatal=True).rstrip().split() - mbtotal = int(meminfo_value) * 1024 / 1000 / 1000 - mbtotalhugepages = self.nr_1g_hugepages * 1024 - - # headroom for operating system and daemons in instance - mbsystemheadroom = 2048 - # memory to be consumed by the nested instance - mbinstance = 1024 - - # the amount of hugepage memory OVS / DPDK EAL will allocate - mbovshugepages = sockets * 1024 - # the amount of hugepage memory available for nested instance - mbfreehugepages = mbtotalhugepages - mbovshugepages - - assert (mbtotal - mbtotalhugepages >= mbsystemheadroom and - mbfreehugepages >= mbinstance), ( - 'Unit {} is not suitable for test, please adjust instance ' - 'type CPU topology or provide suitable physical machine. ' - 'CPU Sockets: {} ' - 'Available memory: {} MB ' - 'Details:\n{}' - .format(unit.name, - sockets, - mbtotal, - self._assert_unit_cpu_topology.__doc__)) - def test_instances_have_networking(self): """Enable DPDK then Validate North/South and East/West networking.""" + self.enable_hugepages_vfio_on_hvs_in_vms(4) with self.config_change( { 'enable-dpdk': False, @@ -1270,22 +1184,7 @@ class DPDKNeutronNetworkingTest(NeutronNetworkingTest): openstack_utils.update_subnet_dhcp( self.neutron_client, self.external_subnet, False) - for unit in zaza.model.get_units( - zaza.utilities.machine_os.get_hv_application(), - model_name=self.model_name): - if not zaza.utilities.machine_os.is_vm(unit.name, - model_name=self.model_name): - logging.info('Unit {} is a physical machine, assuming ' - 'hugepages and IOMMU configuration already ' - 'performed through kernel command line.') - continue - logging.info('Disabling hugepages on {}'.format(unit.name)) - zaza.utilities.machine_os.disable_hugepages( - unit, model_name=self.model_name) - logging.info('Disabling unsafe VFIO NOIOMMU mode on {}' - .format(unit.name)) - zaza.utilities.machine_os.disable_vfio_unsafe_noiommu_mode( - unit, model_name=self.model_name) + self.disable_hugepages_vfio_on_hvs_in_vms() class NeutronNetworkingVRRPTests(NeutronNetworkingBase): diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 7abd331..7fa2261 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -28,6 +28,7 @@ import zaza.openstack.configure.guest as configure_guest import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.utilities.machine_os def skipIfNotHA(service_name): @@ -599,6 +600,122 @@ class BaseCharmTest(unittest.TestCase): for unit in units: model.run_on_unit(unit, "hooks/update-status") + def assert_unit_cpu_topology(self, unit, nr_1g_hugepages): + r"""Assert unit under test CPU topology. + + When using OpenStack as CI substrate: + + By default, when instance NUMA placement is not specified, + a topology of N sockets, each with one core and one thread, + is used for an instance, where N corresponds to the number of + instance vCPUs requested. + + In this context a socket is a physical socket on the motherboard + where a CPU is connected. + + The DPDK Environment Abstraction Layer (EAL) allocates memory per + CPU socket, so we want the CPU topology inside the instance to + mimic something we would be likely to find in the real world and + at the same time not make the test too heavy. + + The charm default is to have Open vSwitch allocate 1GB RAM per + CPU socket. + + The following command would set the apropriate CPU topology for a + 4 VCPU, 8 GB RAM flavor: + + openstack flavor set onesocketm1.large \ + --property hw:cpu_sockets=1 \ + --property hw:cpu_cores=2 \ + --property hw:cpu_threads=2 + + For validation of operation with multiple sockets, the following + command would set the apropriate CPU topology for a + 8 VCPU, 16GB RAM flavor: + + openstack flavor set twosocketm1.xlarge \ + --property hw:cpu_sockets=2 \ + --property hw:cpu_cores=2 \ + --property hw:cpu_threads=2 \ + --property hw:numa_nodes=2 + """ + # Get number of sockets + cmd = 'lscpu -p|grep -v ^#|cut -f3 -d,|sort|uniq|wc -l' + sockets = int(zaza.utilities.juju.remote_run( + unit.name, cmd, model_name=self.model_name, fatal=True).rstrip()) + + # Get total memory + cmd = 'cat /proc/meminfo |grep ^MemTotal' + _, meminfo_value, _ = zaza.utilities.juju.remote_run( + unit.name, + cmd, + model_name=self.model_name, + fatal=True).rstrip().split() + mbtotal = int(meminfo_value) * 1024 / 1000 / 1000 + mbtotalhugepages = nr_1g_hugepages * 1024 + + # headroom for operating system and daemons in instance + mbsystemheadroom = 2048 + # memory to be consumed by the nested instance + mbinstance = 1024 + + # the amount of hugepage memory OVS / DPDK EAL will allocate + mbovshugepages = sockets * 1024 + # the amount of hugepage memory available for nested instance + mbfreehugepages = mbtotalhugepages - mbovshugepages + + assert (mbtotal - mbtotalhugepages >= mbsystemheadroom and + mbfreehugepages >= mbinstance), ( + 'Unit {} is not suitable for test, please adjust instance ' + 'type CPU topology or provide suitable physical machine. ' + 'CPU Sockets: {} ' + 'Available memory: {} MB ' + 'Details:\n{}' + .format(unit.name, + sockets, + mbtotal, + self.assert_unit_cpu_topology.__doc__)) + + def enable_hugepages_vfio_on_hvs_in_vms(self, nr_1g_hugepages): + """Enable hugepages and unsafe VFIO NOIOMMU on virtual hypervisors.""" + for unit in model.get_units( + zaza.utilities.machine_os.get_hv_application(), + model_name=self.model_name): + if not zaza.utilities.machine_os.is_vm(unit.name, + model_name=self.model_name): + logging.info('Unit {} is a physical machine, assuming ' + 'hugepages and IOMMU configuration already ' + 'performed through kernel command line.') + continue + logging.info('Checking CPU topology on {}'.format(unit.name)) + self.assert_unit_cpu_topology(unit, nr_1g_hugepages) + logging.info('Enabling hugepages on {}'.format(unit.name)) + zaza.utilities.machine_os.enable_hugepages( + unit, nr_1g_hugepages, model_name=self.model_name) + logging.info('Enabling unsafe VFIO NOIOMMU mode on {}' + .format(unit.name)) + zaza.utilities.machine_os.enable_vfio_unsafe_noiommu_mode( + unit, model_name=self.model_name) + + def disable_hugepages_vfio_on_hvs_in_vms(self): + """Disable hugepages and unsafe VFIO NOIOMMU on virtual hypervisors.""" + for unit in model.get_units( + zaza.utilities.machine_os.get_hv_application(), + model_name=self.model_name): + if not zaza.utilities.machine_os.is_vm(unit.name, + model_name=self.model_name): + logging.info('Unit {} is a physical machine, assuming ' + 'hugepages and IOMMU configuration already ' + 'performed through kernel command line.') + continue + logging.info('Disabling hugepages on {}'.format(unit.name)) + zaza.utilities.machine_os.disable_hugepages( + unit, model_name=self.model_name) + logging.info('Disabling unsafe VFIO NOIOMMU mode on {}' + .format(unit.name)) + zaza.utilities.machine_os.disable_vfio_unsafe_noiommu_mode( + unit, model_name=self.model_name) + class OpenStackBaseTest(BaseCharmTest): """Generic helpers for testing OpenStack API charms.""" From 7984d783fa510a9d254e47a72a0fdda46d56eb80 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 22 Apr 2022 09:07:33 +0200 Subject: [PATCH 09/11] Add undercloud_and_charm_setup configure step The neutron ``basic_overcloud_network`` setup job performs both undercloud and charm configuration prior to configuring the overcloud network. The undercloud and charm configuration steps are useful outside the context of OpenStack Neutron. Split undercloud and charm setup into a new configure step. --- zaza/openstack/charm_tests/neutron/setup.py | 41 +++++++++++++++------ zaza/openstack/configure/network.py | 22 +++++++---- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index ba913da..1402c8f 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -69,29 +69,19 @@ DEFAULT_UNDERCLOUD_NETWORK_CONFIG = { } -def basic_overcloud_network(limit_gws=None): - """Run setup for neutron networking. - - Configure the following: - The overcloud network using subnet pools +def undercloud_and_charm_setup(limit_gws=None): + """Perform undercloud and charm setup for network plumbing. :param limit_gws: Limit the number of gateways that get a port attached :type limit_gws: int """ - cli_utils.setup_logging() - # Get network configuration settings network_config = {} - # Declared overcloud settings - network_config.update(OVERCLOUD_NETWORK_CONFIG) # Default undercloud settings network_config.update(DEFAULT_UNDERCLOUD_NETWORK_CONFIG) # Environment specific settings network_config.update(generic_utils.get_undercloud_env_vars()) - # Get keystone session - keystone_session = openstack_utils.get_overcloud_keystone_session() - # Get optional use_juju_wait for network option options = (lifecycle_utils .get_charm_config(fatal=False) @@ -120,6 +110,33 @@ def basic_overcloud_network(limit_gws=None): ' charm network configuration.' .format(provider_type)) + +def basic_overcloud_network(limit_gws=None): + """Run setup for neutron networking. + + Configure the following: + The overcloud network using subnet pools + + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: int + """ + cli_utils.setup_logging() + + # Get network configuration settings + network_config = {} + # Declared overcloud settings + network_config.update(OVERCLOUD_NETWORK_CONFIG) + # Default undercloud settings + network_config.update(DEFAULT_UNDERCLOUD_NETWORK_CONFIG) + # Environment specific settings + network_config.update(generic_utils.get_undercloud_env_vars()) + + # Get keystone session + keystone_session = openstack_utils.get_overcloud_keystone_session() + + # Perform undercloud and charm setup for network plumbing + undercloud_and_charm_setup(limit_gws=limit_gws) + # Configure the overcloud network network.setup_sdn(network_config, keystone_session=keystone_session) diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 7e4e6f0..cef23f9 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -87,6 +87,7 @@ from zaza.openstack.utilities import ( generic as generic_utils, openstack as openstack_utils, ) +import zaza.openstack.utilities.exceptions import zaza.utilities.juju as juju_utils @@ -276,14 +277,19 @@ def setup_gateway_ext_port(network_config, keystone_session=None, else: net_id = None - # If we're using netplan, we need to add the new interface to the guest - current_release = openstack_utils.get_os_release() - bionic_queens = openstack_utils.get_os_release('bionic_queens') - if current_release >= bionic_queens: - logging.warn("Adding second interface for dataport to guest netplan " - "for bionic-queens and later") - add_dataport_to_netplan = True - else: + try: + # If we're using netplan, we need to add the new interface to the guest + current_release = openstack_utils.get_os_release() + bionic_queens = openstack_utils.get_os_release('bionic_queens') + if current_release >= bionic_queens: + logging.warn("Adding second interface for dataport to guest " + "netplan for bionic-queens and later") + add_dataport_to_netplan = True + else: + add_dataport_to_netplan = False + except zaza.openstack.utilities.exceptions.ApplicationNotFound: + # The setup_gateway_ext_port helper may be used with non-OpenStack + # workloads. add_dataport_to_netplan = False logging.info("Configuring network for OpenStack undercloud/provider") From 7a192e56c9751f268a7f3f84d8b438b5a1ec552a Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 22 Apr 2022 09:19:49 +0200 Subject: [PATCH 10/11] dvr_enabled: Avoid KeyError when Neutron not present --- zaza/openstack/utilities/openstack.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index fdadcb1..1dfe5a5 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -650,7 +650,10 @@ def dvr_enabled(): :returns: True when DVR is enabled, False otherwise :rtype: bool """ - return get_application_config_option('neutron-api', 'enable-dvr') + try: + return get_application_config_option('neutron-api', 'enable-dvr') + except KeyError: + return False def ngw_present(): From ffa98f31b20d853aeefd0e7c08d6e88ff9940746 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 22 Apr 2022 07:46:35 +0200 Subject: [PATCH 11/11] ovn: Add chassis DPDK test --- zaza/openstack/charm_tests/ovn/tests.py | 132 ++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index c99b2b9..5bf551b 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -26,6 +26,7 @@ import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.utilities.juju class BaseCharmOperationTest(test_utils.BaseCharmTest): @@ -196,6 +197,137 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): self.test_config[ 'target_deploy_status'] = stored_target_deploy_status + def _openvswitch_switch_dpdk_installed(self): + """Assert that the openvswitch-switch-dpdk package is installed. + + :raises: zaza.model.CommandRunFailed + """ + cmd = 'dpkg-query -s openvswitch-switch-dpdk' + for unit in zaza.model.get_units(self.application_name): + zaza.utilities.juju.remote_run( + unit.name, cmd, model_name=self.model_name, fatal=True) + + def _ovs_dpdk_init_configured(self): + """Assert that DPDK is configured. + + :raises: AssertionError, zaza.model.CommandRunFailed + """ + cmd = 'ovs-vsctl get open-vswitch . other_config:dpdk-init' + for unit in zaza.model.get_units(self.application_name): + result = zaza.utilities.juju.remote_run( + unit.name, + cmd, + model_name=self.model_name, + fatal=True).rstrip() + assert result == '"true"', ( + 'DPDK not configured on {}'.format(unit.name)) + + def _ovs_dpdk_initialized(self): + """Assert that OVS successfully initialized DPDK. + + :raises: AssertionError, zaza.model.CommandRunFailed + """ + cmd = 'ovs-vsctl get open-vswitch . dpdk_initialized' + for unit in zaza.model.get_units(self.application_name): + result = zaza.utilities.juju.remote_run( + unit.name, + cmd, + model_name=self.model_name, + fatal=True).rstrip() + assert result == 'true', ( + 'DPDK not initialized on {}'.format(unit.name)) + + def _ovs_br_ex_port_is_system_interface(self): + """Assert br-ex bridge is created and has system port in it. + + :raises: zaza.model.CommandRunFailed + """ + cmd = ('ip link show dev $(ovs-vsctl --bare --columns name ' + 'find port external_ids:charm-ovn-chassis=br-ex)') + for unit in zaza.model.get_units(self.application_name): + zaza.utilities.juju.remote_run( + unit.name, cmd, model_name=self.model_name, fatal=True) + + def _ovs_br_ex_port_is_dpdk_interface(self): + """Assert br-ex bridge is created and has DPDK port in it. + + :raises: zaza.model.CommandRunFailed + """ + cmd = ( + 'dpdk-devbind.py --status-dev net ' + '| grep ^$(ovs-vsctl --bare --columns options ' + 'find interface external_ids:charm-ovn-chassis=br-ex ' + '|cut -f2 -d=)' + '|grep "drv=vfio-pci unused=$"') + for unit in zaza.model.get_units(self.application_name): + zaza.utilities.juju.remote_run( + unit.name, cmd, model_name=self.model_name, fatal=True) + + def _ovs_br_ex_interface_not_in_error(self): + """Assert br-ex bridge is created and interface is not in error. + + :raises: AssertionError, zaza.model.CommandRunFailed + """ + cmd = ( + 'ovs-vsctl --bare --columns error ' + 'find interface external_ids:charm-ovn-chassis=br-ex') + for unit in zaza.model.get_units(self.application_name): + result = zaza.utilities.juju.remote_run( + unit.name, + cmd, + model_name=self.model_name, + fatal=True).rstrip() + assert result == '', result + + def _dpdk_pre_post_flight_check(self): + """Assert state of the system before and after enable/disable DPDK.""" + with self.assertRaises( + zaza.model.CommandRunFailed, + msg='openvswitch-switch-dpdk unexpectedly installed'): + self._openvswitch_switch_dpdk_installed() + with self.assertRaises( + zaza.model.CommandRunFailed, + msg='OVS unexpectedly configured for DPDK'): + self._ovs_dpdk_init_configured() + with self.assertRaises( + AssertionError, + msg='OVS unexpectedly has DPDK initialized'): + self._ovs_dpdk_initialized() + + def test_enable_dpdk(self): + """Confirm that transitioning to/from DPDK works.""" + logging.info('Pre-flight check') + self._dpdk_pre_post_flight_check() + self._ovs_br_ex_port_is_system_interface() + + self.enable_hugepages_vfio_on_hvs_in_vms(4) + with self.config_change( + { + 'enable-dpdk': False, + 'dpdk-driver': '', + }, + { + 'enable-dpdk': True, + 'dpdk-driver': 'vfio-pci', + }, + application_name='ovn-chassis'): + logging.info('Checking openvswitch-switch-dpdk is installed') + self._openvswitch_switch_dpdk_installed() + logging.info('Checking DPDK is configured in OVS') + self._ovs_dpdk_init_configured() + logging.info('Checking DPDK is successfully initialized in OVS') + self._ovs_dpdk_initialized() + logging.info('Checking that br-ex configed with DPDK interface...') + self._ovs_br_ex_port_is_dpdk_interface() + logging.info('and is not in error.') + self._ovs_br_ex_interface_not_in_error() + + logging.info('Post-flight check') + self._dpdk_pre_post_flight_check() + + self.disable_hugepages_vfio_on_hvs_in_vms() + self._ovs_br_ex_port_is_system_interface() + class OVSOVNMigrationTest(test_utils.BaseCharmTest): """OVS to OVN migration tests."""