From 899f2b7cd22c75e3a487bd02d7f65564c4007f3b Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 22 Apr 2022 07:46:09 +0200 Subject: [PATCH] Move hugepages/vfio helpers to common test utils Update CPU topology flavor advice. The example currently listed would create a two socket CPU with only one NUMA node, which is a invalid configuration which leads to undefined results. Instead list two examples one with a single socket for simpler end to end tests and a two socket two numa node example for more advanced low level tests. --- zaza/openstack/charm_tests/neutron/tests.py | 105 +----------------- zaza/openstack/charm_tests/test_utils.py | 117 ++++++++++++++++++++ 2 files changed, 119 insertions(+), 103 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 91e77f5..cf925b0 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -1154,95 +1154,9 @@ class DPDKNeutronNetworkingTest(NeutronNetworkingTest): openstack_utils.update_subnet_dhcp( cls.neutron_client, cls.external_subnet, True) - cls.nr_1g_hugepages = 4 - # Prepare hypervisor units for the test - for unit in zaza.model.get_units( - zaza.utilities.machine_os.get_hv_application(), - model_name=cls.model_name): - if not zaza.utilities.machine_os.is_vm(unit.name, - model_name=cls.model_name): - logging.info('Unit {} is a physical machine, assuming ' - 'hugepages and IOMMU configuration already ' - 'performed through kernel command line.') - continue - logging.info('Checking CPU topology on {}'.format(unit.name)) - cls._assert_unit_cpu_topology(cls, unit, model_name=cls.model_name) - logging.info('Enabling hugepages on {}'.format(unit.name)) - zaza.utilities.machine_os.enable_hugepages( - unit, cls.nr_1g_hugepages, model_name=cls.model_name) - logging.info('Enabling unsafe VFIO NOIOMMU mode on {}' - .format(unit.name)) - zaza.utilities.machine_os.enable_vfio_unsafe_noiommu_mode( - unit, model_name=cls.model_name) - - def _assert_unit_cpu_topology(self, unit, model_name=None): - r"""Assert unit under test CPU topology. - - When using OpenStack as CI substrate: - - By default, when instance NUMA placement is not specified, - a topology of N sockets, each with one core and one thread, - is used for an instance, where N corresponds to the number of - instance vCPUs requested. - - In this context a socket is a physical socket on the motherboard - where a CPU is connected. - - The DPDK Environment Abstraction Layer (EAL) allocates memory per - CPU socket, so we want the CPU topology inside the instance to - mimic something we would be likely to find in the real world and - at the same time not make the test too heavy. - - The charm default is to have Open vSwitch allocate 1GB RAM per - CPU socket. - - The following command would set the apropriate CPU topology for a - 4 VCPU flavor: - - openstack flavor set m1.large \ - --property hw:cpu_sockets=2 \ - --property hw:cpu_cores=2 \ - --property hw:cpu_threads=2 - """ - # Get number of sockets - cmd = 'lscpu -p|grep -v ^#|cut -f3 -d,|sort|uniq|wc -l' - sockets = int(zaza.utilities.juju.remote_run( - unit.name, cmd, model_name=model_name, fatal=True).rstrip()) - - # Get total memory - cmd = 'cat /proc/meminfo |grep ^MemTotal' - _, meminfo_value, _ = zaza.utilities.juju.remote_run( - unit.name, - cmd, - model_name=model_name, - fatal=True).rstrip().split() - mbtotal = int(meminfo_value) * 1024 / 1000 / 1000 - mbtotalhugepages = self.nr_1g_hugepages * 1024 - - # headroom for operating system and daemons in instance - mbsystemheadroom = 2048 - # memory to be consumed by the nested instance - mbinstance = 1024 - - # the amount of hugepage memory OVS / DPDK EAL will allocate - mbovshugepages = sockets * 1024 - # the amount of hugepage memory available for nested instance - mbfreehugepages = mbtotalhugepages - mbovshugepages - - assert (mbtotal - mbtotalhugepages >= mbsystemheadroom and - mbfreehugepages >= mbinstance), ( - 'Unit {} is not suitable for test, please adjust instance ' - 'type CPU topology or provide suitable physical machine. ' - 'CPU Sockets: {} ' - 'Available memory: {} MB ' - 'Details:\n{}' - .format(unit.name, - sockets, - mbtotal, - self._assert_unit_cpu_topology.__doc__)) - def test_instances_have_networking(self): """Enable DPDK then Validate North/South and East/West networking.""" + self.enable_hugepages_vfio_on_hvs_in_vms(4) with self.config_change( { 'enable-dpdk': False, @@ -1270,22 +1184,7 @@ class DPDKNeutronNetworkingTest(NeutronNetworkingTest): openstack_utils.update_subnet_dhcp( self.neutron_client, self.external_subnet, False) - for unit in zaza.model.get_units( - zaza.utilities.machine_os.get_hv_application(), - model_name=self.model_name): - if not zaza.utilities.machine_os.is_vm(unit.name, - model_name=self.model_name): - logging.info('Unit {} is a physical machine, assuming ' - 'hugepages and IOMMU configuration already ' - 'performed through kernel command line.') - continue - logging.info('Disabling hugepages on {}'.format(unit.name)) - zaza.utilities.machine_os.disable_hugepages( - unit, model_name=self.model_name) - logging.info('Disabling unsafe VFIO NOIOMMU mode on {}' - .format(unit.name)) - zaza.utilities.machine_os.disable_vfio_unsafe_noiommu_mode( - unit, model_name=self.model_name) + self.disable_hugepages_vfio_on_hvs_in_vms() class NeutronNetworkingVRRPTests(NeutronNetworkingBase): diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 7abd331..7fa2261 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -28,6 +28,7 @@ import zaza.openstack.configure.guest as configure_guest import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.utilities.machine_os def skipIfNotHA(service_name): @@ -599,6 +600,122 @@ class BaseCharmTest(unittest.TestCase): for unit in units: model.run_on_unit(unit, "hooks/update-status") + def assert_unit_cpu_topology(self, unit, nr_1g_hugepages): + r"""Assert unit under test CPU topology. + + When using OpenStack as CI substrate: + + By default, when instance NUMA placement is not specified, + a topology of N sockets, each with one core and one thread, + is used for an instance, where N corresponds to the number of + instance vCPUs requested. + + In this context a socket is a physical socket on the motherboard + where a CPU is connected. + + The DPDK Environment Abstraction Layer (EAL) allocates memory per + CPU socket, so we want the CPU topology inside the instance to + mimic something we would be likely to find in the real world and + at the same time not make the test too heavy. + + The charm default is to have Open vSwitch allocate 1GB RAM per + CPU socket. + + The following command would set the apropriate CPU topology for a + 4 VCPU, 8 GB RAM flavor: + + openstack flavor set onesocketm1.large \ + --property hw:cpu_sockets=1 \ + --property hw:cpu_cores=2 \ + --property hw:cpu_threads=2 + + For validation of operation with multiple sockets, the following + command would set the apropriate CPU topology for a + 8 VCPU, 16GB RAM flavor: + + openstack flavor set twosocketm1.xlarge \ + --property hw:cpu_sockets=2 \ + --property hw:cpu_cores=2 \ + --property hw:cpu_threads=2 \ + --property hw:numa_nodes=2 + """ + # Get number of sockets + cmd = 'lscpu -p|grep -v ^#|cut -f3 -d,|sort|uniq|wc -l' + sockets = int(zaza.utilities.juju.remote_run( + unit.name, cmd, model_name=self.model_name, fatal=True).rstrip()) + + # Get total memory + cmd = 'cat /proc/meminfo |grep ^MemTotal' + _, meminfo_value, _ = zaza.utilities.juju.remote_run( + unit.name, + cmd, + model_name=self.model_name, + fatal=True).rstrip().split() + mbtotal = int(meminfo_value) * 1024 / 1000 / 1000 + mbtotalhugepages = nr_1g_hugepages * 1024 + + # headroom for operating system and daemons in instance + mbsystemheadroom = 2048 + # memory to be consumed by the nested instance + mbinstance = 1024 + + # the amount of hugepage memory OVS / DPDK EAL will allocate + mbovshugepages = sockets * 1024 + # the amount of hugepage memory available for nested instance + mbfreehugepages = mbtotalhugepages - mbovshugepages + + assert (mbtotal - mbtotalhugepages >= mbsystemheadroom and + mbfreehugepages >= mbinstance), ( + 'Unit {} is not suitable for test, please adjust instance ' + 'type CPU topology or provide suitable physical machine. ' + 'CPU Sockets: {} ' + 'Available memory: {} MB ' + 'Details:\n{}' + .format(unit.name, + sockets, + mbtotal, + self.assert_unit_cpu_topology.__doc__)) + + def enable_hugepages_vfio_on_hvs_in_vms(self, nr_1g_hugepages): + """Enable hugepages and unsafe VFIO NOIOMMU on virtual hypervisors.""" + for unit in model.get_units( + zaza.utilities.machine_os.get_hv_application(), + model_name=self.model_name): + if not zaza.utilities.machine_os.is_vm(unit.name, + model_name=self.model_name): + logging.info('Unit {} is a physical machine, assuming ' + 'hugepages and IOMMU configuration already ' + 'performed through kernel command line.') + continue + logging.info('Checking CPU topology on {}'.format(unit.name)) + self.assert_unit_cpu_topology(unit, nr_1g_hugepages) + logging.info('Enabling hugepages on {}'.format(unit.name)) + zaza.utilities.machine_os.enable_hugepages( + unit, nr_1g_hugepages, model_name=self.model_name) + logging.info('Enabling unsafe VFIO NOIOMMU mode on {}' + .format(unit.name)) + zaza.utilities.machine_os.enable_vfio_unsafe_noiommu_mode( + unit, model_name=self.model_name) + + def disable_hugepages_vfio_on_hvs_in_vms(self): + """Disable hugepages and unsafe VFIO NOIOMMU on virtual hypervisors.""" + for unit in model.get_units( + zaza.utilities.machine_os.get_hv_application(), + model_name=self.model_name): + if not zaza.utilities.machine_os.is_vm(unit.name, + model_name=self.model_name): + logging.info('Unit {} is a physical machine, assuming ' + 'hugepages and IOMMU configuration already ' + 'performed through kernel command line.') + continue + logging.info('Disabling hugepages on {}'.format(unit.name)) + zaza.utilities.machine_os.disable_hugepages( + unit, model_name=self.model_name) + logging.info('Disabling unsafe VFIO NOIOMMU mode on {}' + .format(unit.name)) + zaza.utilities.machine_os.disable_vfio_unsafe_noiommu_mode( + unit, model_name=self.model_name) + class OpenStackBaseTest(BaseCharmTest): """Generic helpers for testing OpenStack API charms."""