From b3996949b803f99716614a3bd09e195159c7f8a3 Mon Sep 17 00:00:00 2001 From: Nicolas Pochet Date: Wed, 10 Oct 2018 09:09:06 +0200 Subject: [PATCH] Add ceph osd tests (#93) Create functional tests replacing the Amulet tests that are currently used for testing. --- setup.py | 1 + .../utilities/test_zaza_utilities_ceph.py | 103 ++++ .../utilities/test_zaza_utilities_generic.py | 177 ++++++ zaza/charm_tests/ceph/__init__.py | 15 + zaza/charm_tests/ceph/setup.py | 20 + zaza/charm_tests/ceph/tests.py | 505 ++++++++++++++++++ zaza/utilities/ceph.py | 94 ++++ zaza/utilities/exceptions.py | 54 ++ zaza/utilities/generic.py | 151 ++++++ zaza/utilities/openstack.py | 12 + 10 files changed, 1132 insertions(+) create mode 100644 unit_tests/utilities/test_zaza_utilities_ceph.py create mode 100644 zaza/charm_tests/ceph/__init__.py create mode 100644 zaza/charm_tests/ceph/setup.py create mode 100644 zaza/charm_tests/ceph/tests.py create mode 100644 zaza/utilities/ceph.py diff --git a/setup.py b/setup.py index 070495e..32b0701 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ install_require = [ 'python-keystoneclient', 'python-novaclient', 'python-neutronclient', + 'python-cinderclient', ] tests_require = [ diff --git a/unit_tests/utilities/test_zaza_utilities_ceph.py b/unit_tests/utilities/test_zaza_utilities_ceph.py new file mode 100644 index 0000000..9e6658e --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_ceph.py @@ -0,0 +1,103 @@ +import unit_tests.utils as ut_utils +import zaza.model as model +import zaza.utilities.ceph as ceph_utils +import zaza.utilities.openstack as openstack_utils + + +class TestCephUtils(ut_utils.BaseTestCase): + + def setUp(self): + super(TestCephUtils, self).setUp() + + def _test_expected_pools(self, + os_release_pair, + expected_pools, + radosgw=False): + self.get_current_os_release_pair.return_value = os_release_pair + actual_pools = ceph_utils.get_expected_pools(radosgw) + self.assertEqual(expected_pools, actual_pools) + + def test_get_expected_pools(self): + self.patch_object(openstack_utils, 'get_current_os_release_pair') + + # Trusty Icehouse + os_release_pair = 'trusty_icehouse' + self.get_current_os_release_pair.return_value = 'trusty_icehouse' + expected_pools = [ + 'data', + 'metadata', + 'rbd', + 'cinder-ceph', + 'glance' + ] + self._test_expected_pools(os_release_pair, expected_pools) + + # Xenial Ocata + os_release_pair = 'xenial_ocata' + expected_pools = [ + 'rbd', + 'cinder-ceph', + 'glance' + ] + self._test_expected_pools(os_release_pair, expected_pools) + + # Xenial Queens + os_release_pair = 'xenial_queens' + expected_pools = [ + 'cinder-ceph', + 'glance' + ] + self._test_expected_pools(os_release_pair, expected_pools) + + # Xenial Queens with radosgw + os_release_pair = 'xenial_queens' + expected_pools = [ + 'cinder-ceph', + 'glance', + '.rgw.root', + '.rgw.control', + '.rgw', + '.rgw.gc', + '.users.uid' + ] + self._test_expected_pools(os_release_pair, expected_pools, True) + + def test_get_ceph_pools(self): + self.patch_object(model, 'run_on_unit') + + # Bad return code + result = { + 'Code': '1', + 'Stdout': '', + 'Stderr': 'something went wrong', + } + self.run_on_unit.return_value = result + with self.assertRaises(model.CommandRunFailed): + ceph_utils.get_ceph_pools('ceph-mon/0') + + # Xenial Queens output + result = { + 'Code': '0', + 'Stdout': '1 cinder-ceph,2 glance,', + 'Stderr': '' + } + self.run_on_unit.return_value = result + expected = { + 'cinder-ceph': 1, + 'glance': 2 + } + actual = ceph_utils.get_ceph_pools('ceph-mon/0') + self.assertEqual(expected, actual) + # Bionic Queens output + result = { + 'Code': '0', + 'Stdout': '1 cinder-ceph\n2 glance', + 'Stderr': '' + } + self.run_on_unit.return_value = result + expected = { + 'cinder-ceph': 1, + 'glance': 2 + } + actual = ceph_utils.get_ceph_pools('ceph-mon/0') + self.assertEqual(expected, actual) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 46313b1..3036238 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -15,6 +15,7 @@ import mock import unit_tests.utils as ut_utils from zaza.utilities import generic as generic_utils +import zaza.utilities.exceptions as zaza_exceptions FAKE_STATUS = { 'can-upgrade-to': '', @@ -368,3 +369,179 @@ class TestGenericUtils(ut_utils.BaseTestCase): '/etc/apt/apt.conf.d/50unattended-upgrades || ' 'echo \'DPkg::options { "--force-confdef"; };\' >> ' '/etc/apt/apt.conf.d/50unattended-upgrades') + + def test_get_process_id_list(self): + self.patch( + "zaza.utilities.generic.model.run_on_unit", + new_callable=mock.MagicMock(), + name="_run" + ) + + # Return code is OK and STDOUT contains output + returns_ok = { + "Code": 0, + "Stdout": "1 2", + "Stderr": "" + } + self._run.return_value = returns_ok + p_id_list = generic_utils.get_process_id_list( + "ceph-osd/0", + "ceph-osd", + False + ) + expected = ["1", "2"] + cmd = 'pidof -x "ceph-osd" || exit 0 && exit 1' + self.assertEqual(p_id_list, expected) + self._run.assert_called_once_with(unit_name="ceph-osd/0", + command=cmd) + + # Return code is not OK + returns_nok = { + "Code": 1, + "Stdout": "", + "Stderr": "Something went wrong" + } + self._run.return_value = returns_nok + with self.assertRaises(zaza_exceptions.ProcessIdsFailed): + generic_utils.get_process_id_list("ceph-osd/0", "ceph") + cmd = 'pidof -x "ceph"' + self._run.assert_called_once_with(unit_name="ceph-osd/0", + command=cmd) + + def test_get_unit_process_ids(self): + self.patch( + "zaza.utilities.generic.get_process_id_list", + new_callable=mock.MagicMock(), + name="_get_pids" + ) + + pids = ["1", "2"] + self._get_pids.return_value = pids + unit_processes = { + "ceph-osd/0": { + "ceph-osd": 2 + }, + "unit/0": { + "pr1": 2, + "pr2": 2 + } + } + expected = { + "ceph-osd/0": { + "ceph-osd": ["1", "2"] + }, + "unit/0": { + "pr1": ["1", "2"], + "pr2": ["1", "2"] + } + } + result = generic_utils.get_unit_process_ids(unit_processes) + self.assertEqual(result, expected) + + def test_validate_unit_process_ids(self): + expected = { + "ceph-osd/0": { + "ceph-osd": 2 + }, + "unit/0": { + "pr1": 2, + "pr2": [1, 2] + } + } + + # Unit count mismatch + actual = {} + with self.assertRaises(zaza_exceptions.UnitCountMismatch): + generic_utils.validate_unit_process_ids(expected, actual) + + # Unit not found in actual dict + actual = { + "ceph-osd/0": { + "ceph-osd": ["1", "2"] + }, + # unit/0 not in the dict + "unit/1": { + "pr1": ["1", "2"], + "pr2": ["1", "2"] + } + } + with self.assertRaises(zaza_exceptions.UnitNotFound): + generic_utils.validate_unit_process_ids(expected, actual) + + # Process names count doesn't match + actual = { + "ceph-osd/0": { + "ceph-osd": ["1", "2"] + }, + "unit/0": { + # Only one process name instead of 2 expected + "pr1": ["1", "2"] + } + } + with self.assertRaises(zaza_exceptions.ProcessNameCountMismatch): + generic_utils.validate_unit_process_ids(expected, actual) + + # Process name doesn't match + actual = { + "ceph-osd/0": { + "ceph-osd": ["1", "2"] + }, + "unit/0": { + # Bad process name + "bad_name": ["1", "2"], + "pr2": ["1", "2"] + } + } + with self.assertRaises(zaza_exceptions.ProcessNameMismatch): + generic_utils.validate_unit_process_ids(expected, actual) + + # PID count doesn't match + actual = { + "ceph-osd/0": { + "ceph-osd": ["1", "2"] + }, + "unit/0": { + # Only one PID instead of 2 expected + "pr1": ["2"], + "pr2": ["1", "2"] + } + } + with self.assertRaises(zaza_exceptions.PIDCountMismatch): + generic_utils.validate_unit_process_ids(expected, actual) + + actual = { + "ceph-osd/0": { + "ceph-osd": ["1", "2"] + }, + "unit/0": { + "pr1": ["1", "2"], + # 3 PID instead of [1, 2] expected + "pr2": ["1", "2", "3"] + } + } + with self.assertRaises(zaza_exceptions.PIDCountMismatch): + generic_utils.validate_unit_process_ids(expected, actual) + + # It should work now... + actual = { + "ceph-osd/0": { + "ceph-osd": ["1", "2"] + }, + "unit/0": { + "pr1": ["1", "2"], + "pr2": ["1", "2"] + } + } + ret = generic_utils.validate_unit_process_ids(expected, actual) + self.assertTrue(ret) + + def test_get_ubuntu_release(self): + # Normal case + expected = 0 + actual = generic_utils.get_ubuntu_release('oneiric') + self.assertEqual(expected, actual) + + # Ubuntu release doesn't exist + bad_name = 'bad_name' + with self.assertRaises(zaza_exceptions.UbuntuReleaseNotFound): + generic_utils.get_ubuntu_release(bad_name) diff --git a/zaza/charm_tests/ceph/__init__.py b/zaza/charm_tests/ceph/__init__.py new file mode 100644 index 0000000..2e8a46b --- /dev/null +++ b/zaza/charm_tests/ceph/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ceph-osd.""" diff --git a/zaza/charm_tests/ceph/setup.py b/zaza/charm_tests/ceph/setup.py new file mode 100644 index 0000000..c53ff3c --- /dev/null +++ b/zaza/charm_tests/ceph/setup.py @@ -0,0 +1,20 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup for ceph-osd deployments.""" + + +def basic_setup(): + """Run basic setup for ceph-osd.""" + pass diff --git a/zaza/charm_tests/ceph/tests.py b/zaza/charm_tests/ceph/tests.py new file mode 100644 index 0000000..6a4a84f --- /dev/null +++ b/zaza/charm_tests/ceph/tests.py @@ -0,0 +1,505 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ceph-osd Testing.""" + +import logging +from os import ( + listdir, + path +) +import tempfile + +import zaza.charm_tests.test_utils as test_utils +import zaza.model as zaza_model +import zaza.utilities.ceph as zaza_ceph +import zaza.utilities.exceptions as zaza_exceptions +import zaza.utilities.generic as zaza_utils +import zaza.utilities.juju as zaza_juju +import zaza.utilities.openstack as zaza_openstack + + +class CephLowLevelTest(test_utils.OpenStackBaseTest): + """Ceph Low Level Test Class.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph low level tests.""" + super(CephLowLevelTest, cls).setUpClass() + + def test_processes(self): + """Verify Ceph processes. + + Verify that the expected service processes are running + on each ceph unit. + """ + logging.info('Checking ceph-mon and ceph-osd processes...') + # Process name and quantity of processes to expect on each unit + ceph_mon_processes = { + 'ceph-mon': 1, + } + + ceph_osd_processes = { + 'ceph-osd': [2, 3] + } + + # Units with process names and PID quantities expected + expected_processes = { + 'ceph-mon/0': ceph_mon_processes, + 'ceph-mon/1': ceph_mon_processes, + 'ceph-mon/2': ceph_mon_processes, + 'ceph-osd/0': ceph_osd_processes, + 'ceph-osd/1': ceph_osd_processes, + 'ceph-osd/2': ceph_osd_processes + } + + actual_pids = zaza_utils.get_unit_process_ids(expected_processes) + ret = zaza_utils.validate_unit_process_ids(expected_processes, + actual_pids) + self.assertTrue(ret) + + def test_services(self): + """Verify the ceph services. + + Verify the expected services are running on the service units. + """ + logging.info('Checking ceph-osd and ceph-mon services...') + services = {} + ceph_services = ['ceph-mon'] + services['ceph-osd/0'] = ['ceph-osd'] + + services['ceph-mon/0'] = ceph_services + services['ceph-mon/1'] = ceph_services + services['ceph-mon/2'] = ceph_services + + for unit_name, unit_services in services.items(): + zaza_model.block_until_service_status( + unit_name=unit_name, + services=unit_services, + target_status='running' + ) + + +class CephRelationTest(test_utils.OpenStackBaseTest): + """Ceph's relations test class.""" + + @classmethod + def setUpClass(cls): + """Run the ceph's relations class setup.""" + super(CephRelationTest, cls).setUpClass() + + def test_ceph_osd_ceph_relation_address(self): + """Verify the ceph-osd to ceph relation data.""" + logging.info('Checking ceph-osd:ceph-mon relation data...') + unit_name = 'ceph-osd/0' + remote_unit_name = 'ceph-mon/0' + relation_name = 'osd' + remote_unit = zaza_model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + # Get private-address in relation + rel_private_ip = relation.get('private-address') + # The private address in relation should match ceph-mon/0 address + self.assertEqual(rel_private_ip, remote_ip) + + def _ceph_to_ceph_osd_relation(self, remote_unit_name): + """Verify the cephX to ceph-osd relation data. + + Helper function to test the relation. + """ + logging.info('Checking {}:ceph-osd mon relation data...'. + format(remote_unit_name)) + unit_name = 'ceph-osd/0' + relation_name = 'osd' + remote_unit = zaza_model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + cmd = 'leader-get fsid' + result = zaza_model.run_on_unit(remote_unit_name, cmd) + fsid = result.get('Stdout').strip() + expected = { + 'private-address': remote_ip, + 'auth': 'none', + 'ceph-public-address': remote_ip, + 'fsid': fsid, + } + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + for e_key, e_value in expected.items(): + a_value = relation[e_key] + self.assertEqual(e_value, a_value) + self.assertTrue(relation['osd_bootstrap_key'] is not None) + + def test_ceph0_to_ceph_osd_relation(self): + """Verify the ceph0 to ceph-osd relation data.""" + remote_unit_name = 'ceph-mon/0' + self._ceph_to_ceph_osd_relation(remote_unit_name) + + def test_ceph1_to_ceph_osd_relation(self): + """Verify the ceph1 to ceph-osd relation data.""" + remote_unit_name = 'ceph-mon/1' + self._ceph_to_ceph_osd_relation(remote_unit_name) + + def test_ceph2_to_ceph_osd_relation(self): + """Verify the ceph2 to ceph-osd relation data.""" + remote_unit_name = 'ceph-mon/2' + self._ceph_to_ceph_osd_relation(remote_unit_name) + + +class CephTest(test_utils.OpenStackBaseTest): + """Ceph common functional tests.""" + + @classmethod + def setUpClass(cls): + """Run the ceph's common class setup.""" + super(CephTest, cls).setUpClass() + + def pause_resume(self, services): + """Run Pause and resume tests. + + Override the default implementation since pausing ceph units + doesn't stop the services. + Pause and then resume a unit checking that services are in the + required state after each action + + :param services: Services expected to be restarted when config_file is + changed. + :type services: list + """ + zaza_model.block_until_service_status( + self.lead_unit, + services, + 'running', + model_name=self.model_name) + zaza_model.block_until_unit_wl_status( + self.lead_unit, + 'active', + model_name=self.model_name) + zaza_model.run_action( + self.lead_unit, + 'pause', + model_name=self.model_name) + zaza_model.block_until_unit_wl_status( + self.lead_unit, + 'maintenance', + model_name=self.model_name) + zaza_model.block_until_all_units_idle(model_name=self.model_name) + zaza_model.run_action( + self.lead_unit, + 'resume', + model_name=self.model_name) + zaza_model.block_until_unit_wl_status( + self.lead_unit, + 'active', + model_name=self.model_name) + zaza_model.block_until_all_units_idle(model_name=self.model_name) + zaza_model.block_until_service_status( + self.lead_unit, + services, + 'running', + model_name=self.model_name) + + def test_ceph_check_osd_pools(self): + """Check OSD pools. + + Check osd pools on all ceph units, expect them to be + identical, and expect specific pools to be present. + """ + logging.info('Checking pools on ceph units...') + + expected_pools = zaza_ceph.get_expected_pools() + results = [] + unit_name = 'ceph-mon/0' + + # Check for presence of expected pools on each unit + logging.debug('Expected pools: {}'.format(expected_pools)) + pools = zaza_ceph.get_ceph_pools(unit_name) + results.append(pools) + + for expected_pool in expected_pools: + if expected_pool not in pools: + msg = ('{} does not have pool: ' + '{}'.format(unit_name, expected_pool)) + raise zaza_exceptions.CephPoolNotFound(msg) + logging.debug('{} has (at least) the expected ' + 'pools.'.format(unit_name)) + + # Check that all units returned the same pool name:id data + for i, result in enumerate(results): + for other in results[i+1:]: + logging.debug('result: {}, other: {}'.format(result, other)) + self.assertEqual(result, other) + + def test_ceph_pool_creation_with_text_file(self): + """Check the creation of a pool and a text file. + + Create a pool, add a text file to it and retrieve its content. + Verify that the content matches the original file. + """ + unit_name = 'ceph-mon/0' + cmd = 'sudo ceph osd pool create test 128; \ + echo 123456789 > /tmp/input.txt; \ + rados put -p test test_input /tmp/input.txt; \ + rados get -p test test_input /dev/stdout' + logging.debug('Creating test pool and putting test file in pool...') + result = zaza_model.run_on_unit(unit_name, cmd) + code = result.get('Code') + if code != '0': + raise zaza_model.CommandRunFailed(cmd, result) + output = result.get('Stdout').strip() + logging.debug('Output received: {}'.format(output)) + self.assertEqual(output, '123456789') + + def test_ceph_encryption(self): + """Test Ceph encryption. + + Verify that the new disk is added with encryption by checking for + Ceph's encryption keys directory. + """ + current_release = zaza_openstack.get_os_release() + trusty_mitaka = zaza_openstack.get_os_release('trusty_mitaka') + if current_release >= trusty_mitaka: + logging.warn("Skipping encryption test for Mitaka and higher") + return + unit_name = 'ceph-osd/0' + set_default = { + 'osd-encrypt': 'False', + 'osd-devices': '/dev/vdb /srv/ceph', + } + set_alternate = { + 'osd-encrypt': 'True', + 'osd-devices': '/dev/vdb /srv/ceph /srv/ceph_encrypted', + } + juju_service = 'ceph-osd' + logging.info('Making config change on {}...'.format(juju_service)) + mtime = zaza_model.get_unit_time(unit_name) + + file_mtime = None + + folder_name = '/etc/ceph/dmcrypt-keys/' + with self.config_change(set_default, set_alternate): + with tempfile.TemporaryDirectory() as tempdir: + # Creating a temp dir to copy keys + temp_folder = '/tmp/dmcrypt-keys' + cmd = 'mkdir {}'.format(temp_folder) + ret = zaza_model.run_on_unit(unit_name, cmd) + logging.debug('Ret for cmd {} is {}'.format(cmd, ret)) + # Copy keys from /etc to /tmp + cmd = 'sudo cp {}* {}'.format(folder_name, temp_folder) + ret = zaza_model.run_on_unit(unit_name, cmd) + logging.debug('Ret for cmd {} is {}'.format(cmd, ret)) + # Changing permissions to be able to SCP the files + cmd = 'sudo chown -R ubuntu:ubuntu {}'.format(temp_folder) + ret = zaza_model.run_on_unit(unit_name, cmd) + logging.debug('Ret for cmd {} is {}'.format(cmd, ret)) + # SCP to retrieve all files in folder + # -p: preserve timestamps + source = '/tmp/dmcrypt-keys/*' + zaza_model.scp_from_unit(unit_name=unit_name, + source=source, + destination=tempdir, + scp_opts='-p') + for elt in listdir(tempdir): + file_path = '/'.join([tempdir, elt]) + if path.isfile(file_path): + file_mtime = path.getmtime(file_path) + if file_mtime: + break + + if not file_mtime: + logging.warn('Could not determine mtime, assuming ' + 'folder does not exist') + raise FileNotFoundError('folder does not exist') + + if file_mtime >= mtime: + logging.info('Folder mtime is newer than provided mtime ' + '(%s >= %s) on %s (OK)' % (file_mtime, + mtime, unit_name)) + else: + logging.warn('Folder mtime is older than provided mtime' + '(%s < on %s) on %s' % (file_mtime, + mtime, unit_name)) + raise Exception('Folder mtime is older than provided mtime') + + def test_blocked_when_non_pristine_disk_appears(self): + """Test blocked state with non-pristine disk. + + Validate that charm goes into blocked state when it is presented with + new block devices that have foreign data on them. + Instances used in UOSCI has a flavour with ephemeral storage in + addition to the bootable instance storage. The ephemeral storage + device is partitioned, formatted and mounted early in the boot process + by cloud-init. + As long as the device is mounted the charm will not attempt to use it. + If we unmount it and trigger the config-changed hook the block device + will appear as a new and previously untouched device for the charm. + One of the first steps of device eligibility checks should be to make + sure we are seeing a pristine and empty device before doing any + further processing. + As the ephemeral device will have data on it we can use it to validate + that these checks work as intended. + """ + logging.info('Checking behaviour when non-pristine disks appear...') + logging.info('Configuring ephemeral-unmount...') + alternate_conf = { + 'ephemeral-unmount': '/mnt', + 'osd-devices': '/dev/vdb' + } + juju_service = 'ceph-osd' + zaza_model.set_application_config(juju_service, alternate_conf) + ceph_osd_states = { + 'ceph-osd': { + 'workload-status': 'blocked', + 'workload-status-message': 'Non-pristine' + } + } + zaza_model.wait_for_application_states(states=ceph_osd_states) + logging.info('Units now in blocked state, running zap-disk action...') + unit_names = ['ceph-osd/0', 'ceph-osd/1', 'ceph-osd/2'] + for unit_name in unit_names: + zap_disk_params = { + 'devices': '/dev/vdb', + 'i-really-mean-it': True, + } + action_obj = zaza_model.run_action( + unit_name=unit_name, + action_name='zap-disk', + action_params=zap_disk_params + ) + logging.debug('Result of action: {}'.format(action_obj)) + + logging.info('Running add-disk action...') + for unit_name in unit_names: + add_disk_params = { + 'osd-devices': '/dev/vdb', + } + action_obj = zaza_model.run_action( + unit_name=unit_name, + action_name='add-disk', + action_params=add_disk_params + ) + logging.debug('Result of action: {}'.format(action_obj)) + + logging.info('Wait for idle/ready status...') + zaza_model.wait_for_application_states() + + logging.info('OK') + + set_default = { + 'ephemeral-unmount': '', + 'osd-devices': '/dev/vdb /srv/ceph', + } + + logging.info('Restoring to default configuration...') + zaza_model.set_application_config(juju_service, set_default) + + zaza_model.wait_for_application_states() + + def test_pause_and_resume(self): + """The services can be paused and resumed.""" + logging.info('Checking pause and resume actions...') + self.pause_resume(['ceph-osd']) + + def test_blacklist(self): + """Check the blacklist action. + + The blacklist actions execute and behave as expected. + """ + logging.info('Checking blacklist-add-disk and' + 'blacklist-remove-disk actions...') + unit_name = 'ceph-osd/0' + + zaza_model.block_until_unit_wl_status( + unit_name, + 'active' + ) + + # Attempt to add device with non-absolute path should fail + action_obj = zaza_model.run_action( + unit_name=unit_name, + action_name='blacklist-add-disk', + action_params={'osd-devices': 'vda'} + ) + self.assertTrue(action_obj.status != 'completed') + zaza_model.block_until_unit_wl_status( + unit_name, + 'active' + ) + + # Attempt to add device with non-existent path should fail + action_obj = zaza_model.run_action( + unit_name=unit_name, + action_name='blacklist-add-disk', + action_params={'osd-devices': '/non-existent'} + ) + self.assertTrue(action_obj.status != 'completed') + zaza_model.block_until_unit_wl_status( + unit_name, + 'active' + ) + + # Attempt to add device with existent path should succeed + action_obj = zaza_model.run_action( + unit_name=unit_name, + action_name='blacklist-add-disk', + action_params={'osd-devices': '/dev/vda'} + ) + self.assertEqual('completed', action_obj.status) + zaza_model.block_until_unit_wl_status( + unit_name, + 'active' + ) + + # Attempt to remove listed device should always succeed + action_obj = zaza_model.run_action( + unit_name=unit_name, + action_name='blacklist-remove-disk', + action_params={'osd-devices': '/dev/vda'} + ) + self.assertEqual('completed', action_obj.status) + zaza_model.block_until_unit_wl_status( + unit_name, + 'active' + ) + logging.debug('OK') + + def test_list_disks(self): + """Test the list-disks action. + + The list-disks action execute. + """ + logging.info('Checking list-disks action...') + unit_name = 'ceph-osd/0' + + zaza_model.block_until_unit_wl_status( + unit_name, + 'active' + ) + + action_obj = zaza_model.run_action( + unit_name=unit_name, + action_name='list-disks', + ) + self.assertEqual('completed', action_obj.status) + zaza_model.block_until_unit_wl_status( + unit_name, + 'active' + ) + logging.debug('OK') diff --git a/zaza/utilities/ceph.py b/zaza/utilities/ceph.py new file mode 100644 index 0000000..08f8b89 --- /dev/null +++ b/zaza/utilities/ceph.py @@ -0,0 +1,94 @@ +"""Module containing Ceph related utilities.""" + +import logging + +import zaza.utilities.openstack as openstack_utils +import zaza.model as zaza_model + + +def get_expected_pools(radosgw=False): + """Get expected ceph pools. + + Return a list of expected ceph pools in a ceph + cinder + glance + test scenario, based on OpenStack release and whether ceph radosgw + is flagged as present or not. + :param radosgw: If radosgw is used or not + :type radosgw: boolean + :returns: List of pools that are expected + :rtype: list + """ + current_release = openstack_utils.get_os_release() + trusty_icehouse = openstack_utils.get_os_release('trusty_icehouse') + trusty_kilo = openstack_utils.get_os_release('trusty_kilo') + zesty_ocata = openstack_utils.get_os_release('zesty_ocata') + if current_release == trusty_icehouse: + # Icehouse + pools = [ + 'data', + 'metadata', + 'rbd', + 'cinder-ceph', + 'glance' + ] + elif (trusty_kilo <= current_release <= zesty_ocata): + # Kilo through Ocata + pools = [ + 'rbd', + 'cinder-ceph', + 'glance' + ] + else: + # Pike and later + pools = [ + 'cinder-ceph', + 'glance' + ] + + if radosgw: + pools.extend([ + '.rgw.root', + '.rgw.control', + '.rgw', + '.rgw.gc', + '.users.uid' + ]) + + return pools + + +def get_ceph_pools(unit_name): + """Get ceph pools. + + Return a dict of ceph pools from a single ceph unit, with + pool name as keys, pool id as vals. + :param unit_name: Name of the unit to get the pools on + :type unit_name: string + :returns: Dict of ceph pools + :rtype: dict + :raise: zaza_model.CommandRunFailed + """ + pools = {} + cmd = 'sudo ceph osd lspools' + result = zaza_model.run_on_unit(unit_name, cmd) + output = result.get('Stdout').strip() + code = int(result.get('Code')) + if code != 0: + raise zaza_model.CommandRunFailed(cmd, result) + + # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance, + # It can also be something link 0 data\n1 metadata + + # First split on new lines + osd_pools = str(output).split('\n') + # If we have a len of 1, no new lines found -> splitting on commas + if len(osd_pools) == 1: + osd_pools = osd_pools[0].split(',') + for pool in osd_pools: + pool_id_name = pool.split(' ') + if len(pool_id_name) == 2: + pool_id = pool_id_name[0] + pool_name = pool_id_name[1] + pools[pool_name] = int(pool_id) + + logging.debug('Pools on {}: {}'.format(unit_name, pools)) + return pools diff --git a/zaza/utilities/exceptions.py b/zaza/utilities/exceptions.py index 749a983..2886d83 100644 --- a/zaza/utilities/exceptions.py +++ b/zaza/utilities/exceptions.py @@ -106,3 +106,57 @@ class KeystoneKeyRepositoryError(Exception): """ pass + + +class ProcessNameCountMismatch(Exception): + """Count of process names doesn't match.""" + + pass + + +class ProcessNameMismatch(Exception): + """Name of processes doesn't match.""" + + pass + + +class PIDCountMismatch(Exception): + """PID's count doesn't match.""" + + pass + + +class ProcessIdsFailed(Exception): + """Process ID lookup failed.""" + + pass + + +class UnitNotFound(Exception): + """Unit not found in actual dict.""" + + pass + + +class UnitCountMismatch(Exception): + """Count of unit doesn't match.""" + + pass + + +class UbuntuReleaseNotFound(Exception): + """Ubuntu release not found in list.""" + + pass + + +class ServiceNotFound(Exception): + """Service not found on unit.""" + + pass + + +class CephPoolNotFound(Exception): + """Ceph pool not found.""" + + pass diff --git a/zaza/utilities/generic.py b/zaza/utilities/generic.py index 2d53c39..16d606c 100644 --- a/zaza/utilities/generic.py +++ b/zaza/utilities/generic.py @@ -21,6 +21,8 @@ import yaml from zaza import model from zaza.utilities import juju as juju_utils +from zaza.utilities import exceptions as zaza_exceptions +from zaza.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE def dict_to_yaml(dict_data): @@ -458,3 +460,152 @@ def set_dpkg_non_interactive_on_unit( cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}" .format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d)) model.run_on_unit(unit_name, cmd) + + +def get_process_id_list(unit_name, process_name, + expect_success=True): + """Get a list of process ID(s). + + Get a list of process ID(s) from a single sentry juju unit + for a single process name. + + :param unit_name: Amulet sentry instance (juju unit) + :param process_name: Process name + :param expect_success: If False, expect the PID to be missing, + raise if it is present. + :returns: List of process IDs + :raises: zaza_exceptions.ProcessIdsFailed + """ + cmd = 'pidof -x "{}"'.format(process_name) + if not expect_success: + cmd += " || exit 0 && exit 1" + results = model.run_on_unit(unit_name=unit_name, command=cmd) + code = results.get("Code", 1) + try: + code = int(code) + except ValueError: + code = 1 + error = results.get("Stderr") + output = results.get("Stdout") + if code != 0: + msg = ('{} `{}` returned {} ' + '{} with error {}'.format(unit_name, cmd, code, output, error)) + raise zaza_exceptions.ProcessIdsFailed(msg) + return str(output).split() + + +def get_unit_process_ids(unit_processes, expect_success=True): + """Get unit process ID(s). + + Construct a dict containing unit sentries, process names, and + process IDs. + + :param unit_processes: A dictionary of unit names + to list of process names. + :param expect_success: if False expect the processes to not be + running, raise if they are. + :returns: Dictionary of unit names to dictionary + of process names to PIDs. + :raises: zaza_exceptions.ProcessIdsFailed + """ + pid_dict = {} + for unit_name, process_list in unit_processes.items(): + pid_dict[unit_name] = {} + for process in process_list: + pids = get_process_id_list( + unit_name, process, expect_success=expect_success) + pid_dict[unit_name].update({process: pids}) + return pid_dict + + +def validate_unit_process_ids(expected, actual): + """Validate process id quantities for services on units. + + :returns: True if the PIDs are validated, raises an exception + if it is not the case. + :raises: zaza_exceptions.UnitCountMismatch + :raises: zaza_exceptions.UnitNotFound + :raises: zaza_exceptions.ProcessNameCountMismatch + :raises: zaza_exceptions.ProcessNameMismatch + :raises: zaza_exceptions.PIDCountMismatch + """ + logging.debug('Checking units for running processes...') + logging.debug('Expected PIDs: {}'.format(expected)) + logging.debug('Actual PIDs: {}'.format(actual)) + + if len(actual) != len(expected): + msg = ('Unit count mismatch. expected, actual: {}, ' + '{} '.format(len(expected), len(actual))) + raise zaza_exceptions.UnitCountMismatch(msg) + + for (e_unit_name, e_proc_names) in expected.items(): + if e_unit_name in actual.keys(): + a_proc_names = actual[e_unit_name] + else: + msg = ('Expected unit ({}) not found in actual dict data.'. + format(e_unit_name)) + raise zaza_exceptions.UnitNotFound(msg) + + if len(e_proc_names.keys()) != len(a_proc_names.keys()): + msg = ('Process name count mismatch. expected, actual: {}, ' + '{}'.format(len(expected), len(actual))) + raise zaza_exceptions.ProcessNameCountMismatch(msg) + + for (e_proc_name, e_pids), (a_proc_name, a_pids) in \ + zip(e_proc_names.items(), a_proc_names.items()): + if e_proc_name != a_proc_name: + msg = ('Process name mismatch. expected, actual: {}, ' + '{}'.format(e_proc_name, a_proc_name)) + raise zaza_exceptions.ProcessNameMismatch(msg) + + a_pids_length = len(a_pids) + fail_msg = ('PID count mismatch. {} ({}) expected, actual: ' + '{}, {} ({})'.format(e_unit_name, e_proc_name, + e_pids, a_pids_length, + a_pids)) + + # If expected is a list, ensure at least one PID quantity match + if isinstance(e_pids, list) and \ + a_pids_length not in e_pids: + raise zaza_exceptions.PIDCountMismatch(fail_msg) + # If expected is not bool and not list, + # ensure PID quantities match + elif not isinstance(e_pids, bool) and \ + not isinstance(e_pids, list) and \ + a_pids_length != e_pids: + raise zaza_exceptions.PIDCountMismatch(fail_msg) + # If expected is bool True, ensure 1 or more PIDs exist + elif isinstance(e_pids, bool) and \ + e_pids is True and a_pids_length < 1: + raise zaza_exceptions.PIDCountMismatch(fail_msg) + # If expected is bool False, ensure 0 PIDs exist + elif isinstance(e_pids, bool) and \ + e_pids is False and a_pids_length != 0: + raise zaza_exceptions.PIDCountMismatch(fail_msg) + else: + logging.debug('PID check OK: {} {} {}: ' + '{}'.format(e_unit_name, e_proc_name, + e_pids, a_pids)) + return True + + +def get_ubuntu_release(ubuntu_name): + """Get index of Ubuntu release. + + Returns the index of the name of the Ubuntu release in + UBUNTU_OPENSTACK_RELEASE. + + :param ubuntu_name: Name of the Ubuntu release. + :type ubuntu_name: string + :returns: Index of the Ubuntu release + :rtype: integer + :raises: zaza_exceptions.UbuntuReleaseNotFound + """ + ubuntu_releases = list(UBUNTU_OPENSTACK_RELEASE.keys()) + try: + index = ubuntu_releases.index(ubuntu_name) + except ValueError: + msg = ('Could not find Ubuntu release {} in {}'. + format(ubuntu_name, UBUNTU_OPENSTACK_RELEASE)) + raise zaza_exceptions.UbuntuReleaseNotFound(msg) + return index diff --git a/zaza/utilities/openstack.py b/zaza/utilities/openstack.py index d977bf8..fdf3c1e 100644 --- a/zaza/utilities/openstack.py +++ b/zaza/utilities/openstack.py @@ -23,6 +23,7 @@ from .os_versions import ( OPENSTACK_RELEASES_PAIRS, ) +from cinderclient import client as cinderclient from glanceclient import Client as GlanceClient from keystoneclient.v2_0 import client as keystoneclient_v2 @@ -199,6 +200,17 @@ def get_neutron_session_client(session): return neutronclient.Client(session=session) +def get_cinder_session_client(session): + """Return cinderclient authenticated by keystone session. + + :param session: Keystone session object + :type session: keystoneauth1.session.Session object + :returns: Authenticated cinderclient + :rtype: cinderclient.Client object + """ + return cinderclient.Client(session=session) + + def get_keystone_scope(): """Return Keystone scope based on OpenStack release of the overcloud.