Add ceph osd tests (#93)

Create functional tests replacing the Amulet tests that are currently used for testing.

setup.py

@@ -38,6 +38,7 @@ install_require = [
     'python-keystoneclient',
     'python-novaclient',
     'python-neutronclient',
+    'python-cinderclient',
 ]
 
 tests_require = [

unit_tests/utilities/test_zaza_utilities_ceph.py

@@ -0,0 +1,103 @@
+import unit_tests.utils as ut_utils
+import zaza.model as model
+import zaza.utilities.ceph as ceph_utils
+import zaza.utilities.openstack as openstack_utils
+
+
+class TestCephUtils(ut_utils.BaseTestCase):
+
+    def setUp(self):
+        super(TestCephUtils, self).setUp()
+
+    def _test_expected_pools(self,
+                             os_release_pair,
+                             expected_pools,
+                             radosgw=False):
+        self.get_current_os_release_pair.return_value = os_release_pair
+        actual_pools = ceph_utils.get_expected_pools(radosgw)
+        self.assertEqual(expected_pools, actual_pools)
+
+    def test_get_expected_pools(self):
+        self.patch_object(openstack_utils, 'get_current_os_release_pair')
+
+        # Trusty Icehouse
+        os_release_pair = 'trusty_icehouse'
+        self.get_current_os_release_pair.return_value = 'trusty_icehouse'
+        expected_pools = [
+            'data',
+            'metadata',
+            'rbd',
+            'cinder-ceph',
+            'glance'
+        ]
+        self._test_expected_pools(os_release_pair, expected_pools)
+
+        # Xenial Ocata
+        os_release_pair = 'xenial_ocata'
+        expected_pools = [
+            'rbd',
+            'cinder-ceph',
+            'glance'
+        ]
+        self._test_expected_pools(os_release_pair, expected_pools)
+
+        # Xenial Queens
+        os_release_pair = 'xenial_queens'
+        expected_pools = [
+            'cinder-ceph',
+            'glance'
+        ]
+        self._test_expected_pools(os_release_pair, expected_pools)
+
+        # Xenial Queens with radosgw
+        os_release_pair = 'xenial_queens'
+        expected_pools = [
+            'cinder-ceph',
+            'glance',
+            '.rgw.root',
+            '.rgw.control',
+            '.rgw',
+            '.rgw.gc',
+            '.users.uid'
+        ]
+        self._test_expected_pools(os_release_pair, expected_pools, True)
+
+    def test_get_ceph_pools(self):
+        self.patch_object(model, 'run_on_unit')
+
+        # Bad return code
+        result = {
+            'Code': '1',
+            'Stdout': '',
+            'Stderr': 'something went wrong',
+        }
+        self.run_on_unit.return_value = result
+        with self.assertRaises(model.CommandRunFailed):
+            ceph_utils.get_ceph_pools('ceph-mon/0')
+
+        # Xenial Queens output
+        result = {
+            'Code': '0',
+            'Stdout': '1 cinder-ceph,2 glance,',
+            'Stderr': ''
+        }
+        self.run_on_unit.return_value = result
+        expected = {
+            'cinder-ceph': 1,
+            'glance': 2
+        }
+        actual = ceph_utils.get_ceph_pools('ceph-mon/0')
+        self.assertEqual(expected, actual)
+        # Bionic Queens output
+        result = {
+            'Code': '0',
+            'Stdout': '1 cinder-ceph\n2 glance',
+            'Stderr': ''
+        }
+        self.run_on_unit.return_value = result
+        expected = {
+            'cinder-ceph': 1,
+            'glance': 2
+        }
+        actual = ceph_utils.get_ceph_pools('ceph-mon/0')
+        self.assertEqual(expected, actual)

unit_tests/utilities/test_zaza_utilities_generic.py

@@ -15,6 +15,7 @@
 import mock
 import unit_tests.utils as ut_utils
 from zaza.utilities import generic as generic_utils
+import zaza.utilities.exceptions as zaza_exceptions
 
 FAKE_STATUS = {
     'can-upgrade-to': '',
@@ -368,3 +369,179 @@ class TestGenericUtils(ut_utils.BaseTestCase):
             '/etc/apt/apt.conf.d/50unattended-upgrades || '
             'echo \'DPkg::options { "--force-confdef"; };\' >> '
             '/etc/apt/apt.conf.d/50unattended-upgrades')
+
+    def test_get_process_id_list(self):
+        self.patch(
+            "zaza.utilities.generic.model.run_on_unit",
+            new_callable=mock.MagicMock(),
+            name="_run"
+        )
+
+        # Return code is OK and STDOUT contains output
+        returns_ok = {
+            "Code": 0,
+            "Stdout": "1 2",
+            "Stderr": ""
+        }
+        self._run.return_value = returns_ok
+        p_id_list = generic_utils.get_process_id_list(
+            "ceph-osd/0",
+            "ceph-osd",
+            False
+        )
+        expected = ["1", "2"]
+        cmd = 'pidof -x "ceph-osd" || exit 0 && exit 1'
+        self.assertEqual(p_id_list, expected)
+        self._run.assert_called_once_with(unit_name="ceph-osd/0",
+                                          command=cmd)
+
+        # Return code is not OK
+        returns_nok = {
+            "Code": 1,
+            "Stdout": "",
+            "Stderr": "Something went wrong"
+        }
+        self._run.return_value = returns_nok
+        with self.assertRaises(zaza_exceptions.ProcessIdsFailed):
+            generic_utils.get_process_id_list("ceph-osd/0", "ceph")
+            cmd = 'pidof -x "ceph"'
+            self._run.assert_called_once_with(unit_name="ceph-osd/0",
+                                              command=cmd)
+
+    def test_get_unit_process_ids(self):
+        self.patch(
+            "zaza.utilities.generic.get_process_id_list",
+            new_callable=mock.MagicMock(),
+            name="_get_pids"
+        )
+
+        pids = ["1", "2"]
+        self._get_pids.return_value = pids
+        unit_processes = {
+            "ceph-osd/0": {
+                "ceph-osd": 2
+            },
+            "unit/0": {
+                "pr1": 2,
+                "pr2": 2
+            }
+        }
+        expected = {
+            "ceph-osd/0": {
+                "ceph-osd": ["1", "2"]
+            },
+            "unit/0": {
+                "pr1": ["1", "2"],
+                "pr2": ["1", "2"]
+            }
+        }
+        result = generic_utils.get_unit_process_ids(unit_processes)
+        self.assertEqual(result, expected)
+
+    def test_validate_unit_process_ids(self):
+        expected = {
+            "ceph-osd/0": {
+                "ceph-osd": 2
+            },
+            "unit/0": {
+                "pr1": 2,
+                "pr2": [1, 2]
+            }
+        }
+
+        # Unit count mismatch
+        actual = {}
+        with self.assertRaises(zaza_exceptions.UnitCountMismatch):
+            generic_utils.validate_unit_process_ids(expected, actual)
+
+        # Unit not found in actual dict
+        actual = {
+            "ceph-osd/0": {
+                "ceph-osd": ["1", "2"]
+            },
+            # unit/0 not in the dict
+            "unit/1": {
+                "pr1": ["1", "2"],
+                "pr2": ["1", "2"]
+            }
+        }
+        with self.assertRaises(zaza_exceptions.UnitNotFound):
+            generic_utils.validate_unit_process_ids(expected, actual)
+
+        # Process names count doesn't match
+        actual = {
+            "ceph-osd/0": {
+                "ceph-osd": ["1", "2"]
+            },
+            "unit/0": {
+                # Only one process name instead of 2 expected
+                "pr1": ["1", "2"]
+            }
+        }
+        with self.assertRaises(zaza_exceptions.ProcessNameCountMismatch):
+            generic_utils.validate_unit_process_ids(expected, actual)
+
+        # Process name doesn't match
+        actual = {
+            "ceph-osd/0": {
+                "ceph-osd": ["1", "2"]
+            },
+            "unit/0": {
+                # Bad process name
+                "bad_name": ["1", "2"],
+                "pr2": ["1", "2"]
+            }
+        }
+        with self.assertRaises(zaza_exceptions.ProcessNameMismatch):
+            generic_utils.validate_unit_process_ids(expected, actual)
+
+        # PID count doesn't match
+        actual = {
+            "ceph-osd/0": {
+                "ceph-osd": ["1", "2"]
+            },
+            "unit/0": {
+                # Only one PID instead of 2 expected
+                "pr1": ["2"],
+                "pr2": ["1", "2"]
+            }
+        }
+        with self.assertRaises(zaza_exceptions.PIDCountMismatch):
+            generic_utils.validate_unit_process_ids(expected, actual)
+
+        actual = {
+            "ceph-osd/0": {
+                "ceph-osd": ["1", "2"]
+            },
+            "unit/0": {
+                "pr1": ["1", "2"],
+                # 3 PID instead of [1, 2] expected
+                "pr2": ["1", "2", "3"]
+            }
+        }
+        with self.assertRaises(zaza_exceptions.PIDCountMismatch):
+            generic_utils.validate_unit_process_ids(expected, actual)
+
+        # It should work now...
+        actual = {
+            "ceph-osd/0": {
+                "ceph-osd": ["1", "2"]
+            },
+            "unit/0": {
+                "pr1": ["1", "2"],
+                "pr2": ["1", "2"]
+            }
+        }
+        ret = generic_utils.validate_unit_process_ids(expected, actual)
+        self.assertTrue(ret)
+
+    def test_get_ubuntu_release(self):
+        # Normal case
+        expected = 0
+        actual = generic_utils.get_ubuntu_release('oneiric')
+        self.assertEqual(expected, actual)
+
+        # Ubuntu release doesn't exist
+        bad_name = 'bad_name'
+        with self.assertRaises(zaza_exceptions.UbuntuReleaseNotFound):
+            generic_utils.get_ubuntu_release(bad_name)

zaza/charm_tests/ceph/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of code for setting up and testing ceph-osd."""

zaza/charm_tests/ceph/setup.py

@@ -0,0 +1,20 @@
+# Copyright 2018 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Setup for ceph-osd deployments."""
+
+
+def basic_setup():
+    """Run basic setup for ceph-osd."""
+    pass

zaza/charm_tests/ceph/tests.py

@@ -0,0 +1,505 @@
+# Copyright 2018 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Ceph-osd Testing."""
+
+import logging
+from os import (
+    listdir,
+    path
+)
+import tempfile
+
+import zaza.charm_tests.test_utils as test_utils
+import zaza.model as zaza_model
+import zaza.utilities.ceph as zaza_ceph
+import zaza.utilities.exceptions as zaza_exceptions
+import zaza.utilities.generic as zaza_utils
+import zaza.utilities.juju as zaza_juju
+import zaza.utilities.openstack as zaza_openstack
+
+
+class CephLowLevelTest(test_utils.OpenStackBaseTest):
+    """Ceph Low Level Test Class."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running ceph low level tests."""
+        super(CephLowLevelTest, cls).setUpClass()
+
+    def test_processes(self):
+        """Verify Ceph processes.
+
+        Verify that the expected service processes are running
+        on each ceph unit.
+        """
+        logging.info('Checking ceph-mon and ceph-osd processes...')
+        # Process name and quantity of processes to expect on each unit
+        ceph_mon_processes = {
+            'ceph-mon': 1,
+        }
+
+        ceph_osd_processes = {
+            'ceph-osd': [2, 3]
+        }
+
+        # Units with process names and PID quantities expected
+        expected_processes = {
+            'ceph-mon/0': ceph_mon_processes,
+            'ceph-mon/1': ceph_mon_processes,
+            'ceph-mon/2': ceph_mon_processes,
+            'ceph-osd/0': ceph_osd_processes,
+            'ceph-osd/1': ceph_osd_processes,
+            'ceph-osd/2': ceph_osd_processes
+        }
+
+        actual_pids = zaza_utils.get_unit_process_ids(expected_processes)
+        ret = zaza_utils.validate_unit_process_ids(expected_processes,
+                                                   actual_pids)
+        self.assertTrue(ret)
+
+    def test_services(self):
+        """Verify the ceph services.
+
+        Verify the expected services are running on the service units.
+        """
+        logging.info('Checking ceph-osd and ceph-mon services...')
+        services = {}
+        ceph_services = ['ceph-mon']
+        services['ceph-osd/0'] = ['ceph-osd']
+
+        services['ceph-mon/0'] = ceph_services
+        services['ceph-mon/1'] = ceph_services
+        services['ceph-mon/2'] = ceph_services
+
+        for unit_name, unit_services in services.items():
+            zaza_model.block_until_service_status(
+                unit_name=unit_name,
+                services=unit_services,
+                target_status='running'
+            )
+
+
+class CephRelationTest(test_utils.OpenStackBaseTest):
+    """Ceph's relations test class."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run the ceph's relations class setup."""
+        super(CephRelationTest, cls).setUpClass()
+
+    def test_ceph_osd_ceph_relation_address(self):
+        """Verify the ceph-osd to ceph relation data."""
+        logging.info('Checking ceph-osd:ceph-mon relation data...')
+        unit_name = 'ceph-osd/0'
+        remote_unit_name = 'ceph-mon/0'
+        relation_name = 'osd'
+        remote_unit = zaza_model.get_unit_from_name(remote_unit_name)
+        remote_ip = remote_unit.public_address
+        relation = zaza_juju.get_relation_from_unit(
+            unit_name,
+            remote_unit_name,
+            relation_name
+        )
+        # Get private-address in relation
+        rel_private_ip = relation.get('private-address')
+        # The private address in relation should match ceph-mon/0 address
+        self.assertEqual(rel_private_ip, remote_ip)
+
+    def _ceph_to_ceph_osd_relation(self, remote_unit_name):
+        """Verify the cephX to ceph-osd relation data.
+
+        Helper function to test the relation.
+        """
+        logging.info('Checking {}:ceph-osd mon relation data...'.
+                     format(remote_unit_name))
+        unit_name = 'ceph-osd/0'
+        relation_name = 'osd'
+        remote_unit = zaza_model.get_unit_from_name(remote_unit_name)
+        remote_ip = remote_unit.public_address
+        cmd = 'leader-get fsid'
+        result = zaza_model.run_on_unit(remote_unit_name, cmd)
+        fsid = result.get('Stdout').strip()
+        expected = {
+            'private-address': remote_ip,
+            'auth': 'none',
+            'ceph-public-address': remote_ip,
+            'fsid': fsid,
+        }
+        relation = zaza_juju.get_relation_from_unit(
+            unit_name,
+            remote_unit_name,
+            relation_name
+        )
+        for e_key, e_value in expected.items():
+            a_value = relation[e_key]
+            self.assertEqual(e_value, a_value)
+        self.assertTrue(relation['osd_bootstrap_key'] is not None)
+
+    def test_ceph0_to_ceph_osd_relation(self):
+        """Verify the ceph0 to ceph-osd relation data."""
+        remote_unit_name = 'ceph-mon/0'
+        self._ceph_to_ceph_osd_relation(remote_unit_name)
+
+    def test_ceph1_to_ceph_osd_relation(self):
+        """Verify the ceph1 to ceph-osd relation data."""
+        remote_unit_name = 'ceph-mon/1'
+        self._ceph_to_ceph_osd_relation(remote_unit_name)
+
+    def test_ceph2_to_ceph_osd_relation(self):
+        """Verify the ceph2 to ceph-osd relation data."""
+        remote_unit_name = 'ceph-mon/2'
+        self._ceph_to_ceph_osd_relation(remote_unit_name)
+
+
+class CephTest(test_utils.OpenStackBaseTest):
+    """Ceph common functional tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run the ceph's common class setup."""
+        super(CephTest, cls).setUpClass()
+
+    def pause_resume(self, services):
+        """Run Pause and resume tests.
+
+        Override the default implementation since pausing ceph units
+        doesn't stop the services.
+        Pause and then resume a unit checking that services are in the
+        required state after each action
+
+        :param services: Services expected to be restarted when config_file is
+                         changed.
+        :type services: list
+        """
+        zaza_model.block_until_service_status(
+            self.lead_unit,
+            services,
+            'running',
+            model_name=self.model_name)
+        zaza_model.block_until_unit_wl_status(
+            self.lead_unit,
+            'active',
+            model_name=self.model_name)
+        zaza_model.run_action(
+            self.lead_unit,
+            'pause',
+            model_name=self.model_name)
+        zaza_model.block_until_unit_wl_status(
+            self.lead_unit,
+            'maintenance',
+            model_name=self.model_name)
+        zaza_model.block_until_all_units_idle(model_name=self.model_name)
+        zaza_model.run_action(
+            self.lead_unit,
+            'resume',
+            model_name=self.model_name)
+        zaza_model.block_until_unit_wl_status(
+            self.lead_unit,
+            'active',
+            model_name=self.model_name)
+        zaza_model.block_until_all_units_idle(model_name=self.model_name)
+        zaza_model.block_until_service_status(
+            self.lead_unit,
+            services,
+            'running',
+            model_name=self.model_name)
+
+    def test_ceph_check_osd_pools(self):
+        """Check OSD pools.
+
+        Check osd pools on all ceph units, expect them to be
+        identical, and expect specific pools to be present.
+        """
+        logging.info('Checking pools on ceph units...')
+
+        expected_pools = zaza_ceph.get_expected_pools()
+        results = []
+        unit_name = 'ceph-mon/0'
+
+        # Check for presence of expected pools on each unit
+        logging.debug('Expected pools: {}'.format(expected_pools))
+        pools = zaza_ceph.get_ceph_pools(unit_name)
+        results.append(pools)
+
+        for expected_pool in expected_pools:
+            if expected_pool not in pools:
+                msg = ('{} does not have pool: '
+                       '{}'.format(unit_name, expected_pool))
+                raise zaza_exceptions.CephPoolNotFound(msg)
+        logging.debug('{} has (at least) the expected '
+                      'pools.'.format(unit_name))
+
+        # Check that all units returned the same pool name:id data
+        for i, result in enumerate(results):
+            for other in results[i+1:]:
+                logging.debug('result: {}, other: {}'.format(result, other))
+                self.assertEqual(result, other)
+
+    def test_ceph_pool_creation_with_text_file(self):
+        """Check the creation of a pool and a text file.
+
+        Create a pool, add a text file to it and retrieve its content.
+        Verify that the content matches the original file.
+        """
+        unit_name = 'ceph-mon/0'
+        cmd = 'sudo ceph osd pool create test 128; \
+               echo 123456789 > /tmp/input.txt; \
+               rados put -p test test_input /tmp/input.txt; \
+               rados get -p test test_input /dev/stdout'
+        logging.debug('Creating test pool and putting test file in pool...')
+        result = zaza_model.run_on_unit(unit_name, cmd)
+        code = result.get('Code')
+        if code != '0':
+            raise zaza_model.CommandRunFailed(cmd, result)
+        output = result.get('Stdout').strip()
+        logging.debug('Output received: {}'.format(output))
+        self.assertEqual(output, '123456789')
+
+    def test_ceph_encryption(self):
+        """Test Ceph encryption.
+
+        Verify that the new disk is added with encryption by checking for
+        Ceph's encryption keys directory.
+        """
+        current_release = zaza_openstack.get_os_release()
+        trusty_mitaka = zaza_openstack.get_os_release('trusty_mitaka')
+        if current_release >= trusty_mitaka:
+            logging.warn("Skipping encryption test for Mitaka and higher")
+            return
+        unit_name = 'ceph-osd/0'
+        set_default = {
+            'osd-encrypt': 'False',
+            'osd-devices': '/dev/vdb /srv/ceph',
+        }
+        set_alternate = {
+            'osd-encrypt': 'True',
+            'osd-devices': '/dev/vdb /srv/ceph /srv/ceph_encrypted',
+        }
+        juju_service = 'ceph-osd'
+        logging.info('Making config change on {}...'.format(juju_service))
+        mtime = zaza_model.get_unit_time(unit_name)
+
+        file_mtime = None
+
+        folder_name = '/etc/ceph/dmcrypt-keys/'
+        with self.config_change(set_default, set_alternate):
+            with tempfile.TemporaryDirectory() as tempdir:
+                # Creating a temp dir to copy keys
+                temp_folder = '/tmp/dmcrypt-keys'
+                cmd = 'mkdir {}'.format(temp_folder)
+                ret = zaza_model.run_on_unit(unit_name, cmd)
+                logging.debug('Ret for cmd {} is {}'.format(cmd, ret))
+                # Copy keys from /etc to /tmp
+                cmd = 'sudo cp {}* {}'.format(folder_name, temp_folder)
+                ret = zaza_model.run_on_unit(unit_name, cmd)
+                logging.debug('Ret for cmd {} is {}'.format(cmd, ret))
+                # Changing permissions to be able to SCP the files
+                cmd = 'sudo chown -R ubuntu:ubuntu {}'.format(temp_folder)
+                ret = zaza_model.run_on_unit(unit_name, cmd)
+                logging.debug('Ret for cmd {} is {}'.format(cmd, ret))
+                # SCP to retrieve all files in folder
+                # -p: preserve timestamps
+                source = '/tmp/dmcrypt-keys/*'
+                zaza_model.scp_from_unit(unit_name=unit_name,
+                                         source=source,
+                                         destination=tempdir,
+                                         scp_opts='-p')
+                for elt in listdir(tempdir):
+                    file_path = '/'.join([tempdir, elt])
+                    if path.isfile(file_path):
+                        file_mtime = path.getmtime(file_path)
+                        if file_mtime:
+                            break
+
+        if not file_mtime:
+            logging.warn('Could not determine mtime, assuming '
+                         'folder does not exist')
+            raise FileNotFoundError('folder does not exist')
+
+        if file_mtime >= mtime:
+            logging.info('Folder mtime is newer than provided mtime '
+                         '(%s >= %s) on %s (OK)' % (file_mtime,
+                                                    mtime, unit_name))
+        else:
+            logging.warn('Folder mtime is older than provided mtime'
+                         '(%s < on %s) on %s' % (file_mtime,
+                                                 mtime, unit_name))
+            raise Exception('Folder mtime is older than provided mtime')
+
+    def test_blocked_when_non_pristine_disk_appears(self):
+        """Test blocked state with non-pristine disk.
+
+        Validate that charm goes into blocked state when it is presented with
+        new block devices that have foreign data on them.
+        Instances used in UOSCI has a flavour with ephemeral storage in
+        addition to the bootable instance storage.  The ephemeral storage
+        device is partitioned, formatted and mounted early in the boot process
+        by cloud-init.
+        As long as the device is mounted the charm will not attempt to use it.
+        If we unmount it and trigger the config-changed hook the block device
+        will appear as a new and previously untouched device for the charm.
+        One of the first steps of device eligibility checks should be to make
+        sure we are seeing a pristine and empty device before doing any
+        further processing.
+        As the ephemeral device will have data on it we can use it to validate
+        that these checks work as intended.
+        """
+        logging.info('Checking behaviour when non-pristine disks appear...')
+        logging.info('Configuring ephemeral-unmount...')
+        alternate_conf = {
+            'ephemeral-unmount': '/mnt',
+            'osd-devices': '/dev/vdb'
+        }
+        juju_service = 'ceph-osd'
+        zaza_model.set_application_config(juju_service, alternate_conf)
+        ceph_osd_states = {
+            'ceph-osd': {
+                'workload-status': 'blocked',
+                'workload-status-message': 'Non-pristine'
+            }
+        }
+        zaza_model.wait_for_application_states(states=ceph_osd_states)
+        logging.info('Units now in blocked state, running zap-disk action...')
+        unit_names = ['ceph-osd/0', 'ceph-osd/1', 'ceph-osd/2']
+        for unit_name in unit_names:
+            zap_disk_params = {
+                'devices': '/dev/vdb',
+                'i-really-mean-it': True,
+            }
+            action_obj = zaza_model.run_action(
+                unit_name=unit_name,
+                action_name='zap-disk',
+                action_params=zap_disk_params
+            )
+            logging.debug('Result of action: {}'.format(action_obj))
+
+        logging.info('Running add-disk action...')
+        for unit_name in unit_names:
+            add_disk_params = {
+                'osd-devices': '/dev/vdb',
+            }
+            action_obj = zaza_model.run_action(
+                unit_name=unit_name,
+                action_name='add-disk',
+                action_params=add_disk_params
+            )
+            logging.debug('Result of action: {}'.format(action_obj))
+
+        logging.info('Wait for idle/ready status...')
+        zaza_model.wait_for_application_states()
+
+        logging.info('OK')
+
+        set_default = {
+            'ephemeral-unmount': '',
+            'osd-devices': '/dev/vdb /srv/ceph',
+        }
+
+        logging.info('Restoring to default configuration...')
+        zaza_model.set_application_config(juju_service, set_default)
+
+        zaza_model.wait_for_application_states()
+
+    def test_pause_and_resume(self):
+        """The services can be paused and resumed."""
+        logging.info('Checking pause and resume actions...')
+        self.pause_resume(['ceph-osd'])
+
+    def test_blacklist(self):
+        """Check the blacklist action.
+
+        The blacklist actions execute and behave as expected.
+        """
+        logging.info('Checking blacklist-add-disk and'
+                     'blacklist-remove-disk actions...')
+        unit_name = 'ceph-osd/0'
+
+        zaza_model.block_until_unit_wl_status(
+            unit_name,
+            'active'
+        )
+
+        # Attempt to add device with non-absolute path should fail
+        action_obj = zaza_model.run_action(
+            unit_name=unit_name,
+            action_name='blacklist-add-disk',
+            action_params={'osd-devices': 'vda'}
+        )
+        self.assertTrue(action_obj.status != 'completed')
+        zaza_model.block_until_unit_wl_status(
+            unit_name,
+            'active'
+        )
+
+        # Attempt to add device with non-existent path should fail
+        action_obj = zaza_model.run_action(
+            unit_name=unit_name,
+            action_name='blacklist-add-disk',
+            action_params={'osd-devices': '/non-existent'}
+        )
+        self.assertTrue(action_obj.status != 'completed')
+        zaza_model.block_until_unit_wl_status(
+            unit_name,
+            'active'
+        )
+
+        # Attempt to add device with existent path should succeed
+        action_obj = zaza_model.run_action(
+            unit_name=unit_name,
+            action_name='blacklist-add-disk',
+            action_params={'osd-devices': '/dev/vda'}
+        )
+        self.assertEqual('completed', action_obj.status)
+        zaza_model.block_until_unit_wl_status(
+            unit_name,
+            'active'
+        )
+
+        # Attempt to remove listed device should always succeed
+        action_obj = zaza_model.run_action(
+            unit_name=unit_name,
+            action_name='blacklist-remove-disk',
+            action_params={'osd-devices': '/dev/vda'}
+        )
+        self.assertEqual('completed', action_obj.status)
+        zaza_model.block_until_unit_wl_status(
+            unit_name,
+            'active'
+        )
+        logging.debug('OK')
+
+    def test_list_disks(self):
+        """Test the list-disks action.
+
+        The list-disks action execute.
+        """
+        logging.info('Checking list-disks action...')
+        unit_name = 'ceph-osd/0'
+
+        zaza_model.block_until_unit_wl_status(
+            unit_name,
+            'active'
+        )
+
+        action_obj = zaza_model.run_action(
+            unit_name=unit_name,
+            action_name='list-disks',
+        )
+        self.assertEqual('completed', action_obj.status)
+        zaza_model.block_until_unit_wl_status(
+            unit_name,
+            'active'
+        )
+        logging.debug('OK')

zaza/utilities/ceph.py

@@ -0,0 +1,94 @@
+"""Module containing Ceph related utilities."""
+
+import logging
+
+import zaza.utilities.openstack as openstack_utils
+import zaza.model as zaza_model
+
+
+def get_expected_pools(radosgw=False):
+    """Get expected ceph pools.
+
+    Return a list of expected ceph pools in a ceph + cinder + glance
+    test scenario, based on OpenStack release and whether ceph radosgw
+    is flagged as present or not.
+    :param radosgw: If radosgw is used or not
+    :type radosgw: boolean
+    :returns: List of pools that are expected
+    :rtype: list
+    """
+    current_release = openstack_utils.get_os_release()
+    trusty_icehouse = openstack_utils.get_os_release('trusty_icehouse')
+    trusty_kilo = openstack_utils.get_os_release('trusty_kilo')
+    zesty_ocata = openstack_utils.get_os_release('zesty_ocata')
+    if current_release == trusty_icehouse:
+        # Icehouse
+        pools = [
+            'data',
+            'metadata',
+            'rbd',
+            'cinder-ceph',
+            'glance'
+        ]
+    elif (trusty_kilo <= current_release <= zesty_ocata):
+        # Kilo through Ocata
+        pools = [
+            'rbd',
+            'cinder-ceph',
+            'glance'
+        ]
+    else:
+        # Pike and later
+        pools = [
+            'cinder-ceph',
+            'glance'
+        ]
+
+    if radosgw:
+        pools.extend([
+            '.rgw.root',
+            '.rgw.control',
+            '.rgw',
+            '.rgw.gc',
+            '.users.uid'
+        ])
+
+    return pools
+
+
+def get_ceph_pools(unit_name):
+    """Get ceph pools.
+
+    Return a dict of ceph pools from a single ceph unit, with
+    pool name as keys, pool id as vals.
+    :param unit_name: Name of the unit to get the pools on
+    :type unit_name: string
+    :returns: Dict of ceph pools
+    :rtype: dict
+    :raise: zaza_model.CommandRunFailed
+    """
+    pools = {}
+    cmd = 'sudo ceph osd lspools'
+    result = zaza_model.run_on_unit(unit_name, cmd)
+    output = result.get('Stdout').strip()
+    code = int(result.get('Code'))
+    if code != 0:
+        raise zaza_model.CommandRunFailed(cmd, result)
+
+    # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
+    # It can also be something link 0 data\n1 metadata
+
+    # First split on new lines
+    osd_pools = str(output).split('\n')
+    # If we have a len of 1, no new lines found -> splitting on commas
+    if len(osd_pools) == 1:
+        osd_pools = osd_pools[0].split(',')
+    for pool in osd_pools:
+        pool_id_name = pool.split(' ')
+        if len(pool_id_name) == 2:
+            pool_id = pool_id_name[0]
+            pool_name = pool_id_name[1]
+            pools[pool_name] = int(pool_id)
+
+    logging.debug('Pools on {}: {}'.format(unit_name, pools))
+    return pools

zaza/utilities/exceptions.py

@@ -106,3 +106,57 @@ class KeystoneKeyRepositoryError(Exception):
     """
 
     pass
+
+
+class ProcessNameCountMismatch(Exception):
+    """Count of process names doesn't match."""
+
+    pass
+
+
+class ProcessNameMismatch(Exception):
+    """Name of processes doesn't match."""
+
+    pass
+
+
+class PIDCountMismatch(Exception):
+    """PID's count doesn't match."""
+
+    pass
+
+
+class ProcessIdsFailed(Exception):
+    """Process ID lookup failed."""
+
+    pass
+
+
+class UnitNotFound(Exception):
+    """Unit not found in actual dict."""
+
+    pass
+
+
+class UnitCountMismatch(Exception):
+    """Count of unit doesn't match."""
+
+    pass
+
+
+class UbuntuReleaseNotFound(Exception):
+    """Ubuntu release not found in list."""
+
+    pass
+
+
+class ServiceNotFound(Exception):
+    """Service not found on unit."""
+
+    pass
+
+
+class CephPoolNotFound(Exception):
+    """Ceph pool not found."""
+
+    pass

zaza/utilities/generic.py

@@ -21,6 +21,8 @@ import yaml
 
 from zaza import model
 from zaza.utilities import juju as juju_utils
+from zaza.utilities import exceptions as zaza_exceptions
+from zaza.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE
 
 
 def dict_to_yaml(dict_data):
@@ -458,3 +460,152 @@ def set_dpkg_non_interactive_on_unit(
     cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}"
            .format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d))
     model.run_on_unit(unit_name, cmd)
+
+
+def get_process_id_list(unit_name, process_name,
+                        expect_success=True):
+    """Get a list of process ID(s).
+
+    Get a list of process ID(s) from a single sentry juju unit
+    for a single process name.
+
+    :param unit_name: Amulet sentry instance (juju unit)
+    :param process_name: Process name
+    :param expect_success: If False, expect the PID to be missing,
+        raise if it is present.
+    :returns: List of process IDs
+    :raises: zaza_exceptions.ProcessIdsFailed
+    """
+    cmd = 'pidof -x "{}"'.format(process_name)
+    if not expect_success:
+        cmd += " || exit 0 && exit 1"
+    results = model.run_on_unit(unit_name=unit_name, command=cmd)
+    code = results.get("Code", 1)
+    try:
+        code = int(code)
+    except ValueError:
+        code = 1
+    error = results.get("Stderr")
+    output = results.get("Stdout")
+    if code != 0:
+        msg = ('{} `{}` returned {} '
+               '{} with error {}'.format(unit_name, cmd, code, output, error))
+        raise zaza_exceptions.ProcessIdsFailed(msg)
+    return str(output).split()
+
+
+def get_unit_process_ids(unit_processes, expect_success=True):
+    """Get unit process ID(s).
+
+    Construct a dict containing unit sentries, process names, and
+    process IDs.
+
+    :param unit_processes: A dictionary of unit names
+        to list of process names.
+    :param expect_success: if False expect the processes to not be
+        running, raise if they are.
+    :returns: Dictionary of unit names to dictionary
+        of process names to PIDs.
+    :raises: zaza_exceptions.ProcessIdsFailed
+    """
+    pid_dict = {}
+    for unit_name, process_list in unit_processes.items():
+        pid_dict[unit_name] = {}
+        for process in process_list:
+            pids = get_process_id_list(
+                unit_name, process, expect_success=expect_success)
+            pid_dict[unit_name].update({process: pids})
+    return pid_dict
+
+
+def validate_unit_process_ids(expected, actual):
+    """Validate process id quantities for services on units.
+
+    :returns: True if the PIDs are validated, raises an exception
+        if it is not the case.
+    :raises: zaza_exceptions.UnitCountMismatch
+    :raises: zaza_exceptions.UnitNotFound
+    :raises: zaza_exceptions.ProcessNameCountMismatch
+    :raises: zaza_exceptions.ProcessNameMismatch
+    :raises: zaza_exceptions.PIDCountMismatch
+    """
+    logging.debug('Checking units for running processes...')
+    logging.debug('Expected PIDs: {}'.format(expected))
+    logging.debug('Actual PIDs: {}'.format(actual))
+
+    if len(actual) != len(expected):
+        msg = ('Unit count mismatch.  expected, actual: {}, '
+               '{} '.format(len(expected), len(actual)))
+        raise zaza_exceptions.UnitCountMismatch(msg)
+
+    for (e_unit_name, e_proc_names) in expected.items():
+        if e_unit_name in actual.keys():
+            a_proc_names = actual[e_unit_name]
+        else:
+            msg = ('Expected unit ({}) not found in actual dict data.'.
+                   format(e_unit_name))
+            raise zaza_exceptions.UnitNotFound(msg)
+
+        if len(e_proc_names.keys()) != len(a_proc_names.keys()):
+            msg = ('Process name count mismatch.  expected, actual: {}, '
+                   '{}'.format(len(expected), len(actual)))
+            raise zaza_exceptions.ProcessNameCountMismatch(msg)
+
+        for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
+                zip(e_proc_names.items(), a_proc_names.items()):
+            if e_proc_name != a_proc_name:
+                msg = ('Process name mismatch.  expected, actual: {}, '
+                       '{}'.format(e_proc_name, a_proc_name))
+                raise zaza_exceptions.ProcessNameMismatch(msg)
+
+            a_pids_length = len(a_pids)
+            fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
+                        '{}, {} ({})'.format(e_unit_name, e_proc_name,
+                                             e_pids, a_pids_length,
+                                             a_pids))
+
+            # If expected is a list, ensure at least one PID quantity match
+            if isinstance(e_pids, list) and \
+                    a_pids_length not in e_pids:
+                raise zaza_exceptions.PIDCountMismatch(fail_msg)
+            # If expected is not bool and not list,
+            # ensure PID quantities match
+            elif not isinstance(e_pids, bool) and \
+                    not isinstance(e_pids, list) and \
+                    a_pids_length != e_pids:
+                raise zaza_exceptions.PIDCountMismatch(fail_msg)
+            # If expected is bool True, ensure 1 or more PIDs exist
+            elif isinstance(e_pids, bool) and \
+                    e_pids is True and a_pids_length < 1:
+                raise zaza_exceptions.PIDCountMismatch(fail_msg)
+            # If expected is bool False, ensure 0 PIDs exist
+            elif isinstance(e_pids, bool) and \
+                    e_pids is False and a_pids_length != 0:
+                raise zaza_exceptions.PIDCountMismatch(fail_msg)
+            else:
+                logging.debug('PID check OK: {} {} {}: '
+                              '{}'.format(e_unit_name, e_proc_name,
+                                          e_pids, a_pids))
+    return True
+
+
+def get_ubuntu_release(ubuntu_name):
+    """Get index of Ubuntu release.
+
+    Returns the index of the name of the Ubuntu release in
+        UBUNTU_OPENSTACK_RELEASE.
+
+    :param ubuntu_name: Name of the Ubuntu release.
+    :type ubuntu_name: string
+    :returns: Index of the Ubuntu release
+    :rtype: integer
+    :raises: zaza_exceptions.UbuntuReleaseNotFound
+    """
+    ubuntu_releases = list(UBUNTU_OPENSTACK_RELEASE.keys())
+    try:
+        index = ubuntu_releases.index(ubuntu_name)
+    except ValueError:
+        msg = ('Could not find Ubuntu release {} in {}'.
+               format(ubuntu_name, UBUNTU_OPENSTACK_RELEASE))
+        raise zaza_exceptions.UbuntuReleaseNotFound(msg)
+    return index

zaza/utilities/openstack.py

@@ -23,6 +23,7 @@ from .os_versions import (
     OPENSTACK_RELEASES_PAIRS,
 )
 
+from cinderclient import client as cinderclient
 from glanceclient import Client as GlanceClient
 
 from keystoneclient.v2_0 import client as keystoneclient_v2
@@ -199,6 +200,17 @@ def get_neutron_session_client(session):
     return neutronclient.Client(session=session)
 
 
+def get_cinder_session_client(session):
+    """Return cinderclient authenticated by keystone session.
+
+    :param session: Keystone session object
+    :type session: keystoneauth1.session.Session object
+    :returns: Authenticated cinderclient
+    :rtype: cinderclient.Client object
+    """
+    return cinderclient.Client(session=session)
+
+
 def get_keystone_scope():
     """Return Keystone scope based on OpenStack release of the overcloud.