Make Ceph tests idempotent (#795)

With the upcoming transition to an operator-framework charm, the
Ceph tests need to be idempotent, since we'll be testing the upgrade
path from classical to operator. The tests that need fixing are the
ones that call out to add/remove disks, since they create loopback
devices to back the OSD's but never remove them.

zaza/openstack/charm_tests/ceph/tests.py

@@ -189,9 +189,15 @@ class CephTest(test_utils.OpenStackBaseTest):
         super(CephTest, cls).setUpClass()
         cls.loop_devs = {}   # Maps osd -> loop device
         for osd in (x.entity_id for x in zaza_model.get_units('ceph-osd')):
-            loop_dev = zaza_utils.add_loop_device(osd, 10).get('Stdout')
+            loop_dev = zaza_utils.add_loop_device(osd, size=10).get('Stdout')
             cls.loop_devs[osd] = loop_dev
 
+    @classmethod
+    def tearDownClass(cls):
+        """Run the ceph's common class teardown."""
+        for osd, loop_dev in cls.loop_devs.items():
+            zaza_utils.remove_loop_device(osd, loop_dev)
+
     def osd_out_in(self, services):
         """Run OSD out and OSD in tests.
 
@@ -604,6 +610,15 @@ class CephTest(test_utils.OpenStackBaseTest):
         zaza_model.wait_for_application_states()
         self.assertEqual(len(osds) * 2, self.get_num_osds(mon))
 
+        # Finally, remove all the added OSDs that are backed by loop devices.
+        for param in params:
+            osd_id = self.get_local_osd_id(param['unit'])
+            zaza_model.run_action(
+                unit_name=param['unit'],
+                action_name='remove-disk',
+                action_params={'osd-ids': osd_id, 'purge': True}
+            )
+
     def test_ceph_auth(self):
         """Test creating and deleting user."""
         logging.info('Creating user and exported keyring...')

zaza/openstack/utilities/generic.py

@@ -725,20 +725,39 @@ def get_leaders_and_non_leaders(application_name):
     return leader, non_leaders
 
 
-def add_loop_device(unit, size=10):
+def add_loop_device(unit, name='loop.img', size=10):
     """Add a loopback device to a Juju unit.
 
     :param unit: The unit name on which to create the device.
     :type unit: str
 
+    :param name: The name of the file used for the loop device.
+    :type unit: str
+
     :param size: The size in GB of the device.
     :type size: int
 
     :returns: The device name.
     """
-    loop_name = '/home/ubuntu/loop.img'
+    loop_name = '/home/ubuntu/{}'.format(name)
     truncate = 'truncate --size {}GB {}'.format(size, loop_name)
     losetup = 'losetup --find {}'.format(loop_name)
     lofind = 'losetup -a | grep {} | cut -f1 -d ":"'.format(loop_name)
     cmd = "sudo sh -c '{} && {} && {}'".format(truncate, losetup, lofind)
     return model.run_on_unit(unit, cmd)
+
+
+def remove_loop_device(unit, device, name='loop.img'):
+    """Remove a loopback device from a Juju unit.
+
+    :param unit: The unit name from which to remove the device.
+    :type unit: str
+
+    :param device: The loop device path to be removed.
+    :type unit: str
+
+    :param name: The name of the file used for the loop device.
+    :type name: str
+    """
+    cmd = "sudo sh -c 'losetup -d {} && rm {}'".format(device, name)
+    return model.run_on_unit(unit, cmd)