Masakari testing fixes

* Add method for deleteing pacemaker nodes.
* Due to LP #1874719 use above to delete node1
* During test cleanup the tests re-enable hosts from a masakari pov,
  if that host still has outstanding notifications it will fail so
  retry the enable.
* pacemakerd process name has changed on focal so account for that
  when killing it.

zaza/openstack/charm_tests/masakari/tests.py

@@ -135,6 +135,10 @@ class MasakariTest(test_utils.OpenStackBaseTest):
 
     def test_instance_failover(self):
         """Test masakari managed guest migration."""
+        # Workaround for Bug #1874719
+        zaza.openstack.configure.hacluster.remove_node(
+            'masakari',
+            'node1')
         # Launch guest
         self.assertTrue(
             zaza.openstack.configure.hacluster.check_all_nodes_online(

zaza/openstack/configure/hacluster.py

@@ -66,6 +66,23 @@ def get_nodes_status(service_name, model_name=None):
     return status
 
 
+def remove_node(service_name, node_name, model_name=None):
+    """Remove given node from pacemaker.
+
+    :param service_name: Name of Juju application to run query against.
+    :type service_name: str
+    :param node_name: Name of node to delete.
+    :type node_name: str
+    :param model_name: Name of model unit_name resides in.
+    :type model_name: str
+    """
+    remove_cmd = 'crm_node --force -R {}'.format(node_name)
+    zaza.model.run_on_leader(
+        service_name,
+        remove_cmd,
+        model_name=model_name)
+
+
 def check_all_nodes_online(service_name, model_name=None):
     """Return whether all the crm nodes are online.
 

zaza/openstack/configure/masakari.py

@@ -19,6 +19,8 @@ and recovery.
 """
 
 import logging
+import openstack.exceptions as ostack_except
+import tenacity
 
 import zaza.model
 import zaza.openstack.utilities.openstack as openstack_utils
@@ -81,6 +83,28 @@ def create_segments(segment_number=1, host_assignment_method=None):
         masakari_client)
 
 
+@tenacity.retry(
+    wait=tenacity.wait_exponential(multiplier=2, max=60),
+    reraise=True, stop=tenacity.stop_after_attempt(5),
+    retry=tenacity.retry_if_exception_type(ostack_except.ConflictException))
+def enable_host(masakari_client, host, segment):
+    """Enable hypervisor within masakari.
+
+    :param masakari_client: Authenticated masakari client
+    :type masakari_client: openstack.instance_ha.v1._proxy.Proxy
+    :param host: Uuid of host to enable
+    :type host: str
+    :param segment: Uuid of segment host is associated with.
+    :type segment: str
+    """
+    logging.info("Removing maintenance mode from masakari "
+                 "host {}".format(host))
+    masakari_client.update_host(
+        host,
+        segment_id=segment,
+        **{'on_maintenance': False})
+
+
 def enable_hosts(masakari_client=None):
     """Enable all hypervisors within masakari.
 
@@ -98,12 +122,7 @@ def enable_hosts(masakari_client=None):
     for segment in masakari_client.segments():
         for host in masakari_client.hosts(segment_id=segment.uuid):
             if host.on_maintenance:
-                logging.info("Removing maintenance mode from masakari "
-                             "host {}".format(host.uuid))
-                masakari_client.update_host(
-                    host.uuid,
-                    segment_id=segment.uuid,
-                    **{'on_maintenance': False})
+                enable_host(masakari_client, host.uuid, segment.uuid)
 
 
 def _svc_control(unit_name, action, services, model_name):
@@ -179,10 +198,18 @@ def simulate_compute_host_failure(unit_name, model_name):
         'stop',
         ['corosync', 'nova-compute'],
         model_name)
-    logging.info('Sending pacemaker_remoted a SIGTERM')
+    compute_app = unit_name.split('/')[0]
+    release_pair = openstack_utils.get_current_os_release_pair(
+        application=compute_app)
+    if (openstack_utils.get_os_release(release_pair=release_pair) >=
+            openstack_utils.get_os_release('focal_ussuri')):
+        pacemaker_proc = '/usr/sbin/pacemaker-remoted'
+    else:
+        pacemaker_proc = '/usr/sbin/pacemaker_remoted'
+    logging.info('Sending {} a SIGTERM'.format(pacemaker_proc))
     zaza.model.run_on_unit(
         unit_name,
-        'pkill -9 -f /usr/sbin/pacemaker_remoted',
+        'pkill -9 -f {}'.format(pacemaker_proc),
         model_name=model_name)