Add method to check OpenStack endpoints (#953)

* Add method to check OpenStack endpoints

Add method to check OpenStack endpoints are returning acceptable
http codes. This should be used with caution as a charm whould
indicate if its payload is not ready via workload status and
workload status messages

* Fix dox string

zaza/openstack/charm_tests/keystone/setup.py

@@ -15,6 +15,8 @@
 """Code for setting up keystone."""
 
 import logging
+import requests
+import tenacity
 
 import keystoneauth1
 
@@ -167,3 +169,43 @@ def add_tempest_roles():
     :rtype: None
     """
     _add_additional_roles(TEMPEST_ROLES)
+
+
+def wait_for_url(url, ok_codes=None):
+    """Wait for url to return acceptable return code.
+
+    :param url: url to test
+    :type url: str
+    :param ok_codes: HTTP codes that are acceptable
+    :type ok_codes: Optional[List[int]]
+    :raises: AssertionError
+    """
+    if not ok_codes:
+        ok_codes = [requests.codes.ok]
+    for attempt in tenacity.Retrying(
+            stop=tenacity.stop_after_attempt(10),
+            wait=tenacity.wait_exponential(
+                multiplier=1, min=2, max=60)):
+        with attempt:
+            r = requests.get(url)
+            logging.info("{} returned {}".format(url, r.status_code))
+            assert r.status_code in ok_codes
+
+
+def wait_for_all_endpoints(interface='public'):
+    """Check all endpoints are returning an acceptable return code.
+
+    :param interface: Endpoint type to check. public, admin or internal
+    :type interface: str
+    :raises: AssertionError
+    """
+    overcloud_auth = openstack_utils.get_overcloud_auth()
+    wait_for_url(overcloud_auth['OS_AUTH_URL'])
+    session = openstack_utils.get_overcloud_keystone_session()
+    keystone_client = openstack_utils.get_keystone_session_client(session)
+    for service in keystone_client.services.list():
+        for ep in keystone_client.endpoints.list(service=service,
+                                                 interface=interface):
+            wait_for_url(
+                ep.url,
+                [requests.codes.ok, requests.codes.multiple_choices])