Add retry mechanism to FW update.

On v3 systems we sometimes get an empty status when monitoring the
status of the BMC FW upgrade. This happens for 1 to 3 seconds when
transitioning from the "apply" phase to the "validating" phase.
Add a retry mechanism so we don't incorrectly exit on the first empty
status.

Change-Id: I02c653f36d09087cdf76fb7d7dee71917a128c51

pyghmi/ipmi/oem/lenovo/imm.py

@@ -2062,13 +2062,18 @@ class XCCClient(IMMClient):
             complete = False
             phase = "apply"
             statetype = 'TaskState'
-            while not complete:
+            # sometimes we get an empty pgress when transitioning from the apply phase to
+            # the validating phase; add a retry here so we don't exit the loop in this case
+            retry = 3
+            while not complete and retry > 0:
                 pgress, status = self.grab_redfish_response_with_status(
                     monitorurl)
                 if status < 200 or status >= 300:
                     raise Exception(pgress)
                 if not pgress:
-                    break
+                    retry -= 1
+                    ipmisession.Session.pause(3)
+                    continue
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])

pyghmi/redfish/oem/generic.py

@@ -820,10 +820,15 @@ class OEMHandler(object):
             complete = False
             phase = "apply"
             statetype = 'TaskState'
-            while not complete:
+            # sometimes we get an empty pgress when transitioning from the apply phase to
+            # the validating phase; add a retry here so we don't exit the loop in this case
+            retry = 3
+            while not complete and retry > 0:
                 pgress = self._do_web_request(monitorurl, cache=False)
                 if not pgress:
-                    break
+                    retry -= 1
+                    time.sleep(3)
+                    continue
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])

pyghmi/redfish/oem/lenovo/xcc.py

@@ -1278,10 +1278,15 @@ class OEMHandler(generic.OEMHandler):
             complete = False
             phase = "apply"
             statetype = 'TaskState'
-            while not complete:
+            # sometimes we get an empty pgress when transitioning from the apply phase to
+            # the validating phase; add a retry here so we don't exit the loop in this case
+            retry = 3
+            while not complete and retry > 0:
                 pgress = self._do_web_request(monitorurl, cache=False)
                 if not pgress:
-                    break
+                    retry -= 1
+                    time.sleep(3)
+                    continue
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])