From 4d6921cabad805511da92e704bf9cdc3c4302df0 Mon Sep 17 00:00:00 2001
From: Vlad Spoiala1 <vspoiala1@lenovo.com>
Date: Mon, 22 Jan 2024 18:49:32 +0200
Subject: [PATCH] Add retry mechanism to FW update.

On v3 systems we sometimes get an empty status when monitoring the
status of the BMC FW upgrade. This happens for 1 to 3 seconds when
transitioning from the "apply" phase to the "validating" phase.
Add a retry mechanism so we don't incorrectly exit on the first empty
status.

Change-Id: I02c653f36d09087cdf76fb7d7dee71917a128c51
---
 pyghmi/ipmi/oem/lenovo/imm.py    | 9 +++++++--
 pyghmi/redfish/oem/generic.py    | 9 +++++++--
 pyghmi/redfish/oem/lenovo/xcc.py | 9 +++++++--
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/pyghmi/ipmi/oem/lenovo/imm.py b/pyghmi/ipmi/oem/lenovo/imm.py
index ba741033..c5deb7cf 100644
--- a/pyghmi/ipmi/oem/lenovo/imm.py
+++ b/pyghmi/ipmi/oem/lenovo/imm.py
@@ -2062,13 +2062,18 @@ class XCCClient(IMMClient):
             complete = False
             phase = "apply"
             statetype = 'TaskState'
-            while not complete:
+            # sometimes we get an empty pgress when transitioning from the apply phase to
+            # the validating phase; add a retry here so we don't exit the loop in this case
+            retry = 3
+            while not complete and retry > 0:
                 pgress, status = self.grab_redfish_response_with_status(
                     monitorurl)
                 if status < 200 or status >= 300:
                     raise Exception(pgress)
                 if not pgress:
-                    break
+                    retry -= 1
+                    ipmisession.Session.pause(3)
+                    continue
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])
diff --git a/pyghmi/redfish/oem/generic.py b/pyghmi/redfish/oem/generic.py
index 1dea64ba..8f989e09 100644
--- a/pyghmi/redfish/oem/generic.py
+++ b/pyghmi/redfish/oem/generic.py
@@ -820,10 +820,15 @@ class OEMHandler(object):
             complete = False
             phase = "apply"
             statetype = 'TaskState'
-            while not complete:
+            # sometimes we get an empty pgress when transitioning from the apply phase to
+            # the validating phase; add a retry here so we don't exit the loop in this case
+            retry = 3
+            while not complete and retry > 0:
                 pgress = self._do_web_request(monitorurl, cache=False)
                 if not pgress:
-                    break
+                    retry -= 1
+                    time.sleep(3)
+                    continue
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])
diff --git a/pyghmi/redfish/oem/lenovo/xcc.py b/pyghmi/redfish/oem/lenovo/xcc.py
index 13c4535e..e093dd6c 100644
--- a/pyghmi/redfish/oem/lenovo/xcc.py
+++ b/pyghmi/redfish/oem/lenovo/xcc.py
@@ -1278,10 +1278,15 @@ class OEMHandler(generic.OEMHandler):
             complete = False
             phase = "apply"
             statetype = 'TaskState'
-            while not complete:
+            # sometimes we get an empty pgress when transitioning from the apply phase to
+            # the validating phase; add a retry here so we don't exit the loop in this case
+            retry = 3
+            while not complete and retry > 0:
                 pgress = self._do_web_request(monitorurl, cache=False)
                 if not pgress:
-                    break
+                    retry -= 1
+                    time.sleep(3)
+                    continue
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])