From ae19b05df49ea8bc2f657256fc9baaded37cecf3 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 15 Apr 2025 09:10:38 -0400
Subject: [PATCH] Tolerate a few timeouts while trying to monitor an update

Very long lived updates may incur hundreds of status checks, and we need to
tolerate a few.

Change-Id: Id4347d0c854679cde60da42c4cf1921ef53e2a20
---
 pyghmi/ipmi/oem/lenovo/imm.py    | 10 ++++++++--
 pyghmi/redfish/oem/generic.py    |  8 +++++++-
 pyghmi/redfish/oem/lenovo/xcc.py |  8 +++++++-
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/pyghmi/ipmi/oem/lenovo/imm.py b/pyghmi/ipmi/oem/lenovo/imm.py
index 71954936..ac9b983d 100644
--- a/pyghmi/ipmi/oem/lenovo/imm.py
+++ b/pyghmi/ipmi/oem/lenovo/imm.py
@@ -2101,14 +2101,18 @@ class XCCClient(IMMClient):
             # the validating phase; add a retry here so we don't exit the loop in this case
             retry = 3
             while not complete and retry > 0:
-                pgress, status = self.grab_redfish_response_with_status(
-                    monitorurl)
+                try:
+                    pgress, status = self.grab_redfish_response_with_status(
+                        monitorurl)
+                except socket.timeout:
+                    pgress = None
                 if status < 200 or status >= 300:
                     raise Exception(pgress)
                 if not pgress:
                     retry -= 1
                     ipmisession.Session.pause(3)
                     continue
+                retry = 3 # reset retry counter
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])
@@ -2129,6 +2133,8 @@ class XCCClient(IMMClient):
                         ipmisession.Session.pause(3)
                 else:
                     ipmisession.Session.pause(3)
+            if not retry:
+                raise Exception('Falied to monitor update progress due to excessive timeouts')
             if bank == 'backup':
                 return 'complete'
             return 'pending'
diff --git a/pyghmi/redfish/oem/generic.py b/pyghmi/redfish/oem/generic.py
index bb0e7647..152dd9d9 100644
--- a/pyghmi/redfish/oem/generic.py
+++ b/pyghmi/redfish/oem/generic.py
@@ -1114,11 +1114,15 @@ class OEMHandler(object):
             retry = 3
             pct = 0.0
             while not complete and retry > 0:
-                pgress = self._do_web_request(monitorurl, cache=False)
+                try:
+                    pgress = self._do_web_request(monitorurl, cache=False)
+                except socket.timeout:
+                    pgress = None
                 if not pgress:
                     retry -= 1
                     time.sleep(3)
                     continue
+                retry = 3 # reset retry counter
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])
@@ -1143,6 +1147,8 @@ class OEMHandler(object):
                         time.sleep(3)
                 else:
                     time.sleep(3)
+            if not retry:
+                raise Exception('Falied to monitor update progress due to excessive timeouts')
             return 'pending'
         finally:
             if 'HttpPushUriTargetsBusy' in usd:
diff --git a/pyghmi/redfish/oem/lenovo/xcc.py b/pyghmi/redfish/oem/lenovo/xcc.py
index fc8b8999..07aeafb5 100644
--- a/pyghmi/redfish/oem/lenovo/xcc.py
+++ b/pyghmi/redfish/oem/lenovo/xcc.py
@@ -1308,11 +1308,15 @@ class OEMHandler(generic.OEMHandler):
             # the validating phase; add a retry here so we don't exit the loop in this case
             retry = 3
             while not complete and retry > 0:
-                pgress = self._do_web_request(monitorurl, cache=False)
+                try:
+                    pgress = self._do_web_request(monitorurl, cache=False)
+                except socket.timeout:
+                    pgress = None
                 if not pgress:
                     retry -= 1
                     time.sleep(3)
                     continue
+                retry = 3 # reset retry counter
                 for msg in pgress.get('Messages', []):
                     if 'Verify failed' in msg.get('Message', ''):
                         raise Exception(msg['Message'])
@@ -1334,6 +1338,8 @@ class OEMHandler(generic.OEMHandler):
                         time.sleep(3)
                 else:
                     time.sleep(3)
+            if not retry:
+                raise Exception('Falied to monitor update progress due to excessive timeouts')
             if bank == 'backup':
                 return 'complete'
             return 'pending'