From cb729147e7ccde738122611786acf99296687276 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 17 Nov 2026 16:25:17 +0100 Subject: [PATCH 1/5] Tolerate credserver clients that disconnect early A client that reset the connection or closed it before finishing the exchange left a stack trace in the trace log (ConnectionResetError, or an IndexError on an empty read). Read each field in full, and treat a client going away as the end of that exchange rather than a server fault. --- confluent_server/confluent/credserver.py | 31 +++++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/confluent_server/confluent/credserver.py b/confluent_server/confluent/credserver.py index a861ed0c..2068ad3a 100644 --- a/confluent_server/confluent/credserver.py +++ b/confluent_server/confluent/credserver.py @@ -55,6 +55,16 @@ async def address_is_somewhat_trusted(address, nodename, cfm): return True return False +async def _recv_exact(cloop, client, size): + data = b'' + while len(data) < size: + chunk = await cloop.sock_recv(client, size - len(data)) + if not chunk: + raise EOFError('Client closed connection') + data += chunk + return data + + class CredServer(object): def __init__(self): self.cfm = cfm.ConfigManager(None) @@ -67,18 +77,18 @@ class CredServer(object): hmacval = None cloop = asyncio.get_running_loop() await cloop.sock_sendall(client, b'\xc2\xd1-\xa8\x80\xd8j\xba') - tlv = bytearray(await cloop.sock_recv(client, 2)) + tlv = bytearray(await _recv_exact(cloop, client, 2)) if tlv[0] != 1: client.close() return - nodename = util.stringify(await cloop.sock_recv(client, tlv[1])) - tlv = bytearray(await cloop.sock_recv(client, 2)) # should always be null + nodename = util.stringify(await _recv_exact(cloop, client, tlv[1])) + tlv = bytearray(await _recv_exact(cloop, client, 2)) # should always be null if tlv[0] == 6: - hmacval = await cloop.sock_recv(client, tlv[1]) + hmacval = await _recv_exact(cloop, client, tlv[1]) hmackey = self.cfm.get_node_attributes(nodename, ['secret.selfapiarmtoken'], decrypt=True) hmackey = hmackey.get(nodename, {}).get('secret.selfapiarmtoken', {}).get('value', None) elif tlv[1]: - await cloop.sock_recv(client, tlv[1]) + await _recv_exact(cloop, client, tlv[1]) apimats = self.cfm.get_node_attributes(nodename, ['deployment.apiarmed', 'deployment.sealedapikey']) apiarmed = apimats.get(nodename, {}).get('deployment.apiarmed', {}).get( @@ -109,19 +119,19 @@ class CredServer(object): rttoken = os.urandom(32) await cloop.sock_sendall(client, rttoken) await cloop.sock_sendall(client, b'\x00\x00') - tlv = bytearray(await cloop.sock_recv(client, 2)) + tlv = bytearray(await _recv_exact(cloop, client, 2)) if tlv[0] != 3: client.close() return - echotoken = await cloop.sock_recv(client, tlv[1]) + echotoken = await _recv_exact(cloop, client, tlv[1]) if echotoken != rttoken: client.close() return - tlv = bytearray(await cloop.sock_recv(client, 2)) + tlv = bytearray(await _recv_exact(cloop, client, 2)) if tlv[0] != 4: client.close() return - echotoken = util.stringify(await cloop.sock_recv(client, tlv[1])) + echotoken = util.stringify(await _recv_exact(cloop, client, tlv[1])) if hmackey: etok = echotoken.encode('utf8') if hmacval != hmac.new(hmackey, etok, hashlib.sha256).digest(): @@ -135,6 +145,9 @@ class CredServer(object): await self.cfm.clear_node_attributes([nodename], ['secret.selfapiarmtoken']) if apiarmed != 'continuous': disarm = {nodename: {'deployment.sealedapikey': '', 'deployment.apiarmed': ''}} + except (ConnectionError, EOFError): + # client went away mid-exchange, not a server fault + pass finally: try: client.close() From e73a42ca5e5d086e7fe8a3249f7338e857ea076c Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 17 Nov 2026 16:25:17 +0100 Subject: [PATCH 2/5] Report request errors from console sessions like other requests A request sent from within a console session, such as the power state check confetty makes, went through a handler that logged every failure as an unexpected 500 with a stack trace. A node with an unreachable BMC filled the trace log with TargetEndpointUnreachable. Share the error reporting of the main request loop, so the client gets the real error code and only unexpected failures are traced. --- confluent_server/confluent/sockapi.py | 49 +++++++++++++-------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py index d02bcce4..81db38c0 100644 --- a/confluent_server/confluent/sockapi.py +++ b/confluent_server/confluent/sockapi.py @@ -165,26 +165,10 @@ async def sessionhdl(connection, authname, skipauth=False, cert=None): try: await process_request( connection, request, cfm, authdata, authname, skipauth) - except exc.ConfluentException as e: - if ((not isinstance(e, exc.LockedCredentials)) and - e.apierrorcode == 500): - tracelog.log( - traceback.format_exc(), ltype=log.DataTypes.event, - event=log.Events.stacktrace) - await send_data(connection, {'errorcode': e.apierrorcode, - 'error': e.apierrorstr, - 'detail': e.get_error_body()}) - await send_data(connection, {'_requestdone': 1}) except SystemExit: sys.exit(0) except Exception as e: - tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, - event=log.Events.stacktrace) - await send_data( - connection, - {'errorcode': 500, - 'error': 'Unexpected error - ' + str(e)}) - await send_data(connection, {'_requestdone': 1}) + await send_request_error(connection, e) try: request = await tlvdata.recv(connection) except Exception: @@ -199,6 +183,28 @@ async def sessionhdl(connection, authname, skipauth=False, cert=None): pass +async def send_request_error(connection, e): + """Report a failed request, call from the except clause that caught e""" + if isinstance(e, exc.ConfluentException): + # expected failures are not traced + if ((not isinstance(e, exc.LockedCredentials)) and + e.apierrorcode == 500): + tracelog.log( + traceback.format_exc(), ltype=log.DataTypes.event, + event=log.Events.stacktrace) + await send_data(connection, {'errorcode': e.apierrorcode, + 'error': e.apierrorstr, + 'detail': e.get_error_body()}) + else: + tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, + event=log.Events.stacktrace) + await send_data( + connection, + {'errorcode': 500, + 'error': 'Unexpected error - ' + str(e)}) + await send_data(connection, {'_requestdone': 1}) + + async def send_response(responses, connection): if responses is None: return @@ -337,14 +343,7 @@ async def term_interact(authdata, authname, ccons, cfm, connection, consession, await process_request(connection, data, cfm, authdata, authname, skipauth) except Exception as e: - tracelog.log(traceback.format_exc(), - ltype=log.DataTypes.event, - event=log.Events.stacktrace) - await send_data( - connection, - {'errorcode': 500, - 'error': 'Unexpected error - ' + str(e)}) - await send_data(connection, {'_requestdone': 1}) + await send_request_error(connection, e) continue if not consession: break From 75a5e75063a77946a678606b564cd2e73fe2c319 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 17 Nov 2026 16:25:17 +0100 Subject: [PATCH 3/5] Log IPMI plugin stack traces to the trace log traceback.print_exc() went to stderr, where the log redirect tags every line with its caller, traceback.py, which made the traces unreadable. --- .../confluent/plugins/hardwaremanagement/ipmi.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index f8c1e389..3b7e714c 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -18,6 +18,7 @@ import confluent.exceptions as exc from confluent.exceptions import exc_text import confluent.firmwaremanager as firmwaremanager import confluent.interface.console as conapi +import confluent.log as log import confluent.messages as msg import confluent.util as util import copy @@ -33,7 +34,6 @@ import aiohmi.ipmi.console as console import aiohmi.ipmi.command as ipmicommand import socket import ssl -import traceback import confluent.vinzmanager as vinzmanager @@ -229,9 +229,8 @@ async def _ipmi_evtloop(): waiter.send() except asyncio.CancelledError: break - except: # TODO(jbjohnso): log the trace into the log - - traceback.print_exc() + except Exception: + log.logtrace() def get_conn_params(node, configdata): @@ -495,7 +494,7 @@ async def perform_request(operator, node, element, except Exception as e: await results.put(msg.ConfluentNodeError( node, 'Unexpected Error: {0}'.format(exc_text(e)))) - traceback.print_exc() + log.logtrace() finally: await results.put('Done') From aa250bb91b9dd4379f6dbc95f2cbdd66d9e98c37 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 17 Nov 2026 16:25:17 +0100 Subject: [PATCH 4/5] Handle IPMI sessions torn down while a response is pending A SOL keepalive reply arriving after the console was closed failed with an AttributeError on the released session. A privilege level request on a session that was logged out or given up on during login raised IpmiException out of the packet handler into the event loop shared by all sessions. Ignore the late keepalive reply, and report the failed login to its waiters instead. --- confluent_server/aiohmi/ipmi/console.py | 3 ++ .../aiohmi/ipmi/private/session.py | 37 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/confluent_server/aiohmi/ipmi/console.py b/confluent_server/aiohmi/ipmi/console.py index 62dafcfe..900a47a5 100644 --- a/confluent_server/aiohmi/ipmi/console.py +++ b/confluent_server/aiohmi/ipmi/console.py @@ -170,6 +170,9 @@ class Console(object): await self._sendpendingoutput() async def _got_payload_instance_info(self, response): + if self.ipmi_session is None: + # console closed while this keepalive was in flight + return if 'error' in response: self.activated = False await self._print_error(response['error']) diff --git a/confluent_server/aiohmi/ipmi/private/session.py b/confluent_server/aiohmi/ipmi/private/session.py index 1b652666..65978901 100644 --- a/confluent_server/aiohmi/ipmi/private/session.py +++ b/confluent_server/aiohmi/ipmi/private/session.py @@ -1147,27 +1147,36 @@ class Session(object): self.logged = 1 self.maxtimeout = 4 # Switch quickly to the relogin recovery self.logontries = 1 # have one relogin waiting to heal - response = await self.raw_command(netfn=0x6, command=0x3b, - data=[self.privlevel]) - if response['code']: + try: + response = await self.raw_command(netfn=0x6, command=0x3b, + data=[self.privlevel]) if response['code'] in (0x80, 0x81) and self.privlevel == 4: # some implementations will let us get this far, # but suddenly get skiddish. Try again in such a case self.privlevel = 3 response = await self.raw_command(netfn=0x6, command=0x3b, data=[self.privlevel]) - if response['code']: - self.logged = 0 - self.onlogpayload = None - self.logging = False - self.logoutexpiry = None + except exc.IpmiException as e: + # session torn down meanwhile, fail the logon rather than + # raising into the shared event loop + self.logged = 0 + self.onlogpayload = None + self.logging = False + self.logoutexpiry = None + await self.onlogon({'error': str(e)}) + return + if response['code']: + self.logged = 0 + self.onlogpayload = None + self.logging = False + self.logoutexpiry = None - mysuffix = " while requesting privelege level %d for %s" % ( - self.privlevel, self.userid) - errstr = get_ipmi_error(response, suffix=mysuffix) - if errstr: - await self.onlogon({'error': errstr}) - return + mysuffix = " while requesting privelege level %d for %s" % ( + self.privlevel, self.userid) + errstr = get_ipmi_error(response, suffix=mysuffix) + if errstr: + await self.onlogon({'error': errstr}) + return self.logging = False self.logoutexpiry = None global KEEPALIVE_SESSIONS From d7a4af65b158fd1b7790f09fd67b67d0cf080a62 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 17 Nov 2026 16:41:50 +0100 Subject: [PATCH 5/5] Tolerate a null system UUID in Redfish inventory A Giga Computing BMC reports the system UUID as null, which made nodeinventory fail with 'NoneType' object has no attribute 'lower'. Treat a null UUID like a missing one. --- confluent_server/aiohmi/redfish/oem/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_server/aiohmi/redfish/oem/generic.py b/confluent_server/aiohmi/redfish/oem/generic.py index e4a89687..f8ea21ba 100644 --- a/confluent_server/aiohmi/redfish/oem/generic.py +++ b/confluent_server/aiohmi/redfish/oem/generic.py @@ -1334,7 +1334,7 @@ class OEMHandler(object): if component.lower() == 'system': nodeinfo = await self._get_node_info() sysinfo = { - 'UUID': nodeinfo.get('UUID', '').lower(), + 'UUID': (nodeinfo.get('UUID') or '').lower(), 'Serial Number': nodeinfo.get('SerialNumber', ''), 'Manufacturer': nodeinfo.get('Manufacturer', ''), 'Product name': nodeinfo.get('Model', ''), @@ -1358,7 +1358,7 @@ class OEMHandler(object): async def get_inventory(self, withids=False): sysinfo = { - 'UUID': self._varsysinfo.get('UUID', ''), + 'UUID': self._varsysinfo.get('UUID') or '', 'Serial Number': self._varsysinfo.get('SerialNumber', ''), 'Manufacturer': self._varsysinfo.get('Manufacturer', ''), 'Product name': self._varsysinfo.get('Model', ''),