diff --git a/confluent_server/aiohmi/ipmi/console.py b/confluent_server/aiohmi/ipmi/console.py index 62dafcfe..900a47a5 100644 --- a/confluent_server/aiohmi/ipmi/console.py +++ b/confluent_server/aiohmi/ipmi/console.py @@ -170,6 +170,9 @@ class Console(object): await self._sendpendingoutput() async def _got_payload_instance_info(self, response): + if self.ipmi_session is None: + # console closed while this keepalive was in flight + return if 'error' in response: self.activated = False await self._print_error(response['error']) diff --git a/confluent_server/aiohmi/ipmi/private/session.py b/confluent_server/aiohmi/ipmi/private/session.py index 1b652666..65978901 100644 --- a/confluent_server/aiohmi/ipmi/private/session.py +++ b/confluent_server/aiohmi/ipmi/private/session.py @@ -1147,27 +1147,36 @@ class Session(object): self.logged = 1 self.maxtimeout = 4 # Switch quickly to the relogin recovery self.logontries = 1 # have one relogin waiting to heal - response = await self.raw_command(netfn=0x6, command=0x3b, - data=[self.privlevel]) - if response['code']: + try: + response = await self.raw_command(netfn=0x6, command=0x3b, + data=[self.privlevel]) if response['code'] in (0x80, 0x81) and self.privlevel == 4: # some implementations will let us get this far, # but suddenly get skiddish. Try again in such a case self.privlevel = 3 response = await self.raw_command(netfn=0x6, command=0x3b, data=[self.privlevel]) - if response['code']: - self.logged = 0 - self.onlogpayload = None - self.logging = False - self.logoutexpiry = None + except exc.IpmiException as e: + # session torn down meanwhile, fail the logon rather than + # raising into the shared event loop + self.logged = 0 + self.onlogpayload = None + self.logging = False + self.logoutexpiry = None + await self.onlogon({'error': str(e)}) + return + if response['code']: + self.logged = 0 + self.onlogpayload = None + self.logging = False + self.logoutexpiry = None - mysuffix = " while requesting privelege level %d for %s" % ( - self.privlevel, self.userid) - errstr = get_ipmi_error(response, suffix=mysuffix) - if errstr: - await self.onlogon({'error': errstr}) - return + mysuffix = " while requesting privelege level %d for %s" % ( + self.privlevel, self.userid) + errstr = get_ipmi_error(response, suffix=mysuffix) + if errstr: + await self.onlogon({'error': errstr}) + return self.logging = False self.logoutexpiry = None global KEEPALIVE_SESSIONS diff --git a/confluent_server/aiohmi/redfish/oem/generic.py b/confluent_server/aiohmi/redfish/oem/generic.py index e4a89687..f8ea21ba 100644 --- a/confluent_server/aiohmi/redfish/oem/generic.py +++ b/confluent_server/aiohmi/redfish/oem/generic.py @@ -1334,7 +1334,7 @@ class OEMHandler(object): if component.lower() == 'system': nodeinfo = await self._get_node_info() sysinfo = { - 'UUID': nodeinfo.get('UUID', '').lower(), + 'UUID': (nodeinfo.get('UUID') or '').lower(), 'Serial Number': nodeinfo.get('SerialNumber', ''), 'Manufacturer': nodeinfo.get('Manufacturer', ''), 'Product name': nodeinfo.get('Model', ''), @@ -1358,7 +1358,7 @@ class OEMHandler(object): async def get_inventory(self, withids=False): sysinfo = { - 'UUID': self._varsysinfo.get('UUID', ''), + 'UUID': self._varsysinfo.get('UUID') or '', 'Serial Number': self._varsysinfo.get('SerialNumber', ''), 'Manufacturer': self._varsysinfo.get('Manufacturer', ''), 'Product name': self._varsysinfo.get('Model', ''), diff --git a/confluent_server/confluent/credserver.py b/confluent_server/confluent/credserver.py index a861ed0c..2068ad3a 100644 --- a/confluent_server/confluent/credserver.py +++ b/confluent_server/confluent/credserver.py @@ -55,6 +55,16 @@ async def address_is_somewhat_trusted(address, nodename, cfm): return True return False +async def _recv_exact(cloop, client, size): + data = b'' + while len(data) < size: + chunk = await cloop.sock_recv(client, size - len(data)) + if not chunk: + raise EOFError('Client closed connection') + data += chunk + return data + + class CredServer(object): def __init__(self): self.cfm = cfm.ConfigManager(None) @@ -67,18 +77,18 @@ class CredServer(object): hmacval = None cloop = asyncio.get_running_loop() await cloop.sock_sendall(client, b'\xc2\xd1-\xa8\x80\xd8j\xba') - tlv = bytearray(await cloop.sock_recv(client, 2)) + tlv = bytearray(await _recv_exact(cloop, client, 2)) if tlv[0] != 1: client.close() return - nodename = util.stringify(await cloop.sock_recv(client, tlv[1])) - tlv = bytearray(await cloop.sock_recv(client, 2)) # should always be null + nodename = util.stringify(await _recv_exact(cloop, client, tlv[1])) + tlv = bytearray(await _recv_exact(cloop, client, 2)) # should always be null if tlv[0] == 6: - hmacval = await cloop.sock_recv(client, tlv[1]) + hmacval = await _recv_exact(cloop, client, tlv[1]) hmackey = self.cfm.get_node_attributes(nodename, ['secret.selfapiarmtoken'], decrypt=True) hmackey = hmackey.get(nodename, {}).get('secret.selfapiarmtoken', {}).get('value', None) elif tlv[1]: - await cloop.sock_recv(client, tlv[1]) + await _recv_exact(cloop, client, tlv[1]) apimats = self.cfm.get_node_attributes(nodename, ['deployment.apiarmed', 'deployment.sealedapikey']) apiarmed = apimats.get(nodename, {}).get('deployment.apiarmed', {}).get( @@ -109,19 +119,19 @@ class CredServer(object): rttoken = os.urandom(32) await cloop.sock_sendall(client, rttoken) await cloop.sock_sendall(client, b'\x00\x00') - tlv = bytearray(await cloop.sock_recv(client, 2)) + tlv = bytearray(await _recv_exact(cloop, client, 2)) if tlv[0] != 3: client.close() return - echotoken = await cloop.sock_recv(client, tlv[1]) + echotoken = await _recv_exact(cloop, client, tlv[1]) if echotoken != rttoken: client.close() return - tlv = bytearray(await cloop.sock_recv(client, 2)) + tlv = bytearray(await _recv_exact(cloop, client, 2)) if tlv[0] != 4: client.close() return - echotoken = util.stringify(await cloop.sock_recv(client, tlv[1])) + echotoken = util.stringify(await _recv_exact(cloop, client, tlv[1])) if hmackey: etok = echotoken.encode('utf8') if hmacval != hmac.new(hmackey, etok, hashlib.sha256).digest(): @@ -135,6 +145,9 @@ class CredServer(object): await self.cfm.clear_node_attributes([nodename], ['secret.selfapiarmtoken']) if apiarmed != 'continuous': disarm = {nodename: {'deployment.sealedapikey': '', 'deployment.apiarmed': ''}} + except (ConnectionError, EOFError): + # client went away mid-exchange, not a server fault + pass finally: try: client.close() diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index f8c1e389..3b7e714c 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -18,6 +18,7 @@ import confluent.exceptions as exc from confluent.exceptions import exc_text import confluent.firmwaremanager as firmwaremanager import confluent.interface.console as conapi +import confluent.log as log import confluent.messages as msg import confluent.util as util import copy @@ -33,7 +34,6 @@ import aiohmi.ipmi.console as console import aiohmi.ipmi.command as ipmicommand import socket import ssl -import traceback import confluent.vinzmanager as vinzmanager @@ -229,9 +229,8 @@ async def _ipmi_evtloop(): waiter.send() except asyncio.CancelledError: break - except: # TODO(jbjohnso): log the trace into the log - - traceback.print_exc() + except Exception: + log.logtrace() def get_conn_params(node, configdata): @@ -495,7 +494,7 @@ async def perform_request(operator, node, element, except Exception as e: await results.put(msg.ConfluentNodeError( node, 'Unexpected Error: {0}'.format(exc_text(e)))) - traceback.print_exc() + log.logtrace() finally: await results.put('Done') diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py index d02bcce4..81db38c0 100644 --- a/confluent_server/confluent/sockapi.py +++ b/confluent_server/confluent/sockapi.py @@ -165,26 +165,10 @@ async def sessionhdl(connection, authname, skipauth=False, cert=None): try: await process_request( connection, request, cfm, authdata, authname, skipauth) - except exc.ConfluentException as e: - if ((not isinstance(e, exc.LockedCredentials)) and - e.apierrorcode == 500): - tracelog.log( - traceback.format_exc(), ltype=log.DataTypes.event, - event=log.Events.stacktrace) - await send_data(connection, {'errorcode': e.apierrorcode, - 'error': e.apierrorstr, - 'detail': e.get_error_body()}) - await send_data(connection, {'_requestdone': 1}) except SystemExit: sys.exit(0) except Exception as e: - tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, - event=log.Events.stacktrace) - await send_data( - connection, - {'errorcode': 500, - 'error': 'Unexpected error - ' + str(e)}) - await send_data(connection, {'_requestdone': 1}) + await send_request_error(connection, e) try: request = await tlvdata.recv(connection) except Exception: @@ -199,6 +183,28 @@ async def sessionhdl(connection, authname, skipauth=False, cert=None): pass +async def send_request_error(connection, e): + """Report a failed request, call from the except clause that caught e""" + if isinstance(e, exc.ConfluentException): + # expected failures are not traced + if ((not isinstance(e, exc.LockedCredentials)) and + e.apierrorcode == 500): + tracelog.log( + traceback.format_exc(), ltype=log.DataTypes.event, + event=log.Events.stacktrace) + await send_data(connection, {'errorcode': e.apierrorcode, + 'error': e.apierrorstr, + 'detail': e.get_error_body()}) + else: + tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, + event=log.Events.stacktrace) + await send_data( + connection, + {'errorcode': 500, + 'error': 'Unexpected error - ' + str(e)}) + await send_data(connection, {'_requestdone': 1}) + + async def send_response(responses, connection): if responses is None: return @@ -337,14 +343,7 @@ async def term_interact(authdata, authname, ccons, cfm, connection, consession, await process_request(connection, data, cfm, authdata, authname, skipauth) except Exception as e: - tracelog.log(traceback.format_exc(), - ltype=log.DataTypes.event, - event=log.Events.stacktrace) - await send_data( - connection, - {'errorcode': 500, - 'error': 'Unexpected error - ' + str(e)}) - await send_data(connection, {'_requestdone': 1}) + await send_request_error(connection, e) continue if not consession: break