From 0cff70c60777391c7a3590c612670488799593c3 Mon Sep 17 00:00:00 2001
From: tkucherera <tkucherera@lenovo.com>
Date: Wed, 9 Aug 2023 10:37:35 -0400
Subject: [PATCH 01/41] firmware update

---
 confluent_server/confluent/httpapi.py | 96 +++++++++++++++++++++------
 1 file changed, 75 insertions(+), 21 deletions(-)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index 5a145a0c..a357d72a 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -55,6 +55,12 @@ except ModuleNotFoundError:
     import urllib.parse as urlparse
 import eventlet.websocket
 import eventlet.wsgi
+
+#these modifications might need to be changed later now just proof of concept
+import os
+import cgi
+#from cgi import parse_qs, escape
+
 #scgi = eventlet.import_patched('flup.server.scgi')
 tlvdata = confluent.tlvdata
 
@@ -171,6 +177,7 @@ def _get_query_dict(env, reqbody, reqtype):
     qdict = {}
     try:
         qstring = env['QUERY_STRING']
+  
     except KeyError:
         qstring = None
     if qstring:
@@ -288,16 +295,12 @@ def _authorize_request(env, operation, reqbody):
             authdata = auth.authorize(name, element=element, operation=operation)
         else:
             element = None
-    if not authdata:
-        if 'HTTP_CONFLUENTSESSION' in env:
-            sessionid = env['HTTP_CONFLUENTSESSION']
+    if (not authdata) and 'HTTP_COOKIE' in env:
+        cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
+        if cidx >= 0:
+            sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
+            sessid = sessionid
             sessid = sessionid
-        elif 'HTTP_COOKIE' in env:
-            cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
-            if cidx >= 0:
-                sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
-                sessid = sessionid
-        if sessionid:
             if sessionid in httpsessions:
                 if _csrf_valid(env, httpsessions[sessionid]):
                     if env['PATH_INFO'] == '/sessions/current/logout':
@@ -487,8 +490,6 @@ def wsock_handler(ws):
                         elif clientmsg[0] == '!':
                             msg = json.loads(clientmsg[1:])
                             action = msg.get('operation', None)
-                            if not action:
-                                action = msg.get('action', None)
                             targ = msg.get('target', None)
                             if targ:
                                 authdata = auth.authorize(name, targ, operation=action)
@@ -526,13 +527,6 @@ def wsock_handler(ws):
                                             datacallback=datacallback,
                                             width=width, height=height)
                                     myconsoles[clientsessid] = consession
-                            elif action == 'resize':
-                                clientsessid = '{0}'.format(msg['sessid'])
-                                myconsoles[clientsessid].resize(
-                                    width=msg['width'], height=msg['height'])
-                            if action == 'break':
-                                clientsessid = '{0}'.format(msg['sessid'])
-                                myconsoles[clientsessid].send_break()
                             elif action == 'stop':
                                 sessid = '{0}'.format(msg.get('sessid', None))
                                 if sessid in myconsoles:
@@ -666,9 +660,13 @@ def resourcehandler_backend(env, start_response):
         start_response('302 Found', headers)
         yield ''
         return
-    if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:
-        reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
+
+    if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0: 
+        reqbody = ''     
+        # reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
         reqtype = env['CONTENT_TYPE']
+        if 'application/json' in reqtype:
+            reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
     operation = opmap[env['REQUEST_METHOD']]
     querydict = _get_query_dict(env, reqbody, reqtype)
     if operation != 'retrieve' and 'restexplorerop' in querydict:
@@ -887,6 +885,60 @@ def resourcehandler_backend(env, start_response):
             start_response('200 OK', headers)
             yield rsp
             return
+    elif (operation == 'create' and ('/firmware/updates/active' in env['PATH_INFO'])):
+        #TODO figure out why content length shows data being there but data string is empty (TK)
+        # Solution the wsgi.iput stream can only be read once and it is being read at the top 
+        if 'multipart/form-data' in reqtype:
+            field_storage = cgi.FieldStorage(fp=env['wsgi.input'], environ=env, keep_blank_values=True)
+            for item in field_storage.list:
+                if item.filename: 
+                    storage_file_path = '/var/lib/confluent/client_assets/' + item.filename       
+                    file_content = item.file.read()    
+                    with open(storage_file_path, 'wb') as file:
+                        file.write(file_content)
+            yield json.dumps({'data': storage_file_path})
+            start_response('200 OK', headers)
+            return 
+
+        url = env['PATH_INFO']
+        if 'application/json' in reqtype:
+            if not isinstance(reqbody, str):
+                reqbody = reqbody.decode('utf8')
+            pbody = json.loads(reqbody)
+            args = pbody['args']
+            args_dict = {'filename': args}
+            try:
+                args_dict.update({'bank': pbody['bank']})
+            except KeyError:
+                pass
+            noderrs = {}
+            nodeurls = {}
+            # start_response('202 Accepted', headers)
+            hdlr = pluginapi.handle_path(url, operation, cfgmgr, args_dict)
+            for res in hdlr:
+                if isinstance(res, confluent.messages.CreatedResource):
+                    watchurl = res.kvpairs['created']
+                    currnode = watchurl.split('/')[1]
+                    nodeurls[currnode] = '/' + watchurl
+            yield json.dumps({'data': nodeurls})
+            start_response('200 OK', headers)
+            return
+
+    elif (operation == 'delete' and ('/firmware/updates/active' in env['PATH_INFO'])):
+        url = env['PATH_INFO']
+        if 'application/json' in reqtype:
+            if not isinstance(reqbody, str):
+                reqbody = reqbody.decode('utf8')
+            pbody = json.loads(reqbody)
+            args = pbody['args']
+            try:
+                os.remove(args)
+                start_response('200 OK', headers)        
+                return
+            except Exception as e:
+                return e
+
+
     else:
         # normal request
         url = env['PATH_INFO']
@@ -972,6 +1024,7 @@ def _assemble_html(responses, resource, querydict, url, extension):
         else:
             pendingrsp.append(rsp)
     for rsp in pendingrsp:
+        print(rsp.html() + "<br>")
         yield rsp.html() + "<br>"
     if iscollection:
         # localpath = url[:-2] (why was this here??)
@@ -1070,6 +1123,7 @@ def serve(bind_host, bind_port):
         try:
             sock = eventlet.listen(
                 (bind_host, bind_port, 0, 0), family=socket.AF_INET6)
+            
         except socket.error as e:
             if e.errno != 98:
                 raise
@@ -1104,4 +1158,4 @@ class HttpApi(object):
         self.server = eventlet.spawn(serve, self.bind_host, self.bind_port)
 
 
-_cleaner = eventlet.spawn(_sessioncleaner)
+_cleaner = eventlet.spawn(_sessioncleaner)
\ No newline at end of file

From 2c90227f636348e6253a8d07859a51f01dd4388d Mon Sep 17 00:00:00 2001
From: tkucherera <tkucherera@lenovo.com>
Date: Wed, 9 Aug 2023 10:48:09 -0400
Subject: [PATCH 02/41] "Firmware Update "

---
 confluent_server/confluent/httpapi.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index a357d72a..d418302d 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -886,7 +886,6 @@ def resourcehandler_backend(env, start_response):
             yield rsp
             return
     elif (operation == 'create' and ('/firmware/updates/active' in env['PATH_INFO'])):
-        #TODO figure out why content length shows data being there but data string is empty (TK)
         # Solution the wsgi.iput stream can only be read once and it is being read at the top 
         if 'multipart/form-data' in reqtype:
             field_storage = cgi.FieldStorage(fp=env['wsgi.input'], environ=env, keep_blank_values=True)

From 3cacbf283ace3ab2f6b70ded7ecc8b9f23463609 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 7 Sep 2023 10:09:19 -0400
Subject: [PATCH 03/41] Fix cloning to NVME boot

The partition device names
have a different scheme
for nvme v. other devices.
---
 .../profiles/default/scripts/image2disk.py          | 13 ++++++++++---
 .../profiles/default/scripts/image2disk.py          | 12 +++++++++---
 .../profiles/default/scripts/image2disk.py          | 12 +++++++++---
 .../profiles/default/scripts/image2disk.py          | 13 ++++++++++---
 4 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
index fa378632..08d48a9c 100644
--- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
+++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
@@ -11,6 +11,13 @@ import struct
 import sys
 import subprocess
 
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -258,7 +265,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk , volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -268,10 +275,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py
index 0f0a6745..92facb4b 100644
--- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py
+++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py
@@ -13,6 +13,12 @@ import subprocess
 
 bootuuid = None
 
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -295,7 +301,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -305,10 +311,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py
index 0f0a6745..92facb4b 100644
--- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py
+++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py
@@ -13,6 +13,12 @@ import subprocess
 
 bootuuid = None
 
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -295,7 +301,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -305,10 +311,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py
index 7371dcf1..5d15e3d4 100644
--- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py
+++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py
@@ -13,6 +13,13 @@ import subprocess
 
 bootuuid = None
 
+
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -292,7 +299,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -302,10 +309,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')

From 8ca1f80ef6f523d722f2f6cb3bc6fa63f2ae187d Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 7 Sep 2023 14:36:56 -0400
Subject: [PATCH 04/41] Fix implicit nic in confignet

If the implicit IP
is not in any of the
attribute groups of net,
then auto-vivify from the normal place.
---
 confluent_server/confluent/netutil.py | 35 +++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py
index dcce1544..ebf88b1d 100644
--- a/confluent_server/confluent/netutil.py
+++ b/confluent_server/confluent/netutil.py
@@ -332,6 +332,18 @@ def get_full_net_config(configmanager, node, serverip=None):
     if serverip:
         myaddrs = get_addresses_by_serverip(serverip)
     nm = NetManager(myaddrs, node, configmanager)
+    nnc = get_nic_config(configmanager, node, serverip=serverip)
+    defaultnic = {}
+    if nnc.get('ipv4_address', None):
+        defaultnic['ipv4_address'] = '{}/{}'.format(nnc['ipv4_address'], nnc['prefix'])
+    if nnc.get('ipv4_gateway', None):
+        defaultnic['ipv4_gateway'] = nnc['ipv4_gateway']
+    if nnc.get('ipv4_method', None):
+        defaultnic['ipv4_method'] = nnc['ipv4_method']
+    if nnc.get('ipv6_address', None):
+        defaultnic['ipv6_address'] = '{}/{}'.format(nnc['ipv6_address'], nnc['ipv6_prefix'])
+    if nnc.get('ipv6_method', None):
+        defaultnic['ipv6_method'] = nnc['ipv6_method']
     if None in attribs:
         nm.process_attribs(None, attribs[None])
         del attribs[None]
@@ -345,6 +357,29 @@ def get_full_net_config(configmanager, node, serverip=None):
     retattrs['extranets'] = nm.myattribs
     for attri in retattrs['extranets']:
         add_netmask(retattrs['extranets'][attri])
+        if retattrs['extranets'][attri].get('ipv4_address', None) == defaultnic.get('ipv4_address', 'NOPE'):
+            defaultnic = {}
+        if retattrs['extranets'][attri].get('ipv6_address', None) == defaultnic.get('ipv6_address', 'NOPE'):
+            defaultnic = {}
+    if 'default' not in retattrs and defaultnic:
+        retattrs['default'] = defaultnic
+        add_netmask(retattrs['default'])
+        ipv4addr = defaultnic.get('ipv4_address', None)
+        if '/' in ipv4addr:
+            ipv4bytes = socket.inet_pton(socket.AF_INET, ipv4addr.split('/')[0])
+            for addr in nm.myaddrs:
+                if addr[0] != socket.AF_INET:
+                    continue
+                if ipn_on_same_subnet(addr[0], addr[1], ipv4bytes, addr[2]):
+                    defaultnic['current_nic'] = True
+        ipv6addr = defaultnic.get('ipv6_address', None)
+        if '/' in ipv6addr:
+            ipv6bytes = socket.inet_pton(socket.AF_INET6, ipv6addr.split('/')[0])
+            for addr in nm.myaddrs:
+                if addr[0] != socket.AF_INET6:
+                    continue
+                if ipn_on_same_subnet(addr[0], addr[1], ipv6bytes, addr[2]):
+                    defaultnic['current_nic'] = True
     return retattrs
 
 

From 691d92f735cd82a4ab73a36b418ff823248351f1 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 7 Sep 2023 14:41:16 -0400
Subject: [PATCH 05/41] Avoid calling implicit nic config if nowhere to put it

If 'None' attributes are in use,
we'd have no where to
stick implicit configuration anyway.
---
 confluent_server/confluent/netutil.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py
index ebf88b1d..66ed2169 100644
--- a/confluent_server/confluent/netutil.py
+++ b/confluent_server/confluent/netutil.py
@@ -332,18 +332,7 @@ def get_full_net_config(configmanager, node, serverip=None):
     if serverip:
         myaddrs = get_addresses_by_serverip(serverip)
     nm = NetManager(myaddrs, node, configmanager)
-    nnc = get_nic_config(configmanager, node, serverip=serverip)
     defaultnic = {}
-    if nnc.get('ipv4_address', None):
-        defaultnic['ipv4_address'] = '{}/{}'.format(nnc['ipv4_address'], nnc['prefix'])
-    if nnc.get('ipv4_gateway', None):
-        defaultnic['ipv4_gateway'] = nnc['ipv4_gateway']
-    if nnc.get('ipv4_method', None):
-        defaultnic['ipv4_method'] = nnc['ipv4_method']
-    if nnc.get('ipv6_address', None):
-        defaultnic['ipv6_address'] = '{}/{}'.format(nnc['ipv6_address'], nnc['ipv6_prefix'])
-    if nnc.get('ipv6_method', None):
-        defaultnic['ipv6_method'] = nnc['ipv6_method']
     if None in attribs:
         nm.process_attribs(None, attribs[None])
         del attribs[None]
@@ -354,6 +343,18 @@ def get_full_net_config(configmanager, node, serverip=None):
         retattrs['default'] = nm.myattribs[None]
         add_netmask(retattrs['default'])
         del nm.myattribs[None]
+    else:
+        nnc = get_nic_config(configmanager, node, serverip=serverip)
+        if nnc.get('ipv4_address', None):
+            defaultnic['ipv4_address'] = '{}/{}'.format(nnc['ipv4_address'], nnc['prefix'])
+        if nnc.get('ipv4_gateway', None):
+            defaultnic['ipv4_gateway'] = nnc['ipv4_gateway']
+        if nnc.get('ipv4_method', None):
+            defaultnic['ipv4_method'] = nnc['ipv4_method']
+        if nnc.get('ipv6_address', None):
+            defaultnic['ipv6_address'] = '{}/{}'.format(nnc['ipv6_address'], nnc['ipv6_prefix'])
+        if nnc.get('ipv6_method', None):
+            defaultnic['ipv6_method'] = nnc['ipv6_method']
     retattrs['extranets'] = nm.myattribs
     for attri in retattrs['extranets']:
         add_netmask(retattrs['extranets'][attri])
@@ -361,7 +362,7 @@ def get_full_net_config(configmanager, node, serverip=None):
             defaultnic = {}
         if retattrs['extranets'][attri].get('ipv6_address', None) == defaultnic.get('ipv6_address', 'NOPE'):
             defaultnic = {}
-    if 'default' not in retattrs and defaultnic:
+    if defaultnic:
         retattrs['default'] = defaultnic
         add_netmask(retattrs['default'])
         ipv4addr = defaultnic.get('ipv4_address', None)

From fe78034eaa880de83f1857b02f21b96d3d9a9c4a Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 7 Sep 2023 14:56:07 -0400
Subject: [PATCH 06/41] Add '-y' to imgutil build

Allow non-interactive imgutil build.
---
 imgutil/imgutil | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/imgutil/imgutil b/imgutil/imgutil
index b683b1e5..73005c39 100644
--- a/imgutil/imgutil
+++ b/imgutil/imgutil
@@ -437,6 +437,7 @@ def get_mydir(oscategory):
 class OsHandler(object):
     def __init__(self, name, version, arch, args):
         self.name = name
+        self._interactive = True
         self.version = version
         self.arch = arch
         self.sourcepath = None
@@ -464,6 +465,9 @@ class OsHandler(object):
         except AttributeError:
             self.addrepos = []
 
+    def set_interactive(self, shouldbeinteractive):
+        self._interactive = shouldbeinteractive
+
     def get_json(self):
         odata = [self.oscategory, self.version, self.arch, self.name]
         for idx in range(len(odata)):
@@ -577,7 +581,10 @@ class SuseHandler(OsHandler):
         cmd = ['chmod', 'a+x']
         cmd.extend(glob.glob(os.path.join(targdir, '*')))
         subprocess.check_call(cmd)
-        subprocess.check_call(['zypper', '-R', self.targpath, 'install'] + self.zyppargs)
+        if self._interactive:
+            subprocess.check_call(['zypper', '-R', self.targpath, 'install'] + self.zyppargs)
+        else:
+            subprocess.check_call(['zypper', '-n', '-R', self.targpath, 'install'] + self.zyppargs)
         os.symlink('/usr/lib/systemd/system/sshd.service', os.path.join(self.targpath, 'etc/systemd/system/multi-user.target.wants/sshd.service'))
         if os.path.exists(os.path.join(self.targpath, 'sbin/mkinitrd')):
             args.cmd = ['mkinitrd']
@@ -625,7 +632,6 @@ class ElHandler(OsHandler):
         self.yumargs = []
         super().__init__(name, version, arch, args)
 
-
     def add_pkglists(self):
         self.yumargs.extend(self.list_packages())
 
@@ -657,7 +663,10 @@ class ElHandler(OsHandler):
         cmd = ['chmod', 'a+x']
         cmd.extend(glob.glob(os.path.join(targdir, '*')))
         subprocess.check_call(cmd)
-        subprocess.check_call(['yum'] + self.yumargs)
+        if self._interactive:
+            subprocess.check_call(['yum'] + self.yumargs)
+        else:
+            subprocess.check_call(['yum', '-y'] + self.yumargs)
         with open('/proc/mounts') as mountinfo:
             for line in mountinfo.readlines():
                 if line.startswith('selinuxfs '):
@@ -794,6 +803,7 @@ def main():
     buildp.add_argument('-a', '--addpackagelist', action='append', default=[],
                         help='A list of additional packages to include, may be specified multiple times')
     buildp.add_argument('-s', '--source', help='Directory to pull installation from, typically a subdirectory of /var/lib/confluent/distributions. By default, the repositories for the build system are used.')
+    buildp.add_argument('-y', '--non-interactive', help='Avoid prompting for confirmation', action='store_true')
     buildp.add_argument('-v', '--volume',
         help='Directory to make available in the build environment.  -v / will '
         'cause it to be mounted in image as /run/external/, -v /:/run/root '
@@ -1128,6 +1138,8 @@ def build_root(args):
             sys.stderr.write(
                 'Unable to recognize build system os\n')
         sys.exit(1)
+    if args.non_interactive:
+        oshandler.set_interactive(True)
     oshandler.set_target(args.scratchdir)
     oshandler.add_pkglists()
     for dirname in ('proc', 'sys', 'dev', 'run'):

From 8de6f4356d7b3eacd91f1aef94337dfab466a498 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Thu, 7 Sep 2023 15:12:43 -0400
Subject: [PATCH 07/41] Fix sense of new flag

Accidently made flag exactly opposite of intended
---
 imgutil/imgutil | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/imgutil/imgutil b/imgutil/imgutil
index 73005c39..39c40024 100644
--- a/imgutil/imgutil
+++ b/imgutil/imgutil
@@ -1139,7 +1139,7 @@ def build_root(args):
                 'Unable to recognize build system os\n')
         sys.exit(1)
     if args.non_interactive:
-        oshandler.set_interactive(True)
+        oshandler.set_interactive(False)
     oshandler.set_target(args.scratchdir)
     oshandler.add_pkglists()
     for dirname in ('proc', 'sys', 'dev', 'run'):

From 9441221150aa0aa085f5c0b882828bb4a6c36ca9 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 8 Sep 2023 11:30:57 -0400
Subject: [PATCH 08/41] Have cooltera plugin adapt

As new sensors appear, be more adaptive
to continue tracking existing sensors.
---
 .../confluent/plugins/hardwaremanagement/cooltera.py        | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/confluent_server/confluent/plugins/hardwaremanagement/cooltera.py b/confluent_server/confluent/plugins/hardwaremanagement/cooltera.py
index 1b89271e..c6e4b070 100644
--- a/confluent_server/confluent/plugins/hardwaremanagement/cooltera.py
+++ b/confluent_server/confluent/plugins/hardwaremanagement/cooltera.py
@@ -210,10 +210,12 @@ def xml2stateinfo(statdata):
     stateinfo = []
     sensornames = sorted([x.tag for x in statdata])
     themodel = None
-    for model in sensorsbymodel:
-        if sensorsbymodel[model] == sensornames:
+    for model in sorted(sensorsbymodel):
+        if all([x in sensornames for x in sensorsbymodel[model]]):
             themodel = model
             break
+    else:
+        print(repr(sensornames))
     thesensors = _thesensors[themodel]
 #['mode', 't1', 't2a', 't2b', 't2c', 't2', 't5', 't3', 't4', 'dw', 't3', 'rh', 'setpoint', 'secflow', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'ps5c', 'sdp', 'valve', 'valve2', 'pumpspeed1', 'pumpspeed2', 'pumpspeed3', 'alarms', 'dt', 'p3state', 'duty']
     for tagname in thesensors:

From 203dabfb0bf4ccd9836ebba153c2a30e6c2e54d1 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 8 Sep 2023 16:56:58 -0400
Subject: [PATCH 09/41] Place nameservers on every NIC in netplan

netplan, like others, makes the questionable choice to designate DNS
as a NIC specific setting, despite not mapping well
to a NIC.

Since we model DNS like NTP, a global, just repeat the DNS config for
every interface.  This redundancy is fine in testing multiple
interfaces.
---
 .../common/profile/scripts/confignet          | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet
index d01a9e45..dec1808d 100644
--- a/confluent_osdeploy/common/profile/scripts/confignet
+++ b/confluent_osdeploy/common/profile/scripts/confignet
@@ -112,9 +112,10 @@ def get_interface_name(iname, settings):
     return None
 
 class NetplanManager(object):
-    def __init__(self):
+    def __init__(self, deploycfg):
         self.cfgbydev = {}
         self.read_connections()
+        self.deploycfg = deploycfg
 
     def read_connections(self):
         for plan in glob.glob('/etc/netplan/*.y*ml'):
@@ -174,6 +175,19 @@ class NetplanManager(object):
                     else:
                         needcfgwrite = True
                         cfgroutes.append({'via': gwaddr, 'to': 'default'})
+            dnsips = self.deploycfg.get('nameservers', [])
+            dnsdomain = self.deploycfg.get('dnsdomain', '')
+            if dnsips:
+                currdnsips = self.getcfgarrpath([devname, 'nameservers', 'addresses'])
+                for dnsip in dnsips:
+                    if dnsip not in currdnsips:
+                        needcfgwrite = True
+                        currdnsips.append(dnsip)
+            if dnsdomain:
+                currdnsdomain = self.getcfgarrpath([devname, 'nameservers', 'search'])
+                if dnsdomain not in currdnsdomain:
+                    needcfgwrite = True
+                    currdnsdomain.append(dnsdomain)
             if needcfgwrite:
                 needcfgapply = True
                 newcfg = {'network': {'version': 2, 'ethernets': {devname: self.cfgbydev[devname]}}}
@@ -403,6 +417,7 @@ if __name__ == '__main__':
     myaddrs = apiclient.get_my_addresses()
     srvs, _ = apiclient.scan_confluents()
     doneidxs = set([])
+    dc = None
     for srv in srvs:
         try:
             s = socket.create_connection((srv, 443))
@@ -422,6 +437,9 @@ if __name__ == '__main__':
             continue
         status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
         nc = json.loads(nc)
+        if not dc:
+            status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2')
+            dc = json.loads(dc)
         iname = get_interface_name(idxmap[curridx], nc.get('default', {}))
         if iname:
             for iname in iname.split(','):
@@ -448,7 +466,7 @@ if __name__ == '__main__':
             del netname_to_interfaces['default']
     rm_tmp_llas(tmpllas)
     if os.path.exists('/usr/sbin/netplan'):
-        nm = NetplanManager()
+        nm = NetplanManager(dc)
     if os.path.exists('/usr/bin/nmcli'):
         nm = NetworkManager(devtypes)
     elif os.path.exists('/usr/sbin/wicked'):

From f82829aa0ce253f89ef7f81f0d5d543749849011 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 8 Sep 2023 17:10:27 -0400
Subject: [PATCH 10/41] Add dependency checking to imgutil capture

This will more quickly indicate problems in a profile trying to capture.

First iteration will address Ubuntu.
---
 imgutil/imgutil | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/imgutil/imgutil b/imgutil/imgutil
index 39c40024..8f805ac5 100644
--- a/imgutil/imgutil
+++ b/imgutil/imgutil
@@ -196,7 +196,13 @@ def capture_remote(args):
     finfo = subprocess.check_output(['ssh', targ, 'python3', '/run/imgutil/capenv/imgutil', 'getfingerprint']).decode('utf8')
     finfo = json.loads(finfo)
     if finfo['oscategory'] not in ('el8', 'el9', 'ubuntu20.04', 'ubuntu22.04'):
-        raise Exception('Not yet supported for capture: ' + repr(finfo))
+        sys.stderr.write('Not yet supported for capture: ' + repr(finfo) + '\n')
+        sys.exit(1)
+    unmet = finfo.get('unmetprereqs', [])
+    if unmet:
+        for cmd in unmet:
+            sys.stderr.write(cmd + '\n')
+        sys.exit(1)
     oscat = finfo['oscategory']
     subprocess.check_call(['ssh', '-o', 'LogLevel=QUIET', '-t', targ, 'python3', '/run/imgutil/capenv/imgutil', 'capturelocal'])
     utillib = __file__.replace('bin/imgutil', 'lib/imgutil')        
@@ -442,6 +448,7 @@ class OsHandler(object):
         self.arch = arch
         self.sourcepath = None
         self.osname = '{}-{}-{}'.format(name, version, arch)
+        self.captureprereqs = []
         try:
             pkglist = args.packagelist
         except AttributeError:
@@ -474,7 +481,7 @@ class OsHandler(object):
             if not isinstance(odata[idx], str):
                 odata[idx] = odata[idx].decode('utf8')
         info = {'oscategory': odata[0],
-                'version': odata[1], 'arch': odata[2], 'name': odata[3]}
+                'version': odata[1], 'arch': odata[2], 'name': odata[3], 'unmetprereqs': self.captureprereqs}
         return json.dumps(info)
 
     def prep_root_premount(self, args):
@@ -594,12 +601,21 @@ class SuseHandler(OsHandler):
 
 
 class DebHandler(OsHandler):
-    def __init__(self, name, version, arch, args, codename):
+    def __init__(self, name, version, arch, args, codename, hostpath):
         self.includepkgs = []
         self.targpath = None
         self.codename = codename
         self.oscategory = name + version
         super().__init__(name, version, arch, args)
+        needpkgs = []
+        if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')):
+            needpkgs.append('tpm2-tools')
+        lfuses = glob.glob('/lib/*/libfuse.so.2')
+        if not lfuses:
+            needpkgs.append('libfuse2')
+        if needpkgs:
+            needapt = 'Missing packages needed in target for capture, to add required packages: apt install ' + ' '.join(needpkgs)
+            self.captureprereqs.append(needapt)
 
     def add_pkglists(self):
         self.includepkgs.extend(self.list_packages())
@@ -1082,7 +1098,7 @@ def fingerprint_host_deb(args, hostpath='/'):
     except IOError:
         pass
     if osname:
-        return DebHandler(osname, vers, os.uname().machine, args, codename)
+        return DebHandler(osname, vers, os.uname().machine, args, codename, hostpath)
 
 
 def fingerprint_host_suse(args, hostpath='/'):

From d7190c893ff1c2211dcb4594bba4c873afb94002 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 11 Sep 2023 13:11:17 -0400
Subject: [PATCH 11/41] Correct networkmanager entries for boot over infiniband

When booting and infiniband comes up as the 'get started' nic,
make the entry use correct type so as to avoid
confusion later, particularly when confignet comes along to fixup
the configuration.
---
 .../lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 12 ++++++++----
 .../lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 11 +++++++----
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh
index 31233c82..b2881e0b 100644
--- a/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh
+++ b/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh
@@ -189,8 +189,15 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
 EOC
 echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
 echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link |grep -A2 ${ifname}|tail -n 1|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+	linktype="infiniband"
+else
+	linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
+
 cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
 autoconnect-retries=1
 EOC
 echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -199,9 +206,6 @@ multi-connect=1
 permissions=
 wait-device-timeout=60000
 
-[ethernet]
-mac-address-blacklist=
-
 EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
diff --git a/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh
index 686e14ce..4fca92cf 100644
--- a/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh
+++ b/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh
@@ -154,8 +154,14 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
 EOC
 echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
 echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link |grep -A2 ${ifname}|tail -n 1|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+        linktype="infiniband"
+else
+        linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
 cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
 autoconnect-retries=1
 EOC
 echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -164,9 +170,6 @@ multi-connect=1
 permissions=
 wait-device-timeout=60000
 
-[ethernet]
-mac-address-blacklist=
-
 EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')

From b77d8b1f210519c5361d5475d190941d9ac8dc11 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 12 Sep 2023 10:45:15 -0400
Subject: [PATCH 12/41] Make yaml import conditional

Only Ubuntu requires it, but tends to have it.

Other distributions do not tend to have it.
---
 imgutil/imgutil | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/imgutil/imgutil b/imgutil/imgutil
index 8f805ac5..4e46ce0f 100644
--- a/imgutil/imgutil
+++ b/imgutil/imgutil
@@ -23,7 +23,10 @@ import subprocess
 import sys
 import tempfile
 import time
-import yaml
+try:
+    import yaml
+except ImportError:
+    pass
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):

From 00eb9e3c9d949db56e82e80820469cb373816c97 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 12 Sep 2023 16:49:15 -0400
Subject: [PATCH 13/41] Fix full_net_config with missing address info

---
 confluent_server/confluent/netutil.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py
index 66ed2169..37e8d198 100644
--- a/confluent_server/confluent/netutil.py
+++ b/confluent_server/confluent/netutil.py
@@ -366,7 +366,7 @@ def get_full_net_config(configmanager, node, serverip=None):
         retattrs['default'] = defaultnic
         add_netmask(retattrs['default'])
         ipv4addr = defaultnic.get('ipv4_address', None)
-        if '/' in ipv4addr:
+        if ipv4addr and '/' in ipv4addr:
             ipv4bytes = socket.inet_pton(socket.AF_INET, ipv4addr.split('/')[0])
             for addr in nm.myaddrs:
                 if addr[0] != socket.AF_INET:
@@ -374,7 +374,7 @@ def get_full_net_config(configmanager, node, serverip=None):
                 if ipn_on_same_subnet(addr[0], addr[1], ipv4bytes, addr[2]):
                     defaultnic['current_nic'] = True
         ipv6addr = defaultnic.get('ipv6_address', None)
-        if '/' in ipv6addr:
+        if ipv6addr and '/' in ipv6addr:
             ipv6bytes = socket.inet_pton(socket.AF_INET6, ipv6addr.split('/')[0])
             for addr in nm.myaddrs:
                 if addr[0] != socket.AF_INET6:

From e6b0b235e959a32a243a8d8a1c330fbd9cc370c7 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 12 Sep 2023 16:53:32 -0400
Subject: [PATCH 14/41] Fix image2disk for nvme clone targets

---
 .../profiles/default/scripts/image2disk.py           | 12 +++++++++---
 .../profiles/default/scripts/image2disk.py           |  2 ++
 .../profiles/default/scripts/image2disk.py           |  2 ++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
index 08d48a9c..8548cebd 100644
--- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
+++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
@@ -60,10 +60,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True)::
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -173,6 +176,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -231,7 +236,8 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    # do this safer, unit s print might bomb
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py
index 92facb4b..aaaca9d4 100644
--- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py
+++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py
@@ -208,6 +208,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if 'nvme' in targblock and targblock[-1] == 'p':
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py
index 92facb4b..7b312a93 100644
--- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py
+++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py
@@ -208,6 +208,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])

From 50d5cead06e51742915ac32057e97a1d0fe498f2 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 12 Sep 2023 16:55:19 -0400
Subject: [PATCH 15/41] Add prereq checks to EL cloning

---
 imgutil/imgutil | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/imgutil/imgutil b/imgutil/imgutil
index 4e46ce0f..5596ef6a 100644
--- a/imgutil/imgutil
+++ b/imgutil/imgutil
@@ -613,7 +613,7 @@ class DebHandler(OsHandler):
         needpkgs = []
         if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')):
             needpkgs.append('tpm2-tools')
-        lfuses = glob.glob('/lib/*/libfuse.so.2')
+        lfuses = glob.glob(os.path.join(hostpath, '/lib/*/libfuse.so.2')
         if not lfuses:
             needpkgs.append('libfuse2')
         if needpkgs:
@@ -650,6 +650,21 @@ class ElHandler(OsHandler):
         self.oscategory = 'el{0}'.format(version.split('.')[0])
         self.yumargs = []
         super().__init__(name, version, arch, args)
+        needpkgs = []
+        if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')):
+            needpkgs.append('tpm2-tools')
+        lfuses = glob.glob(os.path.join(hostpath, '/usr/lib64/libfuse.so.2')
+        if not lfuses:
+            needpkgs.append('fuse-libs')
+        if not os.path.exists(os.path.join(hostpath, '/usr/bin/ipcalc')):
+            needpkgs.append('ipcalc')
+        if not os.path.exists(os.path.join(hostpath, 'usr/sbin/dhclient')):
+            needpkgs.append('dhcp-client')
+        if not os.path.exists(os.path.join(hostpath, 'usr/sbin/mount.nfs')):
+            needpkgs.append('nfs-utils')
+        if needpkgs:
+            needapt = 'Missing packages needed in target for capture, to add required packages: dnf install ' + ' '.join(needpkgs)
+            self.captureprereqs.append(needapt)
 
     def add_pkglists(self):
         self.yumargs.extend(self.list_packages())

From b75979f3ec4eb64ec9cf3f5fa302ab812c0bd7c1 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 12 Sep 2023 16:59:53 -0400
Subject: [PATCH 16/41] Insulate confluent from fatal errors from discovery
 subscription errors

---
 confluent_server/confluent/discovery/core.py | 7 +++++--
 confluent_server/confluent/main.py           | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py
index 13b3aac0..5952529f 100644
--- a/confluent_server/confluent/discovery/core.py
+++ b/confluent_server/confluent/discovery/core.py
@@ -1597,7 +1597,10 @@ def remotescan():
     mycfm = cfm.ConfigManager(None)
     myname = collective.get_myname()
     for remagent in get_subscriptions():
-        affluent.renotify_me(remagent, mycfm, myname)
+        try:
+            affluent.renotify_me(remagent, mycfm, myname)
+        except Exception as e:
+            log.log({'error': 'Unexpected problem asking {} for discovery notifications'.format(remagent)})
 
 
 def blocking_scan():
@@ -1637,7 +1640,7 @@ def start_autosense():
     autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
     #autosensors.add(eventlet.spawn(mdns.snoop, safe_detected, mdns))
     autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe, get_node_guess_by_uuid))
-    remotescan()
+    eventlet.spawn(remotescan)
 
 
 nodes_by_fprint = {}
diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py
index f59bceb7..b49d8f56 100644
--- a/confluent_server/confluent/main.py
+++ b/confluent_server/confluent/main.py
@@ -326,7 +326,7 @@ def run(args):
             break
         except Exception:
             eventlet.sleep(0.5)
-    disco.start_detection()
+    eventlet.spawn_n(disco.start_detection)
     eventlet.sleep(1)
     consoleserver.start_console_sessions()
     while 1:

From 74c6848a0bf8addf48f6c60aed9d1804a077be37 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 13 Sep 2023 09:59:03 -0400
Subject: [PATCH 17/41] Avoid redundant setting of known data

Setting attributes can be a touch expensive, since
there's a high risk
of this being old news,
check that discovery hasn't already set values
before trying to set them again.
---
 confluent_server/confluent/discovery/core.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py
index 5952529f..a81d79e7 100644
--- a/confluent_server/confluent/discovery/core.py
+++ b/confluent_server/confluent/discovery/core.py
@@ -1429,7 +1429,13 @@ def discover_node(cfg, handler, info, nodename, manual):
                 newnodeattribs['pubkeys.tls_hardwaremanager'] = \
                     util.get_fingerprint(handler.https_cert, 'sha256')
             if newnodeattribs:
-                cfg.set_node_attributes({nodename: newnodeattribs})
+                currattrs = cfg.get_node_attributes(nodename, newnodeattribs)
+                for checkattr in newnodeattribs:
+                    checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
+                    if checkval != newnodeattribs[checkattr]:
+                        break
+                else:
+                    cfg.set_node_attributes({nodename: newnodeattribs})
             log.log({'info': 'Discovered {0} ({1})'.format(nodename,
                                                           handler.devname)})
             if nodeconfig:
@@ -1508,7 +1514,13 @@ def do_pxe_discovery(cfg, handler, info, manual, nodename, policies):
                 if info['hwaddr'] != oldhwaddr:
                     attribs[newattrname] = info['hwaddr']
         if attribs:
-            cfg.set_node_attributes({nodename: attribs})
+            currattrs = cfg.get_node_attributes(nodename, attribs)
+            for checkattr in attribs:
+                checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
+                if checkval != attribs[checkattr]:
+                    break
+            else:
+                cfg.set_node_attributes({nodename: attribs})
     if info['uuid'] in known_pxe_uuids:
         return True
     if uuid_is_valid(info['uuid']):

From 97ee8e2372afe922ea27d45bb3c54e158a5a9d16 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 13 Sep 2023 10:50:21 -0400
Subject: [PATCH 18/41] Cerrect the logic of duplicate discovery protection

---
 confluent_server/confluent/discovery/core.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py
index a81d79e7..e18aae0e 100644
--- a/confluent_server/confluent/discovery/core.py
+++ b/confluent_server/confluent/discovery/core.py
@@ -1433,9 +1433,8 @@ def discover_node(cfg, handler, info, nodename, manual):
                 for checkattr in newnodeattribs:
                     checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
                     if checkval != newnodeattribs[checkattr]:
+                        cfg.set_node_attributes({nodename: newnodeattribs})
                         break
-                else:
-                    cfg.set_node_attributes({nodename: newnodeattribs})
             log.log({'info': 'Discovered {0} ({1})'.format(nodename,
                                                           handler.devname)})
             if nodeconfig:
@@ -1518,9 +1517,8 @@ def do_pxe_discovery(cfg, handler, info, manual, nodename, policies):
             for checkattr in attribs:
                 checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
                 if checkval != attribs[checkattr]:
+                    cfg.set_node_attributes({nodename: attribs})
                     break
-            else:
-                cfg.set_node_attributes({nodename: attribs})
     if info['uuid'] in known_pxe_uuids:
         return True
     if uuid_is_valid(info['uuid']):

From df47c6d0fde8cef2abdaa4d8d270012795723d1c Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 13 Sep 2023 17:03:05 -0400
Subject: [PATCH 19/41] Disable attribute notify during json restore

This is guaranteed to be a lot of churn very quickly, disable it for
now.
---
 .../confluent/config/configmanager.py          | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 5a392edd..50d09b1c 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -119,6 +119,7 @@ _cfgstore = None
 _pendingchangesets = {}
 _txcount = 0
 _hasquorum = True
+_ready = False
 
 _attraliases = {
     'bmc': 'hardwaremanagement.manager',
@@ -1273,6 +1274,7 @@ class ConfigManager(object):
     def __init__(self, tenant, decrypt=False, username=None):
         self.clientfiles = {}
         global _cfgstore
+        self.inrestore = False
         with _initlock:
             if _cfgstore is None:
                 init()
@@ -2089,6 +2091,10 @@ class ConfigManager(object):
     def _notif_attribwatchers(self, nodeattrs):
         if self.tenant not in self._attribwatchers:
             return
+        if self.inrestore:
+            # Do not stir up attribute watchers during a collective join or DB restore,
+            # it's too hectic of a time to react
+            return
         notifdata = {}
         attribwatchers = self._attribwatchers[self.tenant]
         for node in nodeattrs:
@@ -2471,6 +2477,13 @@ class ConfigManager(object):
         #TODO: wait for synchronization to suceed/fail??)
 
     def _load_from_json(self, jsondata, sync=True):
+        self.inrestore = True
+        try:
+            _load_from_json_backend(self, jsondata, sync=True)
+        finally:
+            self.inrestore = False
+
+    def _load_from_json_backend(self, jsondata, sync=True):
         """Load fresh configuration data from jsondata
 
         :param jsondata: String of jsondata
@@ -2939,9 +2952,9 @@ def get_globals():
         bkupglobals[globvar] = _cfgstore['globals'][globvar]
     return bkupglobals
 
-
 def init(stateless=False):
     global _cfgstore
+    global _ready
     if stateless:
         _cfgstore = {}
         return
@@ -2949,6 +2962,9 @@ def init(stateless=False):
         ConfigManager._read_from_path()
     except IOError:
         _cfgstore = {}
+    members = list(list_collective())
+    if len(members) < 2:
+        _ready = True
 
 
 if __name__ == '__main__':

From 20f02b5ef732d689b350c82a2316d4da26bf1962 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 10:07:14 -0400
Subject: [PATCH 20/41] Avoid searching switches for foreign nodes

Consult collective.manager
to decide to skip
consideration of a node, if
that node shouldn't be managed anyway.

This should avoid "cross-island" behavior for such
environments.
---
 confluent_server/confluent/collective/manager.py |  6 ++++++
 confluent_server/confluent/networking/macmap.py  | 16 ++++++++++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/confluent_server/confluent/collective/manager.py b/confluent_server/confluent/collective/manager.py
index 8668bc65..a4059bc5 100644
--- a/confluent_server/confluent/collective/manager.py
+++ b/confluent_server/confluent/collective/manager.py
@@ -259,6 +259,9 @@ def get_myname():
         mycachedname[1] = time.time()
         return myname
 
+def in_collective():
+    return bool(list(cfm.list_collective()))
+
 def handle_connection(connection, cert, request, local=False):
     global currentleader
     global retrythread
@@ -815,6 +818,9 @@ def start_collective():
     global follower
     global retrythread
     global initting
+    if initting:
+        # do not nest startup if startup backs up loops of starting collective
+        return
     initting = True
     retrythread = None
     try:
diff --git a/confluent_server/confluent/networking/macmap.py b/confluent_server/confluent/networking/macmap.py
index d1377dbf..cf6012c5 100644
--- a/confluent_server/confluent/networking/macmap.py
+++ b/confluent_server/confluent/networking/macmap.py
@@ -49,10 +49,11 @@ import eventlet.green.select as select
 
 import eventlet.green.socket as socket
 
-
+import confluent.collective.manager as collective
 import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages as msg
+import confluent.noderange as noderange
 import confluent.util as util
 from eventlet.greenpool import GreenPool
 import eventlet.green.subprocess as subprocess
@@ -502,10 +503,21 @@ def _full_updatemacmap(configmanager):
                 'Network topology not available to tenants')
         # here's a list of switches... need to add nodes that are switches
         nodelocations = configmanager.get_node_attributes(
-            configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
+            configmanager.list_nodes(), ('type', 'collective.managercandidates', 'net*.switch', 'net*.switchport'))
         switches = set([])
+        incollective = collective.in_collective()
+        if incollective:
+            mycollectivename = collective.get_myname()
         for node in nodelocations:
             cfg = nodelocations[node]
+            if incollective:
+                candmgrs = cfg.get('collective.managercandidates', {}).get('value', None)
+                if candmgrs:
+                    candmgrs = noderange.NodeRange(candmgrs, configmanager).nodes
+                    if mycollectivename not in candmgrs:
+                        # do not think about trying to find nodes that we aren't possibly
+                        # supposed to be a manager for in a collective
+                        continue
             if cfg.get('type', {}).get('value', None) == 'switch':
                 switches.add(node)
             for attr in cfg:

From 533244458d56b4b1ee83cf05d8333babd99f6b17 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 10:37:51 -0400
Subject: [PATCH 21/41] Do not count as 'initting' until collective starts.

---
 confluent_server/confluent/collective/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_server/confluent/collective/manager.py b/confluent_server/confluent/collective/manager.py
index a4059bc5..80c241b8 100644
--- a/confluent_server/confluent/collective/manager.py
+++ b/confluent_server/confluent/collective/manager.py
@@ -43,7 +43,7 @@ currentleader = None
 follower = None
 retrythread = None
 failovercheck = None
-initting = True
+initting = False
 reassimilate = None
 
 class ContextBool(object):

From 4952e87309779b9161f7bdf2004733c7d1e2421a Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 10:52:13 -0400
Subject: [PATCH 22/41] Undo collective manager changes

Abort attempt to avoid duplicate startups, it was incorrect.
---
 confluent_server/confluent/collective/manager.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/confluent_server/confluent/collective/manager.py b/confluent_server/confluent/collective/manager.py
index 80c241b8..8d98caa0 100644
--- a/confluent_server/confluent/collective/manager.py
+++ b/confluent_server/confluent/collective/manager.py
@@ -43,7 +43,7 @@ currentleader = None
 follower = None
 retrythread = None
 failovercheck = None
-initting = False
+initting = True
 reassimilate = None
 
 class ContextBool(object):
@@ -818,9 +818,6 @@ def start_collective():
     global follower
     global retrythread
     global initting
-    if initting:
-        # do not nest startup if startup backs up loops of starting collective
-        return
     initting = True
     retrythread = None
     try:

From c0629fcce57538d0f83e257d5bc99157eb25e751 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 11:41:12 -0400
Subject: [PATCH 23/41] Fix invocation of json restore change

---
 confluent_server/confluent/config/configmanager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 50d09b1c..1ae5e579 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -2479,7 +2479,7 @@ class ConfigManager(object):
     def _load_from_json(self, jsondata, sync=True):
         self.inrestore = True
         try:
-            _load_from_json_backend(self, jsondata, sync=True)
+            self._load_from_json_backend(jsondata, sync=True)
         finally:
             self.inrestore = False
 

From f2f25fe912ec6c606be40249a55be792495ab5be Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 15:25:26 -0400
Subject: [PATCH 24/41] Implement ready tracking

When going through the
dramatic scenario of initializing collective,
take _ready down
so that other code can
pause operation appropriately.
---
 confluent_server/confluent/config/configmanager.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 1ae5e579..54888dd1 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -831,6 +831,9 @@ _oldcfgstore = None
 _oldtxcount = 0
 
 
+def config_is_ready():
+    return _ready
+
 def rollback_clear():
     global _cfgstore
     global _txcount
@@ -848,6 +851,8 @@ def clear_configuration():
     global _txcount
     global _oldcfgstore
     global _oldtxcount
+    global _ready
+    _ready = False
     stop_leading()
     stop_following()
     _oldcfgstore = _cfgstore
@@ -858,6 +863,7 @@ def clear_configuration():
 def commit_clear():
     global _oldtxcount
     global _oldcfgstore
+    global _ready
     # first, copy over old non-key globals, as those are
     # currently defined as local to each collective member
     # currently just 'autosense' which is intended to be active
@@ -877,6 +883,7 @@ def commit_clear():
                 pass
     ConfigManager.wait_for_sync(True)
     ConfigManager._bg_sync_to_file()
+    ready = True
 
 cfgleader = None
 

From 94b85597776aa4a4eea8f41c8b8f89898edca730 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 15:28:16 -0400
Subject: [PATCH 25/41] Declare ready on becoming leader

Provide for leader scenario to correctly
flag configmanager as ready.
---
 confluent_server/confluent/collective/manager.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/confluent_server/confluent/collective/manager.py b/confluent_server/confluent/collective/manager.py
index 8d98caa0..2519cc39 100644
--- a/confluent_server/confluent/collective/manager.py
+++ b/confluent_server/confluent/collective/manager.py
@@ -716,6 +716,7 @@ def become_leader(connection):
     if reassimilate is not None:
         reassimilate.kill()
     reassimilate = eventlet.spawn(reassimilate_missing)
+    cfm._ready = True
     if _assimilate_missing(skipaddr):
         schedule_rebalance()
 

From d4c535d038c244f753697617cf32f334ec0de79e Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 15:32:33 -0400
Subject: [PATCH 26/41] Halt autonomous discovery handling while configmanager
 is down

This avoids triggering a potential large amount of churn on transiently
"unknown" systems
that are actually discovered.
---
 confluent_server/confluent/discovery/core.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py
index e18aae0e..dfb50b9f 100644
--- a/confluent_server/confluent/discovery/core.py
+++ b/confluent_server/confluent/discovery/core.py
@@ -648,6 +648,8 @@ def detected_models():
 
 
 def _recheck_nodes(nodeattribs, configmanager):
+    if not cfm.config_is_ready():
+        return
     if rechecklock.locked():
         # if already in progress, don't run again
         # it may make sense to schedule a repeat, but will try the easier and less redundant way first
@@ -766,6 +768,9 @@ def eval_detected(info):
 def detected(info):
     global rechecker
     global rechecktime
+    if not cfm.config_is_ready():
+        # drop processing of discovery data while configmanager is 'down'
+        return
     # later, manual and CMM discovery may act on SN and/or UUID
     for service in info['services']:
         if service in nodehandlers:

From aa5de3c6a3128d11737e44028026373fafd1b45d Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 15:48:37 -0400
Subject: [PATCH 27/41] Suspend handling of new socket connections while
 configmanager down

---
 confluent_server/confluent/sockapi.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py
index e90176ce..2d4db15b 100644
--- a/confluent_server/confluent/sockapi.py
+++ b/confluent_server/confluent/sockapi.py
@@ -141,6 +141,8 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
             if 'collective' in response:
                 return collective.handle_connection(connection, cert,
                                             response['collective'])
+            while not configmanager.config_is_ready():
+                eventlet.sleep(1)
             if 'dispatch' in response:
                 dreq = tlvdata.recvall(connection, response['dispatch']['length'])
                 return pluginapi.handle_dispatch(connection, cert, dreq,

From 37b75ba777090b091e632de90a375dd79920f93e Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Fri, 15 Sep 2023 15:54:35 -0400
Subject: [PATCH 28/41] Correct variable name on commit clear

---
 confluent_server/confluent/config/configmanager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 54888dd1..9419e7fe 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -883,7 +883,7 @@ def commit_clear():
                 pass
     ConfigManager.wait_for_sync(True)
     ConfigManager._bg_sync_to_file()
-    ready = True
+    _ready = True
 
 cfgleader = None
 

From 83e3627b47571185ac75f47ddcbad4dcc62da97c Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 18 Sep 2023 10:19:50 -0400
Subject: [PATCH 29/41] Add pre.d to ubuntu 22 diskful

---
 .../profiles/default/scripts/pre.sh           | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh
index ddfe598b..2f671d38 100755
--- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh
+++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh
@@ -1,5 +1,16 @@
 #!/bin/bash
 deploycfg=/custom-installation/confluent/confluent.deploycfg
+mkdir -p /var/log/confluent
+mkdir -p /opt/confluent/bin
+mkdir -p /etc/confluent
+cp /custom-installation/confluent/confluent.info /custom-installation/confluent/confluent.apikey /etc/confluent/
+cat /custom-installation/tls/*.pem >> /etc/confluent/ca.pem
+cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin
+cp $deploycfg /etc/confluent/
+(
+exec >> /var/log/confluent/confluent-pre.log
+exec 2>> /var/log/confluent/confluent-pre.log
+chmod 600 /var/log/confluent/confluent-pre.log
 
 cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
 if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
@@ -23,7 +34,16 @@ echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
 systemctl restart sshd
+mkdir -p /etc/confluent
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
+run_remote_parts pre.d
 curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
-python3 /custom-installation/getinstalldisk
+if [ ! -e /tmp/installdisk ]; then
+    curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
+    python3 /custom-installation/getinstalldisk
+fi
 sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml
+) &
+tail --pid $! -n 0 -F /var/log/confluent/confluent-pre.log > /dev/console
+

From a01b7c6503064606e767749b2c40030a7fce743a Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 18 Sep 2023 10:30:52 -0400
Subject: [PATCH 30/41] Revamp and add missing bits to scripted ubuntu install

---
 .../default/ansible/firstboot.d/README.txt    | 29 +++++++++++++++++++
 .../default/ansible/post.d/README.txt         | 29 +++++++++++++++++++
 .../profiles/default/scripts/firstboot.sh     |  7 ++++-
 .../profiles/default/scripts/post.sh          | 15 ++++++----
 .../profiles/default/scripts/pre.d/.gitignore |  0
 5 files changed, 74 insertions(+), 6 deletions(-)
 create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/ansible/firstboot.d/README.txt
 create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/ansible/post.d/README.txt
 create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.d/.gitignore

diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/ansible/firstboot.d/README.txt b/confluent_osdeploy/ubuntu22.04/profiles/default/ansible/firstboot.d/README.txt
new file mode 100644
index 00000000..ad6fc712
--- /dev/null
+++ b/confluent_osdeploy/ubuntu22.04/profiles/default/ansible/firstboot.d/README.txt
@@ -0,0 +1,29 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+
diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/ansible/post.d/README.txt b/confluent_osdeploy/ubuntu22.04/profiles/default/ansible/post.d/README.txt
new file mode 100644
index 00000000..ad6fc712
--- /dev/null
+++ b/confluent_osdeploy/ubuntu22.04/profiles/default/ansible/post.d/README.txt
@@ -0,0 +1,29 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+
diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh
index d14269cf..22848fe7 100755
--- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh
+++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh
@@ -2,7 +2,10 @@
 echo "Confluent first boot is running"
 HOME=$(getent passwd $(whoami)|cut -d: -f 6)
 export HOME
-seems a potentially relevant thing to put i... by Jarrod Johnson
+(
+exec >> /target/var/log/confluent/confluent-firstboot.log
+exec 2>> /target/var/log/confluent/confluent-firstboot.log
+chmod 600 /target/var/log/confluent/confluent-firstboot.log
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
@@ -22,3 +25,5 @@ source /etc/confluent/functions
 run_remote_parts firstboot.d
 run_remote_config firstboot.d
 curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus
+) &
+tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console
diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
index 7b970285..5f530262 100755
--- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
+++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
@@ -8,7 +8,6 @@ chmod go-rwx /etc/confluent/*
 for i in /custom-installation/ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts
 done
-
 cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/
 cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/
 sshconf=/target/etc/ssh/ssh_config
@@ -19,10 +18,15 @@ echo 'Host *' >> $sshconf
 echo '    HostbasedAuthentication yes' >> $sshconf
 echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
-
+cp /etc/confluent/functions /target/etc/confluent/functions
+source /etc/confluent/functions
+mkdir -p /target/var/log/confluent
+cp /var/log/confluent/* /target/var/log/confluent/
+(
+exec >> /target/var/log/confluent/confluent-post.log
+exec 2>> /target/var/log/confluent/confluent-post.log
+chmod 600 /target/var/log/confluent/confluent-post.log
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /target/etc/confluent/functions
-source /target/etc/confluent/functions
 chmod +x /target/etc/confluent/firstboot.sh
 cp /tmp/allnodes /target/root/.shosts
 cp /tmp/allnodes /target/etc/ssh/shosts.equiv
@@ -85,4 +89,5 @@ source /target/etc/confluent/functions
 run_remote_config post
 
 umount /target/sys /target/dev /target/proc
-
+) &
+tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console
diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.d/.gitignore b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.d/.gitignore
new file mode 100644
index 00000000..e69de29b

From 3e8c6d1ea6ccd33f497a2b5057e512376338a12c Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 18 Sep 2023 11:04:50 -0400
Subject: [PATCH 31/41] Correct syntax issue in el7 image2disk

---
 .../el7-diskless/profiles/default/scripts/image2disk.py         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
index 8548cebd..768aa57d 100644
--- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
+++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py
@@ -60,7 +60,7 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command, check=True)::
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
         if check:

From 0a527f5f399408536f18cfd24f91056e1de77930 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 18 Sep 2023 11:38:41 -0400
Subject: [PATCH 32/41] Add environment to firstboot ubuntu

---
 .../ubuntu22.04/profiles/default/scripts/firstboot.sh          | 3 ++-
 .../ubuntu22.04/profiles/default/scripts/post.sh               | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh
index 22848fe7..c0ba44ab 100755
--- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh
+++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh
@@ -21,7 +21,8 @@ done
 hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
 touch /etc/cloud/cloud-init.disabled
 source /etc/confluent/functions
-
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export confluent_mgr confluent_profile
 run_remote_parts firstboot.d
 run_remote_config firstboot.d
 curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus
diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
index 5f530262..773bf8ad 100755
--- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
+++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh
@@ -87,6 +87,7 @@ chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d
 source /target/etc/confluent/functions
 
 run_remote_config post
+python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
 
 umount /target/sys /target/dev /target/proc
 ) &

From 8f80add0f1645194c2b72b5f1cee5c660f196f79 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 18 Sep 2023 15:19:10 -0400
Subject: [PATCH 33/41] Enhance debian packaging for confluent

---
 confluent_server/builddeb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/confluent_server/builddeb b/confluent_server/builddeb
index fe2bdf96..eb17c25f 100755
--- a/confluent_server/builddeb
+++ b/confluent_server/builddeb
@@ -36,7 +36,7 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
     if grep wheezy /etc/os-release; then
         sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control
     else
-        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko/' debian/control
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd/' debian/control
     fi
     if grep wheezy /etc/os-release; then
         echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
@@ -49,6 +49,13 @@ if ! grep wheezy /etc/os-release; then
 fi
 head -n -1 debian/control > debian/control1
 mv debian/control1 debian/control
+cat > debian/postinst << EOF
+if ! getent passwd confluent > /dev/null; then 
+    useradd -r affluent -d /var/lib/affluent -s /bin/nologin
+    mkdir /etc/confluent
+    chown confluent /etc/confluent
+fi
+EOF
 echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules
 #echo 'Provides: python-'$DPKGNAME >> debian/control
 #echo 'Conflicts: python-'$DPKGNAME >> debian/control

From 47fc233cce8240a20d8ae5eb43914c0d827c91e0 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 18 Sep 2023 15:48:38 -0400
Subject: [PATCH 34/41] Fix debian packaging for confluent

---
 confluent_server/builddeb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/confluent_server/builddeb b/confluent_server/builddeb
index eb17c25f..f71bfce4 100755
--- a/confluent_server/builddeb
+++ b/confluent_server/builddeb
@@ -51,8 +51,8 @@ head -n -1 debian/control > debian/control1
 mv debian/control1 debian/control
 cat > debian/postinst << EOF
 if ! getent passwd confluent > /dev/null; then 
-    useradd -r affluent -d /var/lib/affluent -s /bin/nologin
-    mkdir /etc/confluent
+    useradd -r confluent -d /var/lib/confluent -s /usr/sbin/nologin
+    mkdir -p /etc/confluent
     chown confluent /etc/confluent
 fi
 EOF

From d613d0f546641ef722c06a146b9739a39035332a Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Mon, 18 Sep 2023 16:03:48 -0400
Subject: [PATCH 35/41] Add openbmc plugin for console

---
 .../confluent/config/attributes.py            |   2 +-
 .../confluent/plugins/console/openbmc.py      | 160 ++++++++++++++++++
 2 files changed, 161 insertions(+), 1 deletion(-)
 create mode 100644 confluent_server/confluent/plugins/console/openbmc.py

diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py
index 50f5492b..101ee03d 100644
--- a/confluent_server/confluent/config/attributes.py
+++ b/confluent_server/confluent/config/attributes.py
@@ -371,7 +371,7 @@ node = {
                         'the managed node.  If not specified, then console '
                         'is disabled.  "ipmi" should be specified for most '
                         'systems if console is desired.'),
-        'validvalues': ('ssh', 'ipmi', 'tsmsol'),
+        'validvalues': ('ssh', 'ipmi', 'openbmc', 'tsmsol'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),
diff --git a/confluent_server/confluent/plugins/console/openbmc.py b/confluent_server/confluent/plugins/console/openbmc.py
new file mode 100644
index 00000000..17acae7c
--- /dev/null
+++ b/confluent_server/confluent/plugins/console/openbmc.py
@@ -0,0 +1,160 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This plugin provides an ssh implementation comforming to the 'console'
+# specification.  consoleserver or shellserver would be equally likely
+# to use this.
+
+import confluent.exceptions as cexc
+import confluent.interface.console as conapi
+import confluent.log as log
+import confluent.util as util
+import pyghmi.exceptions as pygexc
+import pyghmi.redfish.command as rcmd
+import pyghmi.util.webclient as webclient
+import eventlet
+import eventlet.green.ssl as ssl
+try:
+    websocket = eventlet.import_patched('websocket')
+    wso = websocket.WebSocket
+except Exception:
+    wso = object
+
+def get_conn_params(node, configdata):
+    if 'secret.hardwaremanagementuser' in configdata:
+        username = configdata['secret.hardwaremanagementuser']['value']
+    else:
+        username = 'USERID'
+    if 'secret.hardwaremanagementpassword' in configdata:
+        passphrase = configdata['secret.hardwaremanagementpassword']['value']
+    else:
+        passphrase = 'PASSW0RD'  # for lack of a better guess
+    if 'hardwaremanagement.manager' in configdata:
+        bmc = configdata['hardwaremanagement.manager']['value']
+    else:
+        bmc = node
+    bmc = bmc.split('/', 1)[0]
+    return {
+        'username': username,
+        'passphrase': passphrase,
+        'bmc': bmc,
+    }
+_configattributes = ('secret.hardwaremanagementuser',
+                     'secret.hardwaremanagementpassword',
+                     'hardwaremanagement.manager')
+
+class WrappedWebSocket(wso):
+
+    def set_verify_callback(self, callback):
+        self._certverify = callback
+
+    def connect(self, url, **options):
+        add_tls = url.startswith('wss://')
+        if add_tls:
+            hostname, port, resource, _ = websocket._url.parse_url(url)
+            if hostname[0] != '[' and ':' in hostname:
+                hostname = '[{0}]'.format(hostname)
+            if resource[0] != '/':
+                resource = '/{0}'.format(resource)
+            url = 'ws://{0}:443{1}'.format(hostname,resource)
+        else:
+            return super(WrappedWebSocket, self).connect(url, **options)
+        self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
+        self.sock, addrs = websocket._http.connect(url, self.sock_opt, websocket._http.proxy_info(**options),
+                                           options.pop('socket', None))
+        self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)
+        # The above is supersedeed by the _certverify, which provides
+        # known-hosts style cert validaiton
+        bincert = self.sock.getpeercert(binary_form=True)
+        if not self._certverify(bincert):
+            raise pygexc.UnrecognizedCertificate('Unknown certificate', bincert)
+        try:
+            self.handshake_response = websocket._handshake.handshake(self.sock, *addrs, **options)
+            if self.handshake_response.status in websocket._handshake.SUPPORTED_REDIRECT_STATUSES:
+                options['redirect_limit'] = options.pop('redirect_limit', 3) - 1
+                if options['redirect_limit'] < 0:
+                     raise Exception('Redirect limit hit')
+                url = self.handshake_response.headers['location']
+                self.sock.close()
+                return self.connect(url, **options)
+            self.connected = True
+        except:
+            if self.sock:
+                self.sock.close()
+                self.sock = None
+            raise
+
+
+         
+
+
+
+class TsmConsole(conapi.Console):
+
+    def __init__(self, node, config):
+        self.node = node
+        self.ws = None
+        configdata = config.get_node_attributes([node], _configattributes, decrypt=True)
+        connparams = get_conn_params(node, configdata[node])
+        self.username = connparams['username']
+        self.password = connparams['passphrase']
+        self.bmc = connparams['bmc']
+        self.origbmc = connparams['bmc']
+        if ':' in self.bmc:
+            self.bmc = '[{0}]'.format(self.bmc)
+        self.datacallback = None
+        self.nodeconfig = config
+        self.connected = False
+
+
+    def recvdata(self):
+        while self.connected:
+            pendingdata = self.ws.recv()
+            if pendingdata == '':
+                self.datacallback(conapi.ConsoleEvent.Disconnect)
+                return
+            self.datacallback(pendingdata)
+
+    def connect(self, callback):
+        self.datacallback = callback
+        kv = util.TLSCertVerifier(
+            self.nodeconfig, self.node, 'pubkeys.tls_hardwaremanager').verify_cert
+        wc = webclient.SecureHTTPConnection(self.origbmc, 443, verifycallback=kv)
+        rsp = wc.grab_json_response_with_status('/login', {'data': [self.username.decode('utf8'), self.password.decode("utf8")]}, headers={'Content-Type': 'application/json'})
+        bmc = self.bmc
+        if '%' in self.bmc:
+            prefix = self.bmc.split('%')[0]
+            bmc = prefix + ']'
+        self.ws = WrappedWebSocket(host=bmc)
+        self.ws.set_verify_callback(kv)
+        self.ws.connect('wss://{0}/console0'.format(self.bmc), host=bmc, cookie='XSRF-TOKEN={0}; SESSION={1}'.format(wc.cookies['XSRF-TOKEN'], wc.cookies['SESSION']))
+        self.connected = True
+        eventlet.spawn_n(self.recvdata)
+        return
+
+    def write(self, data):
+        self.ws.send(data)
+
+    def close(self):
+        if self.ws:
+            self.ws.close()
+        self.connected = False
+        self.datacallback = None
+
+def create(nodes, element, configmanager, inputdata):
+    if len(nodes) == 1:
+        return TsmConsole(nodes[0], configmanager)

From 04505e1bbb53bc1d3bb1335bb14557be17946179 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 19 Sep 2023 13:01:10 -0400
Subject: [PATCH 36/41] Break out partitioning and pagkacelist for EL profiles

---
 confluent_osdeploy/el8/profiles/default/kickstart      | 10 +---------
 confluent_osdeploy/el8/profiles/default/scripts/pre.sh | 10 +++++++---
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/confluent_osdeploy/el8/profiles/default/kickstart b/confluent_osdeploy/el8/profiles/default/kickstart
index fe626e93..95d4fe78 100644
--- a/confluent_osdeploy/el8/profiles/default/kickstart
+++ b/confluent_osdeploy/el8/profiles/default/kickstart
@@ -33,15 +33,7 @@
 reboot
 
 %packages
-@^minimal-environment
-#-kernel-uek # This can opt out of the UEK for the relevant distribution
-bind-utils
-chrony
-pciutils
-python3
-rsync
-tar
--iwl*-firmware
+%include /tmp/pkglist
 %include /tmp/addonpackages
 %include /tmp/cryptpkglist
 %end
diff --git a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh
index e00ea19a..4d76aaa3 100644
--- a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh
+++ b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh
@@ -87,6 +87,7 @@ done
 cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
 LUKSPARTY=''
 touch /tmp/cryptpkglist
+touch /tmp/pkglist
 touch /tmp/addonpackages
 if [ "$cryptboot" == "tpm2" ]; then
 	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
@@ -102,15 +103,18 @@ confluentpython /opt/confluent/bin/apiclient /confluent-public/os/$confluent_pro
 run_remote pre.custom
 run_remote_parts pre.d
 confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart -o /tmp/kickstart.base
+if grep '^%include /tmp/pkglist' /tmp/kickstart.* > /dev/null; then
+    confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/packagelist -o /tmp/pkglist
+fi
 grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || touch /tmp/installdisk
 if [ ! -e /tmp/installdisk ]; then
     run_remote_python getinstalldisk
 fi
+confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/partitioning -o /tmp/partitioning.template
 grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || rm /tmp/installdisk
 if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then
-    echo clearpart --all --initlabel >> /tmp/partitioning
-    echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
-    echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
+    INSTALLDISK=$(cat /tmp/installdisk)
+    sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e s/%%LUKSHOOK%%/$LUKSPARTY/ /tmp/partitioning.template > /tmp/partitioning
     dd if=/dev/zero of=/dev/$(cat /tmp/installdisk) bs=1M count=1 >& /dev/null
     vgchange -a n >& /dev/null
 fi

From d0c97b762365b2405761d28b340acc8d35f91b7e Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Tue, 19 Sep 2023 13:14:07 -0400
Subject: [PATCH 37/41] Add pieces of EL profiles

---
 confluent_osdeploy/el8/profiles/default/packagelist  | 9 +++++++++
 confluent_osdeploy/el8/profiles/default/partitioning | 4 ++++
 2 files changed, 13 insertions(+)
 create mode 100644 confluent_osdeploy/el8/profiles/default/packagelist
 create mode 100644 confluent_osdeploy/el8/profiles/default/partitioning

diff --git a/confluent_osdeploy/el8/profiles/default/packagelist b/confluent_osdeploy/el8/profiles/default/packagelist
new file mode 100644
index 00000000..4e3b9681
--- /dev/null
+++ b/confluent_osdeploy/el8/profiles/default/packagelist
@@ -0,0 +1,9 @@
+@^minimal-environment
+#-kernel-uek # This can opt out of the UEK for the relevant distribution
+bind-utils
+chrony
+pciutils
+python3
+rsync
+tar
+-iwl*-firmware
diff --git a/confluent_osdeploy/el8/profiles/default/partitioning b/confluent_osdeploy/el8/profiles/default/partitioning
new file mode 100644
index 00000000..c11b135b
--- /dev/null
+++ b/confluent_osdeploy/el8/profiles/default/partitioning
@@ -0,0 +1,4 @@
+clearpart --all --initlabel
+ignoredisk --only-use %%INSTALLDISK%%
+autopart --nohome %%LUKSHOOK%%
+

From f88b44dee2bfbcfb8d08d1d8d7c87e67c0c7b680 Mon Sep 17 00:00:00 2001
From: Jarrod Johnson <jjohnson2@lenovo.com>
Date: Wed, 20 Sep 2023 10:13:34 -0400
Subject: [PATCH 38/41] Fix issues with imgutil

---
 imgutil/imgutil | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/imgutil/imgutil b/imgutil/imgutil
index 5596ef6a..de3a9025 100644
--- a/imgutil/imgutil
+++ b/imgutil/imgutil
@@ -613,7 +613,7 @@ class DebHandler(OsHandler):
         needpkgs = []
         if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')):
             needpkgs.append('tpm2-tools')
-        lfuses = glob.glob(os.path.join(hostpath, '/lib/*/libfuse.so.2')
+        lfuses = glob.glob(os.path.join(hostpath, '/lib/*/libfuse.so.2'))
         if not lfuses:
             needpkgs.append('libfuse2')
         if needpkgs:
@@ -646,14 +646,16 @@ class DebHandler(OsHandler):
 
 
 class ElHandler(OsHandler):
-    def __init__(self, name, version, arch, args):
+    def __init__(self, name, version, arch, args, hostpath='/'):
         self.oscategory = 'el{0}'.format(version.split('.')[0])
         self.yumargs = []
         super().__init__(name, version, arch, args)
         needpkgs = []
+        if not hostpath:
+            return
         if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')):
             needpkgs.append('tpm2-tools')
-        lfuses = glob.glob(os.path.join(hostpath, '/usr/lib64/libfuse.so.2')
+        lfuses = glob.glob(os.path.join(hostpath, '/usr/lib64/libfuse.so.2'))
         if not lfuses:
             needpkgs.append('fuse-libs')
         if not os.path.exists(os.path.join(hostpath, '/usr/bin/ipcalc')):
@@ -1040,7 +1042,7 @@ def fingerprint_source_el(files, sourcepath, args):
             if arch == 'noarch':
                 prodinfo = open(os.path.join(sourcepath, '.discinfo')).read()
                 arch = prodinfo.split('\n')[2]
-            return ElHandler(osname, ver, arch, args)
+            return ElHandler(osname, ver, arch, args, None)
     return None
 
 
@@ -1092,7 +1094,7 @@ def fingerprint_host_el(args, hostpath='/'):
     osname = osname.replace('-release', '').replace('-', '_')
     if osname == 'centos_linux':
         osname = 'centos'
-    return ElHandler(osname, version, os.uname().machine, args)
+    return ElHandler(osname, version, os.uname().machine, args, hostpath)
 
 
 def fingerprint_host_deb(args, hostpath='/'):

From 5768514ba9ef05de0f4b9a108adeff7db1433087 Mon Sep 17 00:00:00 2001
From: tkucherera <tkucherera@lenovo.com>
Date: Wed, 20 Sep 2023 14:29:21 -0400
Subject: [PATCH 39/41] nodeattrib -s attrib.batch

---
 confluent_client/bin/nodeattrib | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib
index 4ed96406..8c6f078d 100755
--- a/confluent_client/bin/nodeattrib
+++ b/confluent_client/bin/nodeattrib
@@ -53,6 +53,8 @@ argparser.add_option('-p', '--prompt', action='store_true',
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Prompt if trying to set attributes on more '
                           'than specified number of nodes')
+argparser.add_option('-s', '--set', dest='set', metavar='settings.batch',
+                     default=False, help='set attributes using a batch file')
 (options, args) = argparser.parse_args()
 
 
@@ -109,6 +111,23 @@ elif options.clear or options.environment or options.prompt:
     sys.stderr.write('Attribute names required with specified options\n')
     argparser.print_help()
     exitcode = 400
+
+elif options.set:
+    arglist = [noderange]
+    showtype='current'
+    argfile = open(options.set, 'r')
+    argset = argfile.readline()
+    while argset:
+        try:
+            argset = argset[:argset.index('#')]
+        except ValueError:
+            pass
+        argset = argset.strip()
+        if argset:
+            arglist += shlex.split(argset)
+        argset = argfile.readline() 
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    exitcode=client.updateattrib(session,arglist,nodetype, noderange, options, None)
 if exitcode != 0:
     sys.exit(exitcode)
 

From 4ecbbdfd9a55d351a375ed92370be7799db81f47 Mon Sep 17 00:00:00 2001
From: tkucherera <tkucherera@lenovo.com>
Date: Wed, 20 Sep 2023 14:37:07 -0400
Subject: [PATCH 40/41] nodeattrib -s use with batch file

---
 confluent_server/confluent/httpapi.py | 95 ++++++---------------------
 1 file changed, 21 insertions(+), 74 deletions(-)

diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py
index d418302d..5a145a0c 100644
--- a/confluent_server/confluent/httpapi.py
+++ b/confluent_server/confluent/httpapi.py
@@ -55,12 +55,6 @@ except ModuleNotFoundError:
     import urllib.parse as urlparse
 import eventlet.websocket
 import eventlet.wsgi
-
-#these modifications might need to be changed later now just proof of concept
-import os
-import cgi
-#from cgi import parse_qs, escape
-
 #scgi = eventlet.import_patched('flup.server.scgi')
 tlvdata = confluent.tlvdata
 
@@ -177,7 +171,6 @@ def _get_query_dict(env, reqbody, reqtype):
     qdict = {}
     try:
         qstring = env['QUERY_STRING']
-  
     except KeyError:
         qstring = None
     if qstring:
@@ -295,12 +288,16 @@ def _authorize_request(env, operation, reqbody):
             authdata = auth.authorize(name, element=element, operation=operation)
         else:
             element = None
-    if (not authdata) and 'HTTP_COOKIE' in env:
-        cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
-        if cidx >= 0:
-            sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
-            sessid = sessionid
+    if not authdata:
+        if 'HTTP_CONFLUENTSESSION' in env:
+            sessionid = env['HTTP_CONFLUENTSESSION']
             sessid = sessionid
+        elif 'HTTP_COOKIE' in env:
+            cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
+            if cidx >= 0:
+                sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
+                sessid = sessionid
+        if sessionid:
             if sessionid in httpsessions:
                 if _csrf_valid(env, httpsessions[sessionid]):
                     if env['PATH_INFO'] == '/sessions/current/logout':
@@ -490,6 +487,8 @@ def wsock_handler(ws):
                         elif clientmsg[0] == '!':
                             msg = json.loads(clientmsg[1:])
                             action = msg.get('operation', None)
+                            if not action:
+                                action = msg.get('action', None)
                             targ = msg.get('target', None)
                             if targ:
                                 authdata = auth.authorize(name, targ, operation=action)
@@ -527,6 +526,13 @@ def wsock_handler(ws):
                                             datacallback=datacallback,
                                             width=width, height=height)
                                     myconsoles[clientsessid] = consession
+                            elif action == 'resize':
+                                clientsessid = '{0}'.format(msg['sessid'])
+                                myconsoles[clientsessid].resize(
+                                    width=msg['width'], height=msg['height'])
+                            if action == 'break':
+                                clientsessid = '{0}'.format(msg['sessid'])
+                                myconsoles[clientsessid].send_break()
                             elif action == 'stop':
                                 sessid = '{0}'.format(msg.get('sessid', None))
                                 if sessid in myconsoles:
@@ -660,13 +666,9 @@ def resourcehandler_backend(env, start_response):
         start_response('302 Found', headers)
         yield ''
         return
-
-    if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0: 
-        reqbody = ''     
-        # reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
+    if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:
+        reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
         reqtype = env['CONTENT_TYPE']
-        if 'application/json' in reqtype:
-            reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
     operation = opmap[env['REQUEST_METHOD']]
     querydict = _get_query_dict(env, reqbody, reqtype)
     if operation != 'retrieve' and 'restexplorerop' in querydict:
@@ -885,59 +887,6 @@ def resourcehandler_backend(env, start_response):
             start_response('200 OK', headers)
             yield rsp
             return
-    elif (operation == 'create' and ('/firmware/updates/active' in env['PATH_INFO'])):
-        # Solution the wsgi.iput stream can only be read once and it is being read at the top 
-        if 'multipart/form-data' in reqtype:
-            field_storage = cgi.FieldStorage(fp=env['wsgi.input'], environ=env, keep_blank_values=True)
-            for item in field_storage.list:
-                if item.filename: 
-                    storage_file_path = '/var/lib/confluent/client_assets/' + item.filename       
-                    file_content = item.file.read()    
-                    with open(storage_file_path, 'wb') as file:
-                        file.write(file_content)
-            yield json.dumps({'data': storage_file_path})
-            start_response('200 OK', headers)
-            return 
-
-        url = env['PATH_INFO']
-        if 'application/json' in reqtype:
-            if not isinstance(reqbody, str):
-                reqbody = reqbody.decode('utf8')
-            pbody = json.loads(reqbody)
-            args = pbody['args']
-            args_dict = {'filename': args}
-            try:
-                args_dict.update({'bank': pbody['bank']})
-            except KeyError:
-                pass
-            noderrs = {}
-            nodeurls = {}
-            # start_response('202 Accepted', headers)
-            hdlr = pluginapi.handle_path(url, operation, cfgmgr, args_dict)
-            for res in hdlr:
-                if isinstance(res, confluent.messages.CreatedResource):
-                    watchurl = res.kvpairs['created']
-                    currnode = watchurl.split('/')[1]
-                    nodeurls[currnode] = '/' + watchurl
-            yield json.dumps({'data': nodeurls})
-            start_response('200 OK', headers)
-            return
-
-    elif (operation == 'delete' and ('/firmware/updates/active' in env['PATH_INFO'])):
-        url = env['PATH_INFO']
-        if 'application/json' in reqtype:
-            if not isinstance(reqbody, str):
-                reqbody = reqbody.decode('utf8')
-            pbody = json.loads(reqbody)
-            args = pbody['args']
-            try:
-                os.remove(args)
-                start_response('200 OK', headers)        
-                return
-            except Exception as e:
-                return e
-
-
     else:
         # normal request
         url = env['PATH_INFO']
@@ -1023,7 +972,6 @@ def _assemble_html(responses, resource, querydict, url, extension):
         else:
             pendingrsp.append(rsp)
     for rsp in pendingrsp:
-        print(rsp.html() + "<br>")
         yield rsp.html() + "<br>"
     if iscollection:
         # localpath = url[:-2] (why was this here??)
@@ -1122,7 +1070,6 @@ def serve(bind_host, bind_port):
         try:
             sock = eventlet.listen(
                 (bind_host, bind_port, 0, 0), family=socket.AF_INET6)
-            
         except socket.error as e:
             if e.errno != 98:
                 raise
@@ -1157,4 +1104,4 @@ class HttpApi(object):
         self.server = eventlet.spawn(serve, self.bind_host, self.bind_port)
 
 
-_cleaner = eventlet.spawn(_sessioncleaner)
\ No newline at end of file
+_cleaner = eventlet.spawn(_sessioncleaner)

From bd0b16992d400ad5039a1b28e93be43ecc38cf7a Mon Sep 17 00:00:00 2001
From: tkucherera <tkucherera@lenovo.com>
Date: Wed, 20 Sep 2023 14:42:17 -0400
Subject: [PATCH 41/41] man page

---
 confluent_client/doc/man/nodeattrib.ronn.tmpl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl
index 6684fce0..b1330198 100644
--- a/confluent_client/doc/man/nodeattrib.ronn.tmpl
+++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl
@@ -7,7 +7,8 @@ nodeattrib(8) -- List or change confluent nodes attributes
 `nodeattrib <noderange> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodeattrib -c <noderange> <nodeattribute1> <nodeattribute2> ...`  
 `nodeattrib -e <noderange> <nodeattribute1> <nodeattribute2> ...`  
-`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`  
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...` 
+`nodeattrib <noderange> -s <attributes.batch>` 
 
 ## DESCRIPTION
 
@@ -58,7 +59,10 @@ to a blank value will allow masking a group defined attribute with an empty valu
 * `-p`, `--prompt`:
   Request interactive prompting to provide values rather than the command line
   or environment variables.
-  
+
+* `-s`, `--set`:
+  Set attributes using a batch file
+ 
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Prompt if trying to set attributes on more than
   specified number of nodes.