Favor more utilization of bottom screenshot row

When we have the grid size, cut off any extra columns
so long as it doesn't gain a row.

confluent_client/addattribs.py

@@ -14,11 +14,16 @@ import shutil
 
 shutil.copyfile('doc/man/nodeattrib.ronn.tmpl', 'doc/man/nodeattrib.ronn')
 shutil.copyfile('doc/man/nodegroupattrib.ronn.tmpl', 'doc/man/nodegroupattrib.ronn')
-with open('doc/man/nodeattrib.ronn', 'a') as outf:
-    for field in sorted(attr.node):
-        outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
-with open('doc/man/nodegroupattrib.ronn', 'a') as outf:
-    for field in sorted(attr.node):
-        outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
 
+def append_attributes(filename):
+    with open(filename, 'a') as outf:
+        for field in sorted(attr.node):
+            outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
+            # Optionally write valid values if they exist
+            for key, values in attr.node[field].items():
+                if key.startswith('valid'):
+                    values_formatted = ', '.join("'{0}'".format(v) for v in values)
+                    outf.write(f'\n    Valid values: {values_formatted}\n')
 
+append_attributes('doc/man/nodeattrib.ronn')
+append_attributes('doc/man/nodegroupattrib.ronn')

confluent_client/bin/confetty

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
@@ -654,13 +654,17 @@ def quitconfetty(code=0, fullexit=False, fixterm=True):
 def get_session_node(shellargs):
     # straight to node console
     if len(shellargs) == 1 and ' ' not in shellargs[0]:
-        return shellargs[0]
+        targ = "/nodes/%s/console/session" % shellargs[0]
+        return targ, shellargs[0]
     if len(shellargs) == 2 and shellargs[0] == 'start':
         args = [s for s in shellargs[1].split('/') if s]
         if len(args) == 4 and args[0] == 'nodes' and args[2] == 'console' and \
                 args[3] == 'session':
-            return args[1]
-    return None
+            return shellargs[1], args[1]
+        if len(args) == 5 and args[0] == 'nodes' and args[2] == 'shell' and \
+                args[3] == 'sessions':
+            return shellargs[1], args[1]
+    return None, None
 
 
 def run_inline_command(path, arg, completion, **kwargs):
@@ -917,10 +921,10 @@ def main():
     doexit = False
     inconsole = False
     pendingcommand = ""
-    session_node = get_session_node(shellargs)
+    targ, session_node = get_session_node(shellargs)
     if session_node is not None:
         consoleonly = True
-        do_command("start /nodes/%s/console/session" % session_node, netserver)
+        do_command("start %s" % targ, netserver)
         doexit = True
     elif shellargs:
         do_command(shellargs, netserver)
@@ -948,8 +952,9 @@ def main():
                     except IOError:
                         pass
             if powerstate is None or powertime < time.time() - 10:  # Check powerstate every 10 seconds
+                if powerstate == None:
+                    powerstate = True
                 powertime = time.time()
-                powerstate = True
                 check_power_state()
         else:
             currcommand = prompt()

confluent_client/bin/confluent2hosts

@@ -118,6 +118,7 @@ def main():
     ap.add_argument('-a', '--attrib', help='Pull ip addresses and hostnames from attribute database', action='store_true')
     ap.add_argument('-i', '--ip', help='Expression to generate addresses (e.g. 172.16.1.{n1} or fd2b:246f:8a50::{n1:x})')
     ap.add_argument('-n', '--name', help='Expression for name to add ({node}-compute, etc). If unspecified, "{node} {node}.{dns.domain}" will be used', action='append')
+    ap.add_argument('-f', '--fqdn-first', help='Put the FQDN first in the hosts entries', action='store_true')
     args = ap.parse_args()
     c = client.Command()
     if args.name:
@@ -173,7 +174,13 @@ def main():
                                     break
                             else:
                                 for name in list(names):
-                                    names.append('{0}.{1}'.format(name, mydomain))
+                                    fqdn = '{0}.{1}'.format(name, mydomain)
+                                    if args.fqdn_first:
+                                        # Insert FQDN at the beginning if --fqdn-first flag is set
+                                        names.insert(0, fqdn)
+                                    else:
+                                        # Otherwise, append FQDN at the end (original behavior)
+                                        names.append(fqdn)
                     names = ' '.join(names)
                     merger.add_entry(ipdb[node][currnet], names)
         merger.write_out('/etc/whatnowhosts')

confluent_client/bin/dir2img

@@ -8,30 +8,35 @@ import os
 import subprocess
 import sys
 
-def create_image(directory, image, label=None):
-    ents = 0
-    datasz = 512
-    for dir in os.walk(sys.argv[1]):
-        ents += 1
-        for filen in dir[2]:
+def create_image(directory, image, label=None, esize=0, totalsize=None):
+
+    if totalsize:
+        datasz = totalsize * 1048576
+    else:
+        ents = 0
+        datasz = 512 + (esize * 1048576)
+        for dir in os.walk(sys.argv[1]):
             ents += 1
-            filename = os.path.join(dir[0], filen)
-            currsz = os.path.getsize(filename)
-            # assuming up to 65k cluster
-            currsz = (currsz // 512 +1) * 512
-            datasz += currsz
-    datasz += ents * 32768
-    datasz = datasz // 16384 + 1
+            for filen in dir[2]:
+                ents += 1
+                filename = os.path.join(dir[0], filen)
+                currsz = os.path.getsize(filename)
+                # assuming up to 65k cluster
+                currsz = (currsz // 512 + 1) * 512
+                datasz += currsz
+        datasz += ents * 32768
+    datasz = datasz // 65536 + 1
     with open(image, 'wb') as imgfile:
-        imgfile.seek(datasz * 16384 - 1)
+        imgfile.seek(datasz * 65536 - 1)
         imgfile.write(b'\x00')
     if label:
+        # 4 heads, 32 sectors, means 65k per track
         subprocess.check_call(['mformat', '-i', image, '-v', label,
                                '-r', '16', '-d', '1', '-t', str(datasz),
-                               '-s', '16','-h', '2', '::'])
+                               '-s', '32','-h', '4', '::'])
     else:
         subprocess.check_call(['mformat', '-i', image, '-r', '16', '-d', '1', '-t',
-                                str(datasz), '-s', '16','-h', '2', '::'])
+                                str(datasz), '-s', '32','-h', '4', '::'])
     # Some clustered filesystems will have the lock from mformat
     # linger after close (mformat doesn't unlock)
     # do a blocking wait for shared lock and then explicitly
@@ -56,6 +61,21 @@ if __name__ == '__main__':
             sys.argv[0]))
         sys.exit(1)
     label = None
-    if len(sys.argv) > 3:
-        label = sys.argv[3]
-    create_image(sys.argv[1], sys.argv[2], label)
+    args = sys.argv
+    esize = 0
+    try:
+        earg = args.index('-e')
+        esize = int(args[earg + 1])
+        args = args[:earg] + args[earg +2:]
+    except ValueError:
+        pass
+    totsize = None
+    try:
+        earg = args.index('-s')
+        totsize = int(args[earg + 1])
+        args = args[:earg] + args[earg +2:]
+    except ValueError:
+        pass
+    if len(args) > 3:
+        label = args[3]
+    create_image(args[1], args[2], label, esize, totsize)

confluent_client/bin/nodeconsole

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import optparse
 import os
 import subprocess
@@ -28,8 +29,31 @@ import confluent.client as client
 import confluent.sortutil as sortutil
 import confluent.logreader as logreader
 import time
+import select
 import socket
 import re
+import tty
+import termios
+import fcntl
+import confluent.screensqueeze as sq
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+
+try:
+    # sixel is optional, attempt to import but stub out if unavailable
+    import io
+    import sixel
+
+    class DumbWriter(sixel.SixelWriter):
+        def restore_position(self, output):
+            return
+except ImportError:
+    class DumbWriter():
+        def draw(self, imgfile):
+            sys.stderr.write("PySixel not detected, Sixel format display not supported\n")
+
 
 confettypath = os.path.join(os.path.dirname(sys.argv[0]), 'confetty')
 argparser = optparse.OptionParser(
@@ -46,6 +70,11 @@ argparser.add_option('-l', '--log', action='store_true', default=False,
 argparser.add_option('-T', '--Timestamp', action='store_true', default=False,
                      help= 'Dump log in stdout with timestamps')
 
+argparser.add_option('-s', '--screenshot', action='store_true', default=False,
+                     help='Attempt to grab screenshot and render using kitty image protocol')
+argparser.add_option('-i', '--interval', type='float',
+                     help='Interval in seconds to redraw the screenshot. Currently only '
+                          'works for one node')
 argparser.add_option('-w','--windowed', action='store_true', default=False,
                      help='Open terminal windows for each node.  The '
                           'environment variable NODECONSOLE_WINDOWED_COMMAND '
@@ -69,6 +98,179 @@ argparser.add_option('-w','--windowed', action='store_true', default=False,
 
 (options, args) = argparser.parse_args()
 
+oldtcattr = None
+oldfl = None
+
+def get_coords():
+    sys.stdout.write('\x1b[6n') #
+    sys.stdout.flush()
+    gotreply = select.select([sys.stdin,], [], [], 0.250)[0]
+    if gotreply:
+        response = ''
+        while select.select([sys.stdin,], [], [], 0.1)[0] and 'R' not in response:
+            response += sys.stdin.read()
+        coords = response.replace('R', '').split('[')[1].split(';')
+    #sys.stdout.write('\x1b[{}:{}H'.format(*coords))
+
+def direct_console():
+    global oldtcattr
+    global oldfl
+    oldtcattr = termios.tcgetattr(sys.stdin.fileno())
+    oldfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
+    tty.setraw(sys.stdin.fileno())
+    fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, oldfl | os.O_NONBLOCK)
+
+def indirect_console():
+    fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, oldfl & ~os.O_NONBLOCK)
+    termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
+
+def determine_tile_size(numnodes):
+    # for now, smash everything to a common aspect ratio.  16:11
+    # is pretty much wrong for everything, making 4:3 a bit too wide
+    # and 16:9 significantly too narrow, but it is serviceable
+    # An improvement could come with us owning the scaling
+    # instead of delegating to Kitty, which says if we specify both,
+    # we get stretching.  In theory we should be able to get aspect correct
+    # from kitty by omitting, but:
+    # then we don't know how much to move the cursor left after draw_image
+    # Konsole won't scale at all with only partial scaling specified
+    cheight, cwidth, pixwidth, pixheight = sq.get_screengeom()
+    # 16:12 is to roughly account for the 'titles' of the tiles
+    ratio = (pixwidth / 16) / (pixheight / 12)
+    bestdeviation = None
+    bestdims = []
+    for i in range(1, numnodes + 1):
+        number = numnodes
+        while number % i != 0:
+            number += 1
+        columns = i
+        rows = number // i
+        deviation = abs(ratio -  (columns / rows))
+        if bestdeviation is None:
+            bestdeviation = deviation
+            bestdims = [columns, rows]
+        elif deviation < bestdeviation:
+            bestdeviation = deviation
+            bestdims = [columns, rows]
+    # ok, the above algorithm can still pick things like
+    # 1 2 3
+    # 4
+    # So we will let it pick the number of rows, and
+    # then see if we can chop columns and still fit
+    while (bestdims[0] - 1) * bestdims[1] >= numnodes:
+        bestdims[0] = bestdims[0] - 1
+    cellswide = cwidth // bestdims[0]
+    cellshigh = cheight // bestdims[1]
+    tilewidth = cellswide * pixwidth / cwidth
+    tileheight = cellshigh * pixheight / cheight
+    if tilewidth > (tileheight * 16 / 11):
+        tilewidth = tileheight * 16 / 11
+        cellswide = int(tilewidth // (pixwidth / cwidth))
+    if tileheight > (tilewidth * 11 /16):
+        tileheight = tilewidth * 11 / 16
+        cellshigh = int(tileheight // (pixheight / cheight))
+    bestdims = bestdims + [cellswide, cellshigh, cellshigh * bestdims[1]]
+    # incur any scrolling we might get. This allows us to accurately
+    # save/restore cursor or even get coordinates without scrolling fouling
+    # the desired target
+    sys.stdout.write('\n' * bestdims[4])
+    sys.stdout.flush()
+    cursor_up(bestdims[4])
+    return bestdims
+
+cursor_saved = False
+def sticky_cursor():
+    global cursor_saved
+# get cursor restore_position
+    if sys.stdin.isatty() and not cursor_saved:
+        try:
+            direct_console()
+            sys.stdout.write('\x1b7')
+            cursor_saved = True
+        finally:
+            indirect_console()
+    elif cursor_saved:
+        try:
+            direct_console()
+            sys.stdout.write('\x1b8')
+        finally:
+            indirect_console()
+
+def cursor_up(count=1):
+    sys.stdout.write(f'\x1b[{count}A')
+def cursor_down(count=1):
+    sys.stdout.write(f'\x1b[{count}B')
+def cursor_right(count=1):
+    sys.stdout.write(f'\x1b[{count}C')
+def cursor_left(count=1):
+    sys.stdout.write(f'\x1b[{count}D')
+def cursor_save():
+    sys.stdout.write('\x1b7')
+def cursor_restore():
+    sys.stdout.write('\x1b8')
+def cursor_hide():
+    sys.stdout.write('\x1b[?25l')
+def cursor_show():
+    sys.stdout.write('\x1b[?25h')
+
+def draw_image(data, width, height):
+    imageformat = os.environ.get('CONFLUENT_IMAGE_PROTOCOL', 'kitty')
+    if imageformat == 'sixel':
+        sixel_draw(data)
+    elif imageformat == 'iterm':
+        iterm_draw(data, width, height)
+    else:
+        kitty_draw(data, width, height)
+
+
+def sixel_draw(data):
+    bindata = base64.b64decode(data)
+    binfile = io.BytesIO()
+    binfile.write(bindata)
+    binfile.seek(0)
+    DumbWriter().draw(binfile)
+
+def iterm_draw(data, width, height):
+    if not height:
+        height = 'auto'
+    if not width:
+        width = 'auto'
+    bindata = base64.b64decode(data)
+    datalen = len(bindata)
+    sys.stdout.write(
+        '\x1b]1337;File=inline=1;width={};height={};size={}:'.format(width,height,datalen))
+    sys.stdout.write(data.decode('utf8'))
+    sys.stdout.write('\a')
+    sys.stdout.flush()
+
+def kitty_draw(data, width, height):
+    if Image:
+        bindata = base64.b64decode(data)
+        binfile = io.BytesIO()
+        binfile.write(bindata)
+        binfile.seek(0)
+        img = Image.open(binfile)
+        outfile = io.BytesIO()
+        img.save(outfile, format='PNG')
+        data = base64.b64encode(outfile.getbuffer())
+    preamble = '\x1b_Ga=T,f=100'
+    if height:
+        preamble += f',r={height},c={width}'
+    #sys.stdout.write(repr(preamble))
+    #sys.stdout.write('\xb[{}D'.format(len(repr(preamble))))
+    #return
+    first = True
+    while data:
+        chunk, data = data[:4096], data[4096:]
+        m = 1 if data else 0
+        if first:
+            sys.stdout.write('{},m={};'.format(preamble, m))
+        else:
+            sys.stdout.write('\x1b_Gm={};'.format(m))
+        sys.stdout.write(chunk.decode('utf8'))
+        sys.stdout.write('\x1b\\')
+        sys.stdout.flush()
+
 pass_through_args = []
 killcon = False
 try:
@@ -106,6 +308,102 @@ if options.Timestamp:
     logreader.dump_to_console(logname)
     sys.exit(0)
 
+def prep_node_tile(node):
+    currcolcell, currrowcell = nodepositions[node]
+    if currcolcell:
+        cursor_right(currcolcell)
+    if currrowcell:
+        cursor_down(currrowcell)
+    sys.stdout.write(node)
+    cursor_left(len(node))
+    cursor_down()
+
+def reset_cursor(node):
+    currcolcell, currrowcell = nodepositions[node]
+    if currcolcell:
+        cursor_left(currcolcell)
+    cursor_up(currrowcell + 1)
+
+nodepositions = {}
+numrows = 0
+def do_screenshot():
+    global numrows
+    cwidth = None
+    cheight = None
+    sess = client.Command()
+    if options.tile:
+        imageformat = os.environ.get('CONFLUENT_IMAGE_PROTOCOL', 'kitty')
+        if imageformat not in ('kitty', 'iterm'):
+            sys.stderr.write('Tiled screenshots only supported with kitty or iterm protocol')
+            sys.exit(1)
+        allnodes = []
+        numnodes = 0
+        for res in sess.read('/noderange/{}/nodes/'.format(args[0])):
+            allnodes.append(res['item']['href'].replace('/', ''))
+            numnodes += 1
+        cols, rows, cwidth, cheight, numrows = determine_tile_size(numnodes)
+        currcol = 1
+        currcolcell = 0
+        currrowcell = 0
+        for node in allnodes:
+            nodepositions[node] = currcolcell, currrowcell
+            if currcol < cols:
+                currcol += 1
+                currcolcell += cwidth
+            else:
+                currcol = 1
+                currcolcell = 0
+                currrowcell += cheight
+    elif options.interval is not None:
+        sys.stdout.write('\x1bc')
+    firstnodename = None
+    dorefresh = True
+    while dorefresh:
+        for res in sess.read('/noderange/{}/console/ikvm_screenshot'.format(args[0])):
+            for node in res.get('databynode', {}):
+                if not firstnodename:
+                    firstnodename = node
+                imgdata = res['databynode'][node].get('image', {}).get('imgdata', None)
+                if imgdata:
+                    if len(imgdata) < 32: # We were subjected to error
+                        sys.stderr.write(f'{node}: Unable to get screenshot\n')
+                        continue
+                    if node in nodepositions:
+                        prep_node_tile(node)
+                        cursor_save()
+                    else:
+                        if options.interval is not None:
+                            if node != firstnodename:
+                                sys.stderr.write('Multiple nodes not supported for interval')
+                                sys.exit(1)
+                            sticky_cursor()
+                        sys.stdout.write('{}: '.format(node))
+                    # one row is used by our own name, so cheight - 1 for that allowance
+                    draw_image(imgdata.encode(), cwidth, cheight - 1 if cheight else cheight)
+                    if node in nodepositions:
+                        cursor_restore()
+                        reset_cursor(node)
+                    else:
+                        sys.stdout.write('\n')
+                    sys.stdout.flush()
+        if options.interval is None:
+            dorefresh = False
+        else:
+            dorefresh = True
+            time.sleep(options.interval)
+    sys.exit(0)
+
+if options.screenshot:
+    try:
+        cursor_hide()
+        do_screenshot()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        cursor_show()
+        cursor_down(numrows)
+        sys.stdout.write('\n')
+    sys.exit(0)
 def kill(noderange):
     sess = client.Command()
     envstring=os.environ.get('NODECONSOLE_WINDOWED_COMMAND')

confluent_client/bin/nodeeventlog

@@ -78,8 +78,11 @@ exitcode = 0
 def format_event(evt):
     retparts = []
     if 'timestamp' in evt and evt['timestamp'] is not None:
-        display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
-        retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
+        try:
+            display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
+            retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
+        except ValueError:
+            display = ''    
     dscparts = []
     if evt.get('log_id', None):
         retparts.append(evt['log_id'] + ':')

confluent_client/bin/nodefirmware

@@ -56,7 +56,7 @@ components = ['all']
 
 argparser = optparse.OptionParser(
     usage="Usage: "
-          "%prog <noderange> [list][update [--backup <file>]]|[<components>]")
+          "%prog <noderange> [list][updatestatus][update [--backup <file>]]|[<components>]")
 argparser.add_option('-b', '--backup', action='store_true',
                      help='Target a backup bank rather than primary')
 argparser.add_option('-m', '--maxnodes', type='int',
@@ -65,14 +65,18 @@ argparser.add_option('-m', '--maxnodes', type='int',
              
 (options, args) = argparser.parse_args()
 upfile = None
+querystatus = False
 try:
     noderange = args[0]
     if len(args) > 1:
         if args[1] == 'update':
             upfile = args[2]
         else:
+            comps = []
             if args[1] == 'list':
                 comps = args[2:]
+            elif args[1] == 'updatestatus':
+                querystatus = True
             else:
                 comps = args[1:]
             components = []
@@ -171,7 +175,16 @@ def show_firmware(session):
 
 try:
     session = client.Command()
-    if upfile is None:
+    if querystatus:
+        for res in session.read(
+                '/noderange/{0}/inventory/firmware/updatestatus'.format(noderange)):
+            for node in res.get('databynode', {}):
+                currstat = res['databynode'][node].get('status', None)
+                if currstat:
+                    print('{}: {}'.format(node, currstat))
+                else:
+                    print(repr(res))
+    elif upfile is None:
         show_firmware(session)
     else:
         update_firmware(session, upfile)

confluent_client/bin/nodel2traceroute

@@ -61,7 +61,10 @@ def get_neighbors(switch):
     switch_neigbors = []
     url = '/networking/neighbors/by-switch/{0}/by-peername/'.format(switch)
     for neighbor in session.read(url):
-        switch = neighbor['item']['href'].strip('/')
+        try:
+            switch = neighbor['item']['href'].strip('/')
+        except:
+            continue
         if switch in all_switches:
             switch_neigbors.append(switch)
     return switch_neigbors

confluent_client/bin/nodeping

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -42,6 +42,8 @@ def run():
         usage="Usage: %prog [options] noderange")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-o', '--origname', action='store_true',
+                         help='Use original nodename in print out even if substituted')
     argparser.add_option('-s', '--substitutename',
                          help='Use a different name other than the nodename for ping, with {}, it is the entire name evaluated as an expression, otherwise it is used as a suffix')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -83,7 +85,10 @@ def run():
             cmdv = ['ping', '-c', '1', '-W', '1', pingnode]
             if currprocs < concurrentprocs:
                 currprocs += 1
-                run_cmdv(pingnode, cmdv, all, pipedesc)
+                if options.origname:
+                    run_cmdv(node, cmdv, all, pipedesc)
+                else:
+                    run_cmdv(pingnode, cmdv, all, pipedesc)
             else:
                 pendingexecs.append((pingnode, cmdv))
     if not all or exitcode:

confluent_client/bin/nodesensors

@@ -67,7 +67,7 @@ argparser.add_option('-n', '--numreadings', type='int',
 argparser.add_option('-c', '--csv', action='store_true',
                      help='Output in CSV format')
 argparser.add_option('-s', '--skipnumberless', action='store_true',
-                     help='Output in CSV format')
+                     help='Do not show non-numeric sensors')
 (options, args) = argparser.parse_args()
 repeatmode = False
 if options.interval:

confluent_client/bin/stats

@@ -72,6 +72,8 @@ def plot(gui, output, plotdata, bins, fmt):
             tdata = io.BytesIO()
         plt.savefig(tdata)
     if not gui and not output:
+        if fmt == 'environment':
+            fmt = os.environ.get('CONFLUENT_IMAGE_PROTOCOL', 'kitty')
         if fmt == 'sixel':
             writer = DumbWriter()
             writer.draw(tdata)
@@ -108,7 +110,7 @@ aparser = argparse.ArgumentParser(description='Quick access to common statistics
 aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
 aparser.add_argument('-d', default=None, help='Value used to separate columns')
 aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format')
-aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)')
+aparser.add_argument('-f', default='environment', help='Format for histogram output (sixel/iterm/kitty)')
 aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
 aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
 aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')

confluent_client/confluent/screensqueeze.py

@@ -18,8 +18,9 @@ import struct
 import termios
 
 def get_screengeom():
-    return struct.unpack('hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ,
-                                           b'....'))
+    # returns height in cells, width in cells, width in pixels, height in pixels
+    return struct.unpack('hhhh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ,
+                                           b'........'))
 class ScreenPrinter(object):
 
     def __init__(self, noderange, client, textlen=4):
@@ -58,7 +59,7 @@ class ScreenPrinter(object):
 
     def drawscreen(self, node=None):
         if self.squeeze:
-            currheight, currwidth = get_screengeom()
+            currheight, currwidth, _, _ = get_screengeom()
             currheight -= 2
             if currheight < 1:
                 currheight = 1
@@ -120,6 +121,7 @@ if __name__ == '__main__':
     c = client.Command()
     p = ScreenPrinter('d1-d12', c)
     p.set_output('d3', 'Upload: 67%')
+    p.set_output('d7', 'Upload: 67%')
 
 
 

confluent_client/confluent/textgroup.py

@@ -171,7 +171,9 @@ class GroupedData(object):
                     self.byoutput[outdata])))
                 currout += '\n====================================\n'
             currout += outdata
-            currout += '\n\n'
+            if currout[-1] != '\n':
+                currout += '\n'
+            currout += '\n'
             output.write(currout)
         output.flush()
 
@@ -211,7 +213,9 @@ class GroupedData(object):
             else:
                 currout += '\n'.join(colordiff(modaloutput.split('\n'),
                                                  outdata.split('\n')))
-            currout += '\n\n'
+            if currout[-1] != '\n':
+                currout += '\n'
+            currout += '\n'
             if reverse:
                 revoutput.append(currout)
             else:

confluent_client/doc/man/confluent2hosts.ronn

@@ -13,7 +13,7 @@ noderange.  There are two general approaches.
 
 It can be used ad-hoc, using -i and -n to specify the address and name portions respectively.  This accepts the standard confluent expression syntax, allowing for things like 172.30.1.{n1} or {node}.{dns.domain} or {bmc}.
 
-It can also read from the confluent db, using `-a`.  In this mode, each net.<value>.<attribute> group is pulled together into hosts lines.  ipv4_address and ipv6_address fields are associated with the corresponding hostname attributes.
+It can also read from the confluent db, using `-a`.  In this mode, each net.<value>.<attribute> group is pulled together into hosts lines.  ipv4_address and ipv6_address fields are associated with the corresponding hostname attributes. You can use `-f` to put the FQDN first.
 
 ## EXAMPLES
 

confluent_client/doc/man/confluentdbutil.ronn

@@ -38,5 +38,8 @@ the json files (password protected, removed from the files, or unprotected).
   keys do not change and as such they do not require
   incremental backup.
   
+* `-y`, `--yaml
+  Use YAML instead of JSON as file format
+  
   * `-h`, `--help`:
   Show help message and exit

confluent_client/doc/man/l2traceroute.ronn

@@ -1,38 +0,0 @@
-l2traceroute(8) -- returns the layer 2 route through an Ethernet network managed by confluent given 2 end points.
-==============================
-## SYNOPSIS
-`l2traceroute [options] <start_node> <end_noderange>`  
-
-## DESCRIPTION  
-**l2traceroute** is a command that returns the layer 2 route for the configered interfaces in nodeattrib.
-It can also be used with the -i and -e options to check against specific interfaces on the endpoints. 
-
-
-## PREREQUISITES
-**l2traceroute** the net.<interface>.switch attributes have to be set on the end points if endpoint is not a switch
-
-
-## OPTIONS
-* ` -e` EFACE, --eface=INTERFACE
-   interface to check against for the second end point 
-* ` -i` INTERFACE, --interface=INTERFACE
-   interface to check against for the first end point  
-* ` -c` CUMULUS, --cumulus=CUMULUS
-   return layer 2 route through cumulus switches only 
-* `-h`, `--help`:  
-  Show help message and exit      
-
-   
-## EXAMPLES
- * Checking route between two nodes:  
-  `# l2traceroute_client n244 n1851`  
-  `n244 to n1851: ['switch114']`    
-    
-* Checking route from one node to multiple nodes:  
-  `# l2traceroute_client n244 n1833,n1851`    
-  `n244 to n1833: ['switch114', 'switch7', 'switch32', 'switch253', 'switch85', 'switch72', 'switch21', 'switch2', 'switch96', 'switch103', 'switch115']
-   n244 to n1851: ['switch114']`  
-
-
- 
-

confluent_client/doc/man/nodefirmware.ronn

@@ -3,7 +3,7 @@ nodefirmware(8) -- Report firmware information on confluent nodes
 
 ## SYNOPSIS
 
-`nodefirmware <noderange> [list][update [--backup <file>]]|[<components>]`
+`nodefirmware <noderange> [list][updatestatus][update [--backup <file>]]|[<components>]`
 
 ## DESCRIPTION
 
@@ -17,6 +17,9 @@ not be relevant to redfish.  Additionally, the Lenovo XCC makes certain
 information available over IPMI that is not otherwise available (for example
 the FPGA version where applicable).
 
+The updatestatus argument will describe the state of firmware updates on the
+nodes.
+
 In the update form, it accepts a single file and attempts to update it using
 the out of band facilities.  Firmware updates can end in one of three states:
 

confluent_client/doc/man/nodel2traceroute.ronn

@@ -0,0 +1,42 @@
+nodel2traceroute(8) -- returns the layer 2 route through an Ethernet network managed by confluent given 2 end points.
+==============================
+## SYNOPSIS
+`nodel2traceroute [options] <start_node> <end_noderange>`  
+
+## DESCRIPTION  
+**nodel2traceroute** is a command that returns the layer 2 route for the configered interfaces in nodeattrib.
+It can also be used with the -i and -e options to check against specific interfaces on the endpoints. If the 
+--interface or --eface option are not used then the command will check for routes against all the defined 
+interfaces in nodeattrib (net.*.switch) for the nodes.
+
+
+
+## PREREQUISITES
+**nodel2traceroute** the net.<interface>.switch attributes have to be set on the end points if endpoint is not a switch
+
+
+## OPTIONS
+* ` -e` EFACE, --eface=INTERFACE
+   interface to check against for the second end point or end points if using checking against multiple nodes 
+* ` -i` INTERFACE, --interface=INTERFACE
+   interface to check against for the first end point  
+* ` -c` CUMULUS, --cumulus=CUMULUS
+   return layer 2 route through cumulus switches only 
+* `-h`, `--help`:  
+  Show help message and exit      
+
+   
+## EXAMPLES
+ * Checking route between two nodes:  
+  `# nodel2traceroute n244 n1851`  
+  `n244 to n1851: ['switch114']`    
+    
+* Checking route from one node to multiple nodes:  
+  `# nodel2traceroute n244 n1833,n1851`    
+  `n244 to n1833: ['switch114', 'switch7', 'switch32', 'switch253', 'switch85', 'switch72', 'switch21', 'switch2', 'switch96', 'switch103', 'switch115']
+   n244 to n1851: ['switch114']`  
+
+
+
+ 
+

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -14,6 +14,8 @@ import ssl
 import sys
 import struct
 import time
+import re
+import json
 
 class InvalidApiKey(Exception):
     pass
@@ -89,8 +91,11 @@ def scan_confluents():
                 confluentuuid = line.split(': ')[1]
                 msg += '/confluentuuid=' + confluentuuid
                 break
-    with open('/sys/devices/virtual/dmi/id/product_uuid') as uuidin:
-        msg += '/uuid=' + uuidin.read().strip()
+    try:
+        with open('/sys/devices/virtual/dmi/id/product_uuid') as uuidin:
+            msg += '/uuid=' + uuidin.read().strip()
+    except Exception:
+        pass
     for addrf in glob.glob('/sys/class/net/*/address'):
         with open(addrf) as addrin:
             hwaddr = addrin.read().strip()
@@ -412,8 +417,92 @@ class HTTPSClient(client.HTTPConnection, object):
                     self.node, [self.host], errout=self.errout)
         raise Exception(rsp.read())
 
+def get_current_vmnic_vswitch():
+    uplinkmatch = re.compile(r'^\s*Uplinks:\s*(.*)')
+    switchmatch = re.compile(r'^\s*Name:\s*(.*)')
+
+    vswinfo = subprocess.check_output(['localcli', 'network', 'vswitch', 'standard', 'list']).decode()
+
+    vmnic = None
+    vswitch_name = None
+
+    for info in vswinfo.split('\n'):
+        name_match = switchmatch.match(info)
+        if name_match:
+            vswitch_name = name_match.group(1).strip()
+
+        upinfo = uplinkmatch.match(info)
+        if upinfo:
+            vmnic = upinfo.group(1).strip()
+            if vmnic and 'vusb0' not in vmnic:
+                return vswitch_name, vmnic
+
+    return vswitch_name, vmnic
+
+def get_available_nics():
+    nicinfo = subprocess.check_output(['localcli', 'network', 'nic', 'list']).decode('utf8').split('\n')
+    available_nics = {}
+
+    # Skip headers and separators
+    parsing_started = False
+    for line in nicinfo:
+        if re.match(r'^-+', line):
+            parsing_started = True
+            continue
+        if not parsing_started or not line.strip():
+            continue
+
+        parts = re.split(r'\s{2,}', line.strip())
+        if len(parts) >= 5:
+            nic_name = parts[0]
+            nic_status = parts[4]  # "Link Status" este al 5-lea câmp
+            available_nics[nic_name] = nic_status
+
+    return available_nics
+
+def is_esxi():
+    return os.path.isdir("/etc/vmware")
+
+def fix_vswitch():
+    if is_esxi():
+        start_time = time.time()
+        while True:
+
+            current_vswitch, current_vmnic = get_current_vmnic_vswitch()
+            if current_vswitch is None:
+                raise RuntimeError("Panic: current vswitch is None")
+            if current_vmnic is None:
+                raise RuntimeError("Panic: current vmnic is None")
+
+            available_nics = get_available_nics()
+
+            if current_vmnic and available_nics.get(current_vmnic) == 'Up':
+                break
+
+            new_vmnic = next((nic for nic, status in available_nics.items() if status == 'Up'), None)
+
+            if new_vmnic and new_vmnic != current_vmnic:
+                subprocess.check_call(['localcli', 'network', 'vswitch', 'standard', 'uplink', 'remove',
+                                       '--uplink-name', current_vmnic, '--vswitch-name', current_vswitch])
+                subprocess.check_call(['localcli', 'network', 'vswitch', 'standard', 'uplink', 'add',
+                                       '--uplink-name', new_vmnic, '--vswitch-name', current_vswitch])
+            elif not new_vmnic:
+                if time.time() - start_time > 300:
+                    break
+            time.sleep(5)
+
+        time.sleep(5)
+
 if __name__ == '__main__':
     data = None
+
+    if '-f' in sys.argv:
+        try:
+            fix_vswitch()
+        except Exception as e:
+            print(f"fix_vswitch() error: {e}")
+        sys.argv.remove('-f')
+        sys.exit(0)
     usejson = False
     if '-j' in sys.argv:
         usejson = True
@@ -468,6 +557,7 @@ if __name__ == '__main__':
                 outf.write(chunk)
                 chunk = reader.read(16384)
         sys.exit(0)
+
     client = HTTPSClient(usejson, errout=errout, phmac=phmac, checkonly=checkonly)
     if waitfor:
         status = 201

confluent_osdeploy/common/profile/scripts/autoconsole

@@ -0,0 +1,117 @@
+#!/usr/bin/python3
+
+# This script evaluates whether firmware redirection is likely. It uses three cues:
+# - Does the system offer up SPCR? This would indicate that the firmware is doing serial output.
+#   Otherwise, there's no indication that the firmware cares about serial console.
+# - Is the system EFI?  BIOS implementations may not intercept text draw calls after POST exit,
+#   thus even when BIOS tells us serial port is in use, it may not be doing anything when
+#   grub would be running
+# - Is the serial port connected? In the event that firmware indicates serial port, but
+#   serial port is not reporting DCD, then it doesn't look like a comfortable enough scenario
+
+import fcntl
+import os
+import os.path
+import struct
+import subprocess
+import termios
+
+
+addrtoname = {
+    0x3f8: '/dev/ttyS0',
+    0x2f8: '/dev/ttyS1',
+    0x3e8: '/dev/ttyS2',
+    0x2e8: '/dev/ttyS3',
+}
+speedmap = {
+    0: None,
+    3: 9600,
+    4: 19200,
+    6: 57600,
+    7: 115200,
+}
+
+termiobaud = {
+    9600: termios.B9600,
+    19200: termios.B19200,
+    57600: termios.B57600,
+    115200: termios.B115200,
+}
+
+
+def deserialize_grub_rh():
+    if 'console=ttyS' in open('/proc/cmdline').read():
+        return None # User manually indicated serial config
+        # they own the grub behavior too for now
+    grublines = []
+    with open('/etc/default/grub') as grubin:
+        grublines = grubin.read().split('\n')
+    with open('/etc/default/grub', 'w') as grubout:
+        for grubline in grublines:
+            if grubline.startswith('GRUB_TERMINAL'):
+                grubline = grubline.replace('serial ', '')
+            grubout.write(grubline + '\n')
+    subprocess.check_call(['grub2-mkconfig', '-o', '/boot/grub2/grub.cfg'])
+
+def fixup_ubuntu_grub_serial():
+    # Ubuntu aggressively tries to graphics up
+    # grub. We will counter that for serial
+    # They also aggressively hide UI and
+    # block ability to interject. We will
+    # compromise and lean on nodeboot <node> setup
+    # as a means to give someone reasonable shot at
+    # the short timeout
+    with open('/etc/default/grub') as grubin:
+        grublines = grubin.read().split('\n')
+    with open('/etc/default/grub', 'w') as grubout:
+        for grubline in grublines:
+            if grubline.startswith('GRUB_TIMEOUT_STYLE=hidden'):
+                grubline = 'GRUB_TIMEOUT_STYLE=menu'
+            elif grubline.startswith('GRUB_TIMEOUT=0'):
+                grubline = 'GRUB_TIMEOUT=2'
+            elif grubline.startswith('#GRUB_TERMINAL=console'):
+                grubline = grubline.replace('#', '')
+            grubout.write(grubline + '\n')
+    subprocess.check_call(['update-grub'])
+
+def get_serial_config():
+    if not os.path.exists('/sys/firmware/efi'):
+        return None
+    spcr = open("/sys/firmware/acpi/tables/SPCR", "rb")
+    spcr = bytearray(spcr.read())
+    if spcr[8] != 2 or spcr[36] != 0 or spcr[40] != 1:
+        return None
+    address = struct.unpack('<Q', spcr[44:52])[0]
+    tty = None
+    try:
+        tty = addrtoname[address]
+    except KeyError:
+        return None
+    retval = { 'tty': tty }
+    try:
+        retval['speed'] = speedmap[spcr[58]]
+    except KeyError:
+        return None
+    if retval['speed']:
+        ttyf = os.open(tty, os.O_RDWR | os.O_NOCTTY)
+        currattr = termios.tcgetattr(ttyf)
+        currattr[4:6] = [0, termiobaud[retval['speed']]]
+        termios.tcsetattr(ttyf, termios.TCSANOW, currattr)
+    retval['connected'] = bool(struct.unpack('<I', fcntl.ioctl(
+        ttyf, termios.TIOCMGET, '\x00\x00\x00\x00'))[0] & termios.TIOCM_CAR)
+    os.close(ttyf)
+    return retval
+
+def main():
+    autoconscfg = get_serial_config()
+    if not autoconscfg or not autoconscfg['connected']:
+        return
+    if os.path.exists('/etc/redhat-release'):  # redhat family
+        deserialize_grub_rh()
+    elif os.path.exists('/etc/os-release'):
+        with open('/etc/os-release') as osr:
+            if 'Ubuntu' in osr.read():
+                fixup_ubuntu_grub_serial()
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/common/profile/scripts/confignet

@@ -226,7 +226,11 @@ class WickedManager(object):
             self.cfgbydev[devname] = currcfg
             for cfg in open(ifcfg).read().splitlines():
                 cfg = cfg.split('#', 1)[0]
-                kv = ' '.join(shlex.split(cfg)).split('=', 1)
+                try:
+                    kv = ' '.join(shlex.split(cfg)).split('=', 1)
+                except Exception:
+                    # unparseable line, likely having something we can't handle
+                    del self.cfgbydev[devname]
                 if len(kv) != 2:
                     continue
                 k, v = kv
@@ -297,6 +301,12 @@ class WickedManager(object):
 
 
 class NetworkManager(object):
+    bondtypes = {
+            'lacp': '802.3ad',
+            'loadbalance': 'balance-alb',
+            'roundrobin': 'balance-rr',
+            'activebackup': 'active-backup',
+    }
     def __init__(self, devtypes, deploycfg):
         self.deploycfg = deploycfg
         self.connections = {}
@@ -348,7 +358,8 @@ class NetworkManager(object):
                     bondcfg[stg] = deats[stg]
         if member in self.uuidbyname:
             subprocess.check_call(['nmcli', 'c', 'del', self.uuidbyname[member]])
-        subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond-slave', 'master', team, 'con-name', member, 'connection.interface-name', member])
+        devtype = self.devtypes.get(member, 'bond-slave')
+        subprocess.check_call(['nmcli', 'c', 'add', 'type', devtype, 'master', team, 'con-name', member, 'connection.interface-name', member])
         if bondcfg:
             args = []
             for parm in bondcfg:
@@ -356,7 +367,7 @@ class NetworkManager(object):
                 args.append(bondcfg[parm])
             subprocess.check_call(['nmcli', 'c', 'm', team] + args)
 
-    def apply_configuration(self, cfg):
+    def apply_configuration(self, cfg, lastchance=False):
         cmdargs = {}
         cmdargs['connection.autoconnect'] = 'yes'
         stgs = cfg['settings']
@@ -396,9 +407,9 @@ class NetworkManager(object):
             for arg in cmdargs:
                 cargs.append(arg)
                 cargs.append(cmdargs[arg])
-            if stgs['team_mode'] == 'lacp':
-                stgs['team_mode'] = '802.3ad'
-            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'mode={}'.format(stgs['team_mode'])] + cargs)
+            if stgs['team_mode'] in self.bondtypes:
+                stgs['team_mode'] = self.bondtypes[stgs['team_mode']]
+            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'miimon=100,mode={}'.format(stgs['team_mode'])] + cargs)
             for iface in cfg['interfaces']:
                 self.add_team_member(cname, iface)
             subprocess.check_call(['nmcli', 'c', 'u', cname])
@@ -407,8 +418,9 @@ class NetworkManager(object):
             iname = list(cfg['interfaces'])[0]
             ctype = self.devtypes.get(iname, None)
             if not ctype:
-                sys.stderr.write("Warning, no device found for interface_name ({0}), skipping setup\n".format(iname))
-                return
+                if lastchance:
+                    sys.stderr.write("Warning, no device found for interface_name ({0}), skipping setup\n".format(iname))
+                return 1
             if stgs.get('vlan_id', None):
                 vlan = stgs['vlan_id']
                 if ctype == 'infiniband':
@@ -532,8 +544,21 @@ if __name__ == '__main__':
         nm = NetworkManager(devtypes, dc)
     elif os.path.exists('/usr/sbin/wicked'):
         nm = WickedManager()
+    retrynics = []
     for netn in netname_to_interfaces:
-        nm.apply_configuration(netname_to_interfaces[netn])
+        redo = nm.apply_configuration(netname_to_interfaces[netn])
+        if redo == 1:
+            retrynics.append(netn)
+    if retrynics:
+        idxmap, devtypes = map_idx_to_name()
+        if os.path.exists('/usr/sbin/netplan'):
+            nm = NetplanManager(dc)
+        if os.path.exists('/usr/bin/nmcli'):
+            nm = NetworkManager(devtypes, dc)
+        elif os.path.exists('/usr/sbin/wicked'):
+            nm = WickedManager()
+        for netn in retrynics:
+            nm.apply_configuration(netname_to_interfaces[netn], lastchance=True)
     if havefirewall:
         subprocess.check_call(['systemctl', 'start', 'firewalld'])
     await_tentative()

confluent_osdeploy/common/profile/scripts/setupssh

@@ -10,6 +10,15 @@ for pubkey in /etc/ssh/ssh_host*key.pub; do
     rm $certfile
     confluentpython $confapiclient /confluent-api/self/sshcert $pubkey -o $certfile
 done
+if [ -d /etc/ssh/sshd_config.d/ -a ! -e /etc/ssh/sshd_config.d/90-confluent.conf ]; then
+	for cert in /etc/ssh/ssh*-cert.pub; do
+		echo HostCertificate $cert >> /etc/ssh/sshd_config.d/90-confluent.conf
+	done
+	echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/90-confluent.conf
+	echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/90-confluent.conf
+	echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/90-confluent.conf
+fi
+
 TMPDIR=$(mktemp -d)
 cd $TMPDIR
 confluentpython $confapiclient /confluent-public/site/initramfs.tgz -o initramfs.tgz

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -28,11 +28,15 @@ This contains support utilities for enabling deployment of x86_64 architecture s
 #cp start_root urlmount ../stateless-bin/
 #cd ..
 ln -s el8 el9
-for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9; do
+cp -a el8 el10
+mv el10/initramfs/usr el10/initramfs/var
+for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9 el10; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
         cp -a ../${os}bin/opt .
+    elif [ $os = el10 ]; then
+        cp -a ../el9bin/opt .
     else
         cp -a ../el8bin/opt .
     fi
@@ -73,12 +77,14 @@ cd ..
 cp -a esxi7out esxi6out
 cp -a esxi7 esxi6
 cp -a esxi7out esxi8out
+cp -a esxi7out esxi9out
 cp -a esxi7 esxi8
+cp -a esxi7 esxi9
 
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu18.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do
+for os in rhvh4 el7 el8 el9 el10 genesis suse15 ubuntu20.04 ubuntu18.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 esxi9 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs

confluent_osdeploy/el7-diskless/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/el8-diskless/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 echo -n "" >> /tmp/net.ifaces
 echo -n "" > /tmp/01-autocons.devnode
-cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
+BUNDLENAME=/etc/pki/tls/certs/ca-bundle.crt
+if [ ! -e "$BUNDLENAME" ]; then
+    BUNDLENAME=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+    mkdir -p /etc/pki/tls/certs
+    ln -s $BUNDLENAME /etc/pki/tls/certs/ca-bundle.crt
+fi
+cat /tls/*.0 >> $BUNDLENAME
 if ! grep console= /proc/cmdline >& /dev/null; then
     autocons=$(/opt/confluent/bin/autocons)
     if [ -n "$autocons" ]; then

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 BUNDLENAME=/sysroot/etc/pki/tls/certs/ca-bundle.crt
+if [ ! -e "$BUNDLENAME" ]; then
+    BUNDLENAME=/sysroot/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+fi
 while [ -h $BUNDLENAME ]; do
     BUNDLENAME=/sysroot/$(readlink $BUNDLENAME)
 done

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
-sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/; s/alma/AlmaLinux/' $2/profile.yaml
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/; s/alma/AlmaLinux/;s/fedora/Fedora Linux/' $2/profile.yaml
+if grep Fedora $2/profile.yaml > /dev/null; then
+	sed -i 's/@^minimal-environment/#/' $2/packagelist
+fi
 ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
 ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
 mkdir -p $2/boot/efi/boot

confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories

@@ -6,6 +6,7 @@ except ImportError:
 import importlib.util
 import importlib.machinery
 import sys
+import glob
 modloader = importlib.machinery.SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient')
 modspec = importlib.util.spec_from_file_location('apiclient', '/opt/confluent/bin/apiclient', loader=modloader)
 apiclient = importlib.util.module_from_spec(modspec)
@@ -41,6 +42,7 @@ try:
 except AttributeError:
     f = cStringIO.StringIO(cfgdata)
     c.readfp(f)
+gpgkeys = glob.glob('/etc/pki/rpm-gpg/RPM-GPG-KEY-*')
 for sec in c.sections():
     if sec.startswith('variant-'):
         try:
@@ -56,3 +58,6 @@ for sec in c.sections():
                 repopath = repopath[1:]
             repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath))
             repout.write('enabled=1\n')
+            if gpgkeys:
+                gpgkeyvals = ['file://{}'.format(x) for x in gpgkeys]
+                repout.write('gpgkey=' + ' '.join(gpgkeyvals) + '\n')

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/el8/profiles/default/scripts/post.sh

@@ -34,6 +34,11 @@ run_remote_python add_local_repositories
 # run_remote_python will use the appropriate python interpreter path to run the specified script
 # A post.custom is provided to more conveniently hold customizations, see the post.custom file.
 
+# This will do some serial console fixup for bad grub configuration when serial is firmware
+# managed. See script for details
+run_remote_python autoconsole
+
+
 # This will induce server side processing of the syncfile contents if
 # present
 run_remote_python syncfileclient

confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -292,7 +292,7 @@ if [[ $confluent_websrv == *:* ]] && [[ $confluent_websrv != "["* ]]; then
     confluent_websrv="[$confluent_websrv]"
 fi
 echo -n "Initializing ssh..."
-ssh-keygen -A
+ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -C '' -N ''
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     privfile=${pubkey%.pub}

confluent_osdeploy/el9-diskless/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -65,7 +65,11 @@ def get_image_metadata(imgpath):
                     continue
                 yield md
         else:
-            raise Exception('Installation from single part image not supported')
+            # plausible filesystem structure to apply to a nominally "diskless" image
+            yield {'mount': '/', 'filesystem': 'xfs', 'minsize': 39513563136, 'initsize': 954128662528, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/mapper/root', 'compressed_size': 27022069760}
+            yield {'mount': '/boot', 'filesystem': 'xfs', 'minsize': 232316928, 'initsize': 1006632960, 'flags': 'rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota', 'device': '/dev/nvme1n1p2', 'compressed_size': 171462656}
+            yield {'mount': '/boot/efi', 'filesystem': 'vfat', 'minsize': 7835648, 'initsize': 627900416, 'flags': 'rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro', 'device': '/dev/nvme1n1p1', 'compressed_size': 1576960}
+            #raise Exception('Installation from single part image not supported')
 
 class PartedRunner():
     def __init__(self, disk):
@@ -84,8 +88,17 @@ def fixup(rootdir, vols):
     for vol in vols:
         devbymount[vol['mount']] = vol['targetdisk']
     fstabfile = os.path.join(rootdir, 'etc/fstab')
-    with open(fstabfile) as tfile:
-        fstab = tfile.read().split('\n')
+    if os.path.exists(fstabfile):
+        with open(fstabfile) as tfile:
+            fstab = tfile.read().split('\n')
+    else:
+        #diskless image, need to invent fstab
+        fstab = [
+            "#ORIGFSTAB#/dev/mapper/root# /                       xfs     defaults        0 0",
+            "#ORIGFSTAB#UUID=aaf9e0f9-aa4d-4d74-9e75-3537620cfe23# /boot                   xfs     defaults        0 0",
+            "#ORIGFSTAB#UUID=C21D-B881#          /boot/efi               vfat    umask=0077,shortname=winnt 0 2",
+            "#ORIGFSTAB#/dev/mapper/swap# none                    swap    defaults        0 0",
+        ]
     while not fstab[0]:
         fstab = fstab[1:]
     if os.path.exists(os.path.join(rootdir, '.autorelabel')):
@@ -135,8 +148,10 @@ def fixup(rootdir, vols):
         newcfg = ifcfg.split('/')[-1]
         newcfg = os.path.join(rootdir, 'etc/NetworkManager/system-connections/{0}'.format(newcfg))
         shutil.copy2(ifcfg, newcfg)
-    shutil.rmtree(os.path.join(rootdir, 'etc/confluent/'))
-    shutil.copytree('/etc/confluent', os.path.join(rootdir, 'etc/confluent'))
+    rootconfluentdir = os.path.join(rootdir, 'etc/confluent/')
+    if os.path.exists(rootconfluentdir):
+        shutil.rmtree(rootconfluentdir)
+    shutil.copytree('/etc/confluent', rootconfluentdir)
     if policy:
         sys.stdout.write('Applying SELinux labeling...')
         sys.stdout.flush()
@@ -191,8 +206,24 @@ def fixup(rootdir, vols):
                     else:
                         newcfgparts.append(cfgpart)
                 loadentout.write(' '.join(newcfgparts) + '\n')
-    with open(grubsyscfg) as defgrubin:
-        defgrub = defgrubin.read().split('\n')
+    if os.path.exists(grubsyscfg):
+        with open(grubsyscfg) as defgrubin:
+            defgrub = defgrubin.read().split('\n')
+    else:
+        defgrub = [
+        'GRUB_TIMEOUT=5',
+        'GRUB_DISTRIBUTOR="$(sed ' + "'s, release .*$,,g'" + ' /etc/system-release)"',
+        'GRUB_DEFAULT=saved',
+        'GRUB_DISABLE_SUBMENU=true',
+        'GRUB_TERMINAL=""',
+        'GRUB_SERIAL_COMMAND=""',
+        'GRUB_CMDLINE_LINUX="crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M rd.lvm.lv=vg/root rd.lvm.lv=vg/swap"',
+        'GRUB_DISABLE_RECOVERY="true"',
+        'GRUB_ENABLE_BLSCFG=true',
+        ]
+    if not os.path.exists(os.path.join(rootdir, "etc/kernel/cmdline")):
+        with open(os.path.join(rootdir, "etc/kernel/cmdline"), "w") as cmdlineout:
+            cmdlineout.write("root=/dev/mapper/localstorage-root rd.lvm.lv=localstorage/root")
     with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
@@ -217,6 +248,12 @@ def fixup(rootdir, vols):
         grubcfg = grubcfg[:-1]
     if len(grubcfg) == 1:
         grubcfg = grubcfg[0]
+    elif not grubcfg:
+        grubcfg = '/boot/grub2/grub.cfg'
+        paths = glob.glob(os.path.join(rootdir, 'boot/efi/EFI/*'))
+        for path in paths:
+            with open(os.path.join(path, 'grub.cfg'), 'w') as stubgrubout:
+                stubgrubout.write("search --no-floppy --root-dev-only --fs-uuid --set=dev " + bootuuid + "\nset prefix=($dev)/grub2\nexport $prefix\nconfigfile $prefix/grub.cfg\n")
     else:
         for gcfg in grubcfg:
             rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
@@ -272,10 +309,19 @@ def fixup(rootdir, vols):
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
+
+    try:
+        os.makedirs(os.path.join(rootdir, 'opt/confluent/bin'))
+    except Exception:
+        pass
+    shutil.copy2('/opt/confluent/bin/apiclient', os.path.join(rootdir, 'opt/confluent/bin/apiclient'))
     #other network interfaces
 
 
 def had_swap():
+    if not os.path.exists('/etc/fstab'):
+        # diskless source, assume swap
+        return True
     with open('/etc/fstab') as tabfile:
         tabs = tabfile.read().split('\n')
         for tab in tabs:
@@ -440,6 +486,8 @@ def install_to_disk(imgpath):
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
             source = vol['mount'].replace('/', '_')
             source = '/run/imginst/sources/' + source
+            if not os.path.exists(source):
+                source = '/run/imginst/sources/_' + vol['mount']
             blankfsstat = os.statvfs('/run/imginst/targ')
             blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize
             sys.stdout.write('\nWriting {0}: '.format(vol['mount']))

confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage

@@ -41,6 +41,8 @@ echo "Port 22" >> /etc/ssh/sshd_config
 echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
 echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
 kill -HUP $(cat /run/sshd.pid)
+cp /sysroot/etc/pki/ca-trust/source/anchors/* /sysroot/run/imginst/targ/etc/pki/ca-trust/source/anchors/
+chroot /sysroot/run/imginst/targ update-ca-trust
 
 chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh"
 chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)"

confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh

@@ -2,7 +2,10 @@
 
 # This script is executed 'chrooted' into a cloned disk target before rebooting
 #
-
+if [ -f /etc/dracut.conf.d/diskless.conf ]; then
+    rm /etc/dracut.conf.d/diskless.conf
+fi
+for kver in /lib/modules/*; do kver=$(basename $kver); kernel-install add $kver /boot/vmlinuz-$kver; done
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
@@ -16,6 +19,7 @@ if [[ "$confluent_mgr" == *:* ]]; then
 fi
 export nodename confluent_mgr confluent_profile confluent_websrv
 . /etc/confluent/functions
+run_remote setupssh
 mkdir -p /var/log/confluent
 chmod 700 /var/log/confluent
 exec >> /var/log/confluent/confluent-post.log
@@ -40,6 +44,9 @@ run_remote_parts post.d
 # Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
 run_remote_config post.d
 
+# rebuild initrd, pick up new drivers if needed
+dracut -f /boot/initramfs-$(uname -r).img $(uname -r)
+
 curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_websrv/confluent-api/self/updatestatus
 
 kill $logshowpid

confluent_osdeploy/el9-diskless/profiles/default/scripts/sample/consoleredirect

@@ -0,0 +1,15 @@
+is_rhel=false
+
+if test -f /boot/efi/EFI/redhat/grub.cfg; then
+    grubcfg="/etc/default/grub"
+    is_rhel=true
+else
+    echo "Expected File missing: Check if os redhat"
+    exit
+fi
+
+# Working on Redhat
+if $is_rhel; then
+    sed -i '/^GRUB_TERMINAL/s/serial //' $grubcfg
+    grub2-mkconfig -o /boot/grub2/grub.cfg
+fi

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -6,6 +6,7 @@ if [ ! -f /var/run/vmware/show-esx-shell-login ]; then
     chvt 2
     /etc/init.d/ESXShell start
 fi
+/opt/confluent/bin/apiclient -f
 uuid=$(vsish -e get /hardware/bios/dmiInfo|grep -A15 UUID|sed -e 's/.*://'|sed -e ':a;N;$!ba;s/\n//g' | sed -e 's/ *0x//g')
 uuid=${uuid:0:8}-${uuid:8:4}-${uuid:12:4}-${uuid:16:4}-${uuid:20:12}
 kargs=$(vsish -e get /system/bootCmdLine|grep "command line:")

confluent_osdeploy/esxi7/profiles/hypervisor/profile.yaml

@@ -1,3 +1,3 @@
-label: Confluent installation of VMware ESXi %%VERSION%% Hypervisor
+label: VMware ESXi %%VERSION%% Hypervisor
 ostype: esxi
 kernelargs: runweasel

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -1,5 +1,14 @@
 #!/bin/bash
+mkdir -p /usr/libexec /run/sshd
+if [ ! -x /usr/libexec/platform-python ]; then
+       ln -s /usr/bin/python3 /usr/libexec/platform-python
+fi
 export LANG=en_US.utf8
+
+(while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
+
+
+
 mkdir -p /etc/confluent
 if ! grep console= /proc/cmdline >& /dev/null; then
     autocons=$(/opt/confluent/bin/autocons)
@@ -9,13 +18,9 @@ if ! grep console= /proc/cmdline >& /dev/null; then
     if [ ! -z "$autocons" ]; then
         echo "Using $(cat /tmp/01-autocons.conf)"
         (while :; do TERM=xterm-256color tmux a <> $autocons >&0 2>&1; done) &
-        (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
-    else
-        (while :; do TERM=linux tmux a <> /dev/console >&0 2>&1; done) &
     fi
 else
     (while :; do TERM=xterm-256color tmux a <> /dev/console >&0 2>&1; done) &
-    (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
 fi
 (while :; do TERM=linux tmux <> /dev/tty2 >&0 2>&1; done) &
 echo -n "udevd: "
@@ -29,6 +34,13 @@ modprobe ib_umad
 modprobe hfi1
 modprobe mlx5_ib
 echo "done"
+if [ -x /usr/bin/seatd-launch -a -x /usr/bin/sway ]; then
+    export XDG_RUNTIME_DIR=/run/users/0
+    mkdir -p $XDG_RUNTIME_DIR
+    sed -i '/^output /d' /etc/sway/config
+    echo 'exec foot -t xterm -T Terminal tmux a' > /etc/sway/config.d/genesis
+    (while :; do seatd-launch sway <> /dev/tty1 >& /dev/null; done) &
+fi
 cat > /etc/ssh/sshd_config << EOF
 Port 22
 Port 3389
@@ -43,18 +55,25 @@ mkdir -p /etc/pki/tls/certs
 cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
 TRIES=0
 touch /etc/confluent/confluent.info
+TRIES=5
+echo -n "Waitiing for disks..."
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
+    sleep 1
+    TRIES=$((TRIES - 1))
+done
+echo "Done"
 if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     mkdir -p /media/ident
     mount /dev/disk/by-label/CNFLNT_IDNT /media/ident
     if [ -e /media/ident/genesis_bootstrap.sh ]; then
-        exec sh /media/ident/genesis_bootstrap.sh
+        exec bash /media/ident/genesis_bootstrap.sh
     fi
 fi
 if [ -e /dev/disk/by-label/GENESIS-X86 ]; then
     mkdir -p /media/genesis
     mount /dev/disk/by-label/GENESIS-X86 /media/genesis
     if [ -e /media/genesis/genesis_bootstrap.sh ]; then
-        exec sh /media/genesis/genesis_bootstrap.sh
+        exec bash /media/genesis/genesis_bootstrap.sh
     fi
 fi
 cd /sys/class/net

confluent_osdeploy/genesis/initramfs/usr/lib/dracut/hooks/cmdline/10-genesis.sh

@@ -2,15 +2,17 @@ root=1
 rootok=1
 netroot=genesis
 clear
+mount -t cgroup2 cgroup2 /sys/fs/cgroup
+mount -t efivarfs efivarfs /sys/firmware/efi/efivars
 echo PS1="'"'[genesis running on \H \w]$ '"'" >> ~/.bashrc
 echo PS1="'"'[genesis running on \H \w]$ '"'" >> ~/.bash_profile
 mkdir -p /etc/ssh
 mkdir -p /var/tmp/
 mkdir -p /var/empty/sshd
-sed -i '/^root:x/d' /etc/passwd
+sed -i '/^root:/d' /etc/passwd
 echo root:x:0:0::/:/bin/bash >> /etc/passwd
 echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
-tmux new-session -d sh /opt/confluent/bin/rungenesis
+tmux new-session -d bash /opt/confluent/bin/rungenesis
 while :; do
 	sleep 86400
 done

confluent_osdeploy/genesis/profiles/default/scripts/configbmc

@@ -210,16 +210,28 @@ def set_port_tsm(s, port, model):
 
 
 def set_port_xcc(s, port, model):
+    if '_' in port:
+        port_type = port.split('_')[0]
+        port_number = port.split('_')[1]  # this could be the number or the form sfp28 || rj45
+        port = port_type
     oport = port
     if port.lower() == 'dedicated':
         port = b'\x01'
     elif port.lower() in ('ml2', 'ocp'):
         port = b'\x02\x00'
-    elif port.lower() == 'lom':
-        if model == '7x58':
-            port = b'\x00\x02'
+    elif port.lower() == 'lom':  # potentially have to change to port_form.lower() == 'sfp28 || or rj45  // if it is two port sfp28 it is usually the first port that is ncsi enabled'
+        if port_number == '1' or port_number.lower() == 'sfp28':
+            port = b'\x05\x00'
+        elif port_number == '3' or port_number.lower() == 'rj45':
+            if model == '7x58':
+                port = b'\x00\x02'
+            else:
+                port = b'\x00\x00'
         else:
-            port = b'\x00\x00'
+            if model == '7x58':
+                port = b'\x00\x02'
+            else:
+                port = b'\x00\x00'
     else:
         port = port.split(' ')
         port = bytes(bytearray([int(x) for x in port]))

confluent_osdeploy/genesis/profiles/default/scripts/functions

@@ -23,6 +23,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export nodename confluent_mgr confluent_profile
 }
 
 fetch_remote() {

confluent_osdeploy/genesis/profiles/default/scripts/onboot.sh

@@ -10,6 +10,9 @@
 # present
 run_remote_python syncfileclient
 
+run_remote_parts onboot.d
+
+
 # Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
 run_remote_config onboot
 

confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/suse15-diskless/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/suse15-diskless/profiles/default/scripts/imageboot.sh

@@ -140,4 +140,5 @@ mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 mv /lib/firmware /lib/firmware-ramfs
 ln -s /sysroot/lib/firmware /lib/firmware
+chroot /sysroot chkstat --system --set --noheader > /dev/null
 exec /opt/confluent/bin/start_root

confluent_osdeploy/suse15/profiles/hpc/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/suse15/profiles/hpc/scripts/sample/consoleredirect

@@ -1,16 +1,11 @@
 is_suse=false
-is_rhel=false
 
-if test -f /boot/efi/EFI/redhat/grub.cfg; then
-    grubcfg="/boot/efi/EFI/redhat/grub.cfg"
-    grub2-mkconfig -o $grubcfg
-    is_rhel=true
-elif test -f /boot/efi/EFI/sle_hpc/grub.cfg; then
+if test -f /boot/efi/EFI/sle_hpc/grub.cfg; then
     grubcfg="/boot/efi/EFI/sle_hpc/grub.cfg"
     grub2-mkconfig -o $grubcfg
     is_suse=true
 else
-    echo "Expected File missing: Check if os sle_hpc or redhat"
+    echo "Expected File missing: Check if os sle_hpc"
     exit
 fi
 
@@ -42,8 +37,3 @@ if $is_suse; then
     done
     sed -i 's,^terminal,#terminal,' $grubcfg
 fi
-
-# Working on Redhat
-if $is_rhel; then
-    sed -i 's,^serial,#serial, ; s,^terminal,#terminal,' $grubcfg
-fi

confluent_osdeploy/suse15/profiles/server/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -55,25 +55,28 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
         if [ ! -z "$v4gw" ]; then
             setdebopt netcfg/get_gateway $v4gw string
         fi
-        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
-            ip addr add dev $NICGUESS $v4addr
-            if [ ! -z "$v4gw" ]; then
-                ip route add default via $v4gw
-            fi
-            for dsrv in $deploysrvs; do
-                if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
-                    deploysrvs=$dsrv
-                    NIC=$NICGUESS
-                    setdebopt netcfg/choose_interface $NIC select
+        NIC=""
+        while [ -z "$NIC" ]; do
+            for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+                ip addr add dev $NICGUESS $v4addr
+                if [ ! -z "$v4gw" ]; then
+                    ip route add default via $v4gw
+                fi
+                for dsrv in $deploysrvs; do
+                    if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                        deploysrvs=$dsrv
+                        NIC=$NICGUESS
+                        setdebopt netcfg/choose_interface $NIC select
+                        break
+                    fi
+                done
+                if [ -z "$NIC" ]; then
+                    ip -4 a flush dev $NICGUESS
+                else
                     break
                 fi
             done
-            if [ -z "$NIC" ]; then
-                ip -4 a flush dev $NICGUESS
-            else
-                break
-            fi
-        done 
+        done
         #TODO: nameservers
     elif [ "$v4cfgmeth" = "dhcp" ]; then
         setdebopt netcfg/disable_dhcp false boolean

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/functions

@@ -50,6 +50,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_mgr confluent_profile nodename
 }
 
 fetch_remote() {

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/ubuntu20.04/initramfs/scripts/init-premount/confluent

@@ -1,72 +1,79 @@
 cd /sys/class/net
+cp /tls/* /etc/ssl/certs/
 for nic in *; do
     ip link set $nic up
 done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-TRIES=5
-while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
-    sleep 2
-    TRIES=$((TRIES - 1))
-done
-if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
-    tmnt=/tmp/idntmnt
-    mkdir -p /tmp/identdata/
-    mkdir -p $tmnt
-    tcfg=/tmp/idnttmp
-    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
-    cp -a $tmnt/* /tmp/identdata/
-    cd $tmnt
-    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
-    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
-    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
-    autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    . /scripts/functions
-    if [ "$autoconfigmethod" = "static" ]; then
-        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
-        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
-        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
-        if [ "$MYGW" = "null" ]; then
-            MYGW=""
-        fi
-        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
-        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
-            ip addr add dev $NICGUESS $v4addr
-            if [ ! -z "$MYGW" ]; then
-                ip route add default via $MYGW
+while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+    if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+        tmnt=/tmp/idntmnt
+        mkdir -p /tmp/identdata/
+        mkdir -p $tmnt
+        tcfg=/tmp/idnttmp
+        mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+        cp -a $tmnt/* /tmp/identdata/
+        cd $tmnt
+        deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+        sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+        autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+        autoconfigmethod=${autoconfigmethod#ipv4_method: }
+        . /scripts/functions
+        if [ "$autoconfigmethod" = "static" ]; then
+            MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+            v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+            MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+            if [ "$MYGW" = "null" ]; then
+                MYGW=""
             fi
+            MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+            NIC=""
+            while [ -z "$NIC" ]; do
+                for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+                    ip addr add dev $NICGUESS $v4addr
+                    if [ ! -z "$MYGW" ]; then
+                        ip route add default via $MYGW
+                    fi
+                    for dsrv in $deploysrvs; do
+                        if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                            deploysrvs=$dsrv
+                            NIC=$NICGUESS
+                            break
+                        fi
+                    done
+                    if [ -z "$NIC" ]; then
+                        ip -4 a flush dev $NICGUESS
+                    else
+                        break
+                    fi
+                done
+            done
+            ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+            echo $NIC > /tmp/autodetectnic
+        else
+            configure_networking
             for dsrv in $deploysrvs; do
                 if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
                     deploysrvs=$dsrv
-                    NIC=$NICGUESS
                     break
                 fi
             done
-            if [ -z "$NIC" ]; then
-                ip -4 a flush dev $NICGUESS
-            else
-                break
-            fi
-        done
-        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
-        echo $NIC > /tmp/autodetectnic
+        fi
+        MGR=$deploysrvs
+        NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+        echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+        echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+        echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+        hmackeyfile=/tmp/cnflnthmackeytmp
+        echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+        cd -
+        umount $tmnt
     else
-        configure_networking
-    fi
-    MGR=$deploysrvs
-    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
-    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
-    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
-    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
-    hmackeyfile=/tmp/cnflnthmackeytmp
-    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
-    cd -
-    umount $tmnt
-else
-    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
         /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
-    done
+    fi
+done
+if [ -z "$MGR" ]; then
     MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
 fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
@@ -79,8 +86,25 @@ if [ ! -z "$cons" ]; then
 fi
 echo "Preparing to deploy $osprofile from $MGR"
 echo $osprofile > /custom-installation/confluent/osprofile
-echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
-fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso"
+mv /usr/bin/openssl /usr/bin/ossl
+cat > /usr/bin/openssl << 'EOF'
+#!/bin/sh
+AMENDARGS=0
+nargs=""
+for arg in $*; do
+    if [ "$arg" == "-servername"  -o "$arg" == "-verify" ]; then
+        AMENDARGS=1
+    fi
+    if [ "$AMENDARGS" == "1" ]; then
+        arg=$(echo $arg|sed -e 's/:443$//' -e 's/\[//' -e 's/\]//' -e 's/%.*//')
+    fi
+    nargs="$nargs $arg"
+done
+exec /usr/bin/ossl $nargs
+EOF
+chmod +x /usr/bin/openssl
+echo URL=https://${MGR}:443/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
+fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=https://${MGR}:443/confluent-public/os/$osprofile/distribution/install.iso"
 if [ ! -z "$cons" ]; then
     fcmdline="$fcmdline console=${cons#/dev/}"
 fi

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent

@@ -28,7 +28,15 @@ if [ -e /tmp/cnflnthmackeytmp ]; then
     DEVICE=$(cat /tmp/autodetectnic)
     IP=done
 else
+    APIKEY=
     chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+    APIKEY=$(cat /root/custom-installation/confluent/confluent.apikey)
+    while [ -z "$APIKEY" ]; do
+	echo "Failure trying to get confluent node token registered, check nodedeploy status, retrying in 5 seconds..."
+        sleep 5
+        chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+        APIKEY=$(cat /root/custom-installation/confluent/confluent.apikey)
+    done
     MGR=[$MGR]
     nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
     nic=$(ip link |grep ^$nic:|awk '{print $2}')

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -1,3 +1,4 @@
+cp /tls/* /etc/ssl/certs/
 cd /sys/class/net
 for nic in *; do
     ip link set $nic up
@@ -5,68 +6,75 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-TRIES=5
-while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
-    sleep 2
-    TRIES=$((TRIES - 1))
-done
-if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
-    tmnt=/tmp/idntmnt
-    mkdir -p /tmp/identdata/
-    mkdir -p $tmnt
-    tcfg=/tmp/idnttmp
-    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
-    cp -a $tmnt/* /tmp/identdata/
-    cd $tmnt
-    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
-    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
-    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
-    autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    . /scripts/functions
-    if [ "$autoconfigmethod" = "static" ]; then
-        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
-        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
-        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
-        if [ "$MYGW" = "null" ]; then
-            MYGW=""
-        fi
-        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
-        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
-            ip addr add dev $NICGUESS $v4addr
-            if [ ! -z "$MYGW" ]; then
-                ip route add default via $MYGW
+MGR=""
+while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+    if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+        tmnt=/tmp/idntmnt
+        mkdir -p /tmp/identdata/
+        mkdir -p $tmnt
+        tcfg=/tmp/idnttmp
+        mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+        cp -a $tmnt/* /tmp/identdata/
+        cd $tmnt
+        deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+        sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+        autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+        autoconfigmethod=${autoconfigmethod#ipv4_method: }
+        . /scripts/functions
+        if [ "$autoconfigmethod" = "static" ]; then
+            MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+            v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+            MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+            if [ "$MYGW" = "null" ]; then
+                MYGW=""
             fi
+            MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+            NIC=""
+            while [ -z "$NIC" ]; do
+                for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+                    ip addr add dev $NICGUESS $v4addr
+                    if [ ! -z "$MYGW" ]; then
+                        ip route add default via $MYGW
+                    fi
+                    for dsrv in $deploysrvs; do
+                        if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                            deploysrvs=$dsrv
+                            NIC=$NICGUESS
+                            break
+                        fi
+                    done
+                    if [ -z "$NIC" ]; then
+                        ip -4 a flush dev $NICGUESS
+                    else
+                        break
+                    fi
+                done
+            done
+            ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+            echo $NIC > /tmp/autodetectnic
+        else
+            configure_networking
             for dsrv in $deploysrvs; do
                 if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
                     deploysrvs=$dsrv
-                    NIC=$NICGUESS
                     break
                 fi
             done
-            if [ -z "$NIC" ]; then
-                ip -4 a flush dev $NICGUESS
-            else
-                break
-            fi
-        done
-        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
-        echo $NIC > /tmp/autodetectnic
+        fi
+        MGR=$deploysrvs
+        NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+        echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+        echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+        echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+        hmackeyfile=/tmp/cnflnthmackeytmp
+        echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+        cd -
+        umount $tmnt
     else
-        configure_networking
-    fi
-    MGR=$deploysrvs
-    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
-    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
-    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
-    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
-    hmackeyfile=/tmp/cnflnthmackeytmp
-    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
-    cd -
-    umount $tmnt
-else
-    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
         /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
-    done
+    fi
+done
+if [ -z "$MGR" ]; then
     MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
 fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
@@ -82,8 +90,25 @@ echo $osprofile > /custom-installation/confluent/osprofile
 . /etc/os-release
 DIRECTISO=$(blkid -t TYPE=iso9660 |grep -Ei ' LABEL="Ubuntu-Server '$VERSION_ID)
 if [ -z "$DIRECTISO" ]; then
-    echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
-    fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso"
+    mv /usr/bin/openssl /usr/bin/ossl
+    cat > /usr/bin/openssl << 'EOF'
+#!/bin/sh
+AMENDARGS=0
+nargs=""
+for arg in $*; do
+    if [ "$arg" == "-servername" -o "$arg" == "-verify" ]; then
+        AMENDARGS=1
+    fi
+    if [ "$AMENDARGS" == "1" ]; then
+        arg=$(echo $arg|sed -e 's/:443$//' -e 's/\[//' -e 's/\]//' -e 's/%.*//')
+    fi
+    nargs="$nargs $arg"
+done
+exec /usr/bin/ossl $nargs
+EOF
+    chmod +x /usr/bin/openssl
+    echo URL=https://${MGR}:443/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
+    fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=https://${MGR}:443/confluent-public/os/$osprofile/distribution/install.iso"
 fi
 if [ ! -z "$cons" ]; then
     fcmdline="$fcmdline console=${cons#/dev/}"

confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data

@@ -1,5 +1,13 @@
 #cloud-config
 autoinstall:
+#  The following can help an Ubuntu system skip install-time updates
+#  Only uncomment if you know you really want to do this or plan to manage the updates
+#  a different way.
+#
+#  updates: security
+#  apt:
+#    disable_suites: [security]
+#    fallback: offline-install
   version: 1
   early-commands:
     - /custom-installation/pre.sh

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions

@@ -53,6 +53,7 @@ function set_confluent_vars() {
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
     fi
+    export confluent_profile confluent_mgr nodename
 }
 
 fetch_remote() {

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk

@@ -1,3 +1,4 @@
+#!/usr/bin/python3
 import subprocess
 import os
 
@@ -10,8 +11,9 @@ class DiskInfo(object):
         self.path = None
         self.model = ''
         self.size = 0
-        self.driver = None
+        self.driver = ''
         self.mdcontainer = ''
+        self.subsystype = ''
         devnode = '/dev/{0}'.format(devname)
         qprop = subprocess.check_output(
             ['udevadm', 'info', '--query=property', devnode])
@@ -46,7 +48,9 @@ class DiskInfo(object):
             elif (k == 'DRIVERS' and not self.driver
                     and v not in ('"sd"', '""')):
                 self.driver = v.replace('"', '')
-        if not self.driver and 'imsm' not in self.mdcontainer:
+            elif k == 'ATTRS{subsystype}':
+                self.subsystype = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer and self.subsystype != 'nvm':
             raise Exception("No driver detected")
         if os.path.exists('/sys/block/{0}/size'.format(self.name)):
             with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh

@@ -84,6 +84,10 @@ cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
 cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
 chroot /target update-ca-certificates
+
+# Ubuntu mangles grub function for serial users, undo that mangling
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python autoconsole"
+
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"

confluent_osdeploy/utils/clortho.c

@@ -246,6 +246,6 @@ int main(int argc, char* argv[]) {
             buffer[0] = 255;
             ret = read(sock, buffer, 2);
         }
-        fprintf(stderr, "Password was not accepted\n");
+        fprintf(stderr, "Confluent API token grant denied by server\n");
         exit(1);
 }

confluent_osdeploy/utils/confusebox/Makefile

@@ -0,0 +1,8 @@
+all: confusebox
+
+confusebox: *.go
+	go build -ldflags "-w -s" -gcflags=all="-l" -trimpath
+	upx --brute confusebox
+
+clean:
+	rm confusebox

confluent_osdeploy/utils/confusebox/apiclient.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"net/http"
+	"crypto/x509"
+	"crypto/tls"
+	"strings"
+	"errors"
+)
+type ApiClient struct {
+	server string
+	urlserver string
+	apikey string
+	nodename string
+	webclient *http.Client
+}
+
+func NewApiClient(cafile string, keyfile string, nodename string, server string) (*ApiClient, error) {
+	currcacerts, err := os.ReadFile(cafile)
+	if err != nil {
+		return nil, err
+	}
+	cacerts  := x509.NewCertPool()
+	cacerts.AppendCertsFromPEM(currcacerts)
+	apikey := []byte("")
+	if keyfile != "" {
+		apikey, err = os.ReadFile(keyfile)
+		if err != nil {
+			return nil, err
+		}
+		if apikey[len(apikey) - 1] == 0xa {
+			apikey = apikey[:len(apikey)-1]
+		}
+	}
+	if nodename == "" {
+		cinfo, err := os.ReadFile("/etc/confluent/confliuent.info")
+		if err != nil {
+			nodename, err = os.Hostname()
+			if err != nil { return nil, err }
+		}
+		cinfolines := bytes.Split(cinfo, []byte("\n"))
+		if bytes.Contains(cinfolines[0], []byte("NODENAME")) {
+			cnodebytes := bytes.Split(cinfolines[0], []byte(" "))
+			nodename = string(cnodebytes[0])
+		}
+	}
+	urlserver := server
+	if strings.Contains(server, ":") {
+		if strings.Contains(server, "%") && !strings.Contains(server, "%25") {
+			server = strings.Replace(server, "%", "%25", 1)
+		}
+		urlserver = fmt.Sprintf("[%s]", server)
+		if strings.Contains(server, "%") {
+			server = server[:strings.Index(server, "%")]
+		}
+	}
+	webclient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: cacerts,
+				ServerName: server,
+			},
+		},
+	}
+	vc := ApiClient{server, urlserver, string(apikey), nodename, webclient}
+	return &vc, nil
+}
+
+func (apiclient *ApiClient) RegisterKey(crypted string, hmac string) (error) {
+	cryptbytes := []byte(crypted)
+	cryptbuffer := bytes.NewBuffer(cryptbytes)
+	_, err := apiclient.request("/confluent-api/self/registerapikey", "", cryptbuffer, "", hmac)
+	return err
+}
+
+func (apiclient *ApiClient) Fetch(url string, outputfile string, mime string, body io.Reader) (error)  {
+	outp, err := os.Create(outputfile)
+	if err != nil { return err }
+	defer outp.Close()
+	rsp, err := apiclient.request(url, mime, body, "", "")
+	if err != nil { return err }
+	_, err = io.Copy(outp, rsp)
+	return err
+}
+
+func (apiclient *ApiClient) GrabText(url string, mime string, body io.Reader) (string, error){
+	rsp, err := apiclient.request(url, mime, body, "", "")
+	if err != nil { return "", err }
+	rspdata, err := io.ReadAll(rsp)
+	if err != nil { return "", err }
+	rsptxt := string(rspdata)
+	return rsptxt, nil
+}
+
+func (apiclient *ApiClient) request(url string, mime string, body io.Reader, method string, hmac string) (io.ReadCloser, error) {
+	if ! strings.Contains(url, "https://") {
+		url = fmt.Sprintf("https://%s%s", apiclient.urlserver, url)
+	}
+	if method == "" {
+		if body != nil {
+			method = http.MethodPost
+		} else {
+			method = http.MethodGet
+		}
+	}
+	var err error
+	var rq *http.Request
+	if body == nil {
+		rq, err = http.NewRequest(method, url, nil)
+	} else {
+		rq, err = http.NewRequest(method, url, body)
+	}
+	if err != nil { return nil, err }
+	if (mime != "") { rq.Header.Set("Accept", mime) }
+	rq.Header.Set("CONFLUENT_NODENAME", apiclient.nodename)
+	if len(hmac) > 0 {
+		rq.Header.Set("CONFLUENT_CRYPTHMAC", hmac)
+	} else {
+
+		rq.Header.Set("CONFLUENT_APIKEY", apiclient.apikey)
+	}
+	rsp, err := apiclient.webclient.Do(rq)
+	if err != nil { return nil, err }
+	if rsp.StatusCode >= 300 {
+		err = errors.New(rsp.Status)
+		return nil, err
+	}
+	return rsp.Body, err
+}
+

confluent_osdeploy/utils/confusebox/genpasshmac.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"bytes"
+	"github.com/go-crypt/crypt/algorithm/shacrypt"
+	"os"
+	"crypto/rand"
+	"encoding/base64"
+	"crypto/hmac"
+	"crypto/sha256"
+)
+
+func genpasshmac(hmackeyfile string) (string, string, string, error)  {
+	randbytes := make([]byte, 36)
+	_, err := rand.Read(randbytes)
+	if err != nil {
+		panic(err)
+	}
+	password := base64.StdEncoding.EncodeToString(randbytes)
+	hasher, err := shacrypt.New(shacrypt.WithVariant(shacrypt.VariantSHA256), shacrypt.WithIterations(5000))
+	if err != nil {
+		panic(err)
+	}
+
+	digest, err := hasher.Hash(password)
+	if err != nil {
+		panic(err)
+	}
+	cryptpass := digest.Encode()
+	hmackey, err := os.ReadFile(hmackeyfile)
+	if err != nil { return "", "", "", err }
+	keylines := bytes.Split(hmackey, []byte("\n"))
+	if bytes.Contains(keylines[0], []byte("apitoken:")) {
+		keyparts := bytes.Split(keylines[0], []byte(" "))
+		hmackey = keyparts[1]
+	}
+
+	hmacer := hmac.New(sha256.New, hmackey)
+	hmacer.Write([]byte(cryptpass))
+	hmacresult := hmacer.Sum(nil)
+	hmacout := base64.StdEncoding.EncodeToString(hmacresult)
+	return password, cryptpass, hmacout, nil
+}
+

confluent_osdeploy/utils/confusebox/go.mod

@@ -0,0 +1,7 @@
+module confusebox
+
+go 1.23.6
+
+require github.com/go-crypt/crypt v0.3.3
+
+require github.com/go-crypt/x v0.3.4 // indirect

confluent_osdeploy/utils/confusebox/go.sum

@@ -0,0 +1,4 @@
+github.com/go-crypt/crypt v0.3.3 h1:mBSh8U+vwDm3V+UHNMQqsxV0clzlvKbLcJXcafYFpCs=
+github.com/go-crypt/crypt v0.3.3/go.mod h1:ex5C1b58/tzCW6/rJfcdf5Y2TjgzmWVtX57sjpN3pUQ=
+github.com/go-crypt/x v0.3.4 h1:zgpaI55VOAbkkRup9+tLaZ02IWTV/xz63tohoY0t9+Y=
+github.com/go-crypt/x v0.3.4/go.mod h1:+uHWqfzD3S6YWxm18/Qp+4VcuBb0Le9dGUhX0zaWicU=

confluent_osdeploy/utils/confusebox/main.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"os"
+	"io"
+	"fmt"
+)
+
+func get_confluent_server() (string, error) {
+	var confluentsrv string
+	dcfg, err := os.ReadFile("/etc/confluent/confluent.deploycfg")
+	if err == nil {
+		dcfglines := bytes.Split(dcfg, []byte("\n"))
+		for _, dcfgline := range(dcfglines) {
+			dkeyval := bytes.Split(dcfgline, []byte(" "))
+			if bytes.Contains(dkeyval[0], []byte("deploy_server")) && (bytes.Contains(dkeyval[1], []byte(".")) || bytes.Contains(dkeyval[1], []byte(":"))) {
+				confluentsrv = string(dkeyval[1])
+				return confluentsrv, nil
+			}
+		}
+	} else {
+		_, err := os.ReadFile("/etc/confluent/confluent.info")
+		if err != nil {
+			return "Unable to determine Confluent server", err
+		}
+	}
+	return "", err
+}
+func main() {
+	var nodename string
+	var cacerts string
+	var apikey string
+	var usejson bool
+	var confluentsrv string
+	hmacreg := flag.NewFlagSet("hmacregister", flag.ExitOnError)
+	hmacreg.StringVar(&apikey, "k", "/etc/confluent/apikey", "Output file for the api key")
+	hmacKey := hmacreg.String("i", "", "Identity yaml file")
+	hmacreg.StringVar(&cacerts, "c", "/etc/confluent/ca.pem", "Certeficate authorities to use in PEM")
+	hmacreg.StringVar(&nodename, "n", "", "Node name")
+	hmacreg.StringVar(&confluentsrv, "s", "", "Confluent server to request from")
+
+	invokeapi := flag.NewFlagSet("invoke", flag.ExitOnError)
+	invokeapi.StringVar(&nodename, "n", "", "Node name")
+
+	invokeapi.StringVar(&cacerts, "c", "/etc/confluent/ca.pem", "Certeficate authorities to use in PEM")
+	invokeapi.StringVar(&apikey, "k", "/etc/confluent/confluent.apikey", "File containing Confluent API key")
+	invokeapi.BoolVar(&usejson, "j", false, "Request JSON formatted reply")
+	outputfile := invokeapi.String("o", "", "Filename to store download to")
+	invokeapi.StringVar(&confluentsrv, "s", "", "Confluent server to request from")
+	invokedata := invokeapi.String("d", "", "Data to submit")
+	invokedatafile := invokeapi.String("i", "", "File containing data to submit")
+
+
+
+	if len(os.Args) < 2 {
+		panic("Insufficient arguments, no subcommand")
+	}
+	switch os.Args[1] {
+		case "hmacregister":
+			var err error
+			hmacreg.Parse(os.Args[2:])
+			if confluentsrv == "" {
+				confluentsrv, err = get_confluent_server()
+			}
+			password, crypted, hmac, err := genpasshmac(*hmacKey)
+			if err != nil { panic(err) }
+			//apiclient(cacerts, "/confluent-api/self/registerapikey", apikey, nodename, usejson)
+			apiclient, err := NewApiClient(cacerts, "", nodename, confluentsrv)
+			if err != nil { panic(err) }
+			err = apiclient.RegisterKey(crypted, hmac)
+			if err != nil { panic(err) }
+			outp, err := os.Create(apikey)
+			if err != nil { panic(err) }
+			defer outp.Close()
+			outp.Write([]byte(password))
+		case "invoke":
+			var err error
+			var body io.Reader
+			body = nil
+			invokeapi.Parse(os.Args[2:])
+			if *invokedata != "" {
+				body = bytes.NewBuffer([]byte(*invokedata))
+			}
+			if *invokedatafile != "" {
+				body, err = os.Open(*invokedatafile)
+				if err != nil { panic(err) }
+			}
+			if confluentsrv == "" {
+				confluentsrv, err = get_confluent_server()
+			}
+			apiclient, err := NewApiClient(cacerts, apikey, nodename, confluentsrv)
+			if err != nil { panic(err) }
+			mime := ""
+			if usejson {
+				mime = "application/json"
+			}
+			if *outputfile != "" {
+				err := apiclient.Fetch(invokeapi.Arg(0), *outputfile, mime, body)
+				if err != nil { panic(err) }
+			} else {
+				rsp, err := apiclient.GrabText(invokeapi.Arg(0), mime, body)
+				if err != nil { panic(err) }
+				fmt.Println(rsp)
+			}
+		default:
+			panic("Unrecognized subcommand")
+	}
+}

confluent_osdeploy/utils/goapiclient/go.mod

@@ -0,0 +1,5 @@
+module confluentapiclient
+
+go 1.22
+
+toolchain go1.23.6

confluent_osdeploy/utils/goapiclient/go.sum

@@ -0,0 +1,4 @@
+github.com/go-crypt/crypt v0.3.2 h1:I4i0u2g8X9bxCXIjvv19BDVXqQbddDQrURCJrOyyJos=
+github.com/go-crypt/crypt v0.3.2/go.mod h1:U0YhpCizEtaVC4gVfUUN0qGn1Z6+e3at+B5uLYx/sV0=
+github.com/go-crypt/x v0.3.2 h1:m2wn2+8tp28V4yDiW5NSTiyNSXnCoTs1R1+H+cAJA3M=
+github.com/go-crypt/x v0.3.2/go.mod h1:uelN9rbD2e2eqE8KA26B9R6OQ0TdM6msWdPsoMM1ZFk=

confluent_osdeploy/utils/goapiclient/main.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"net/http"
+	"crypto/x509"
+	"crypto/tls"
+)
+
+func main() {
+	certauthority := flag.String("c", "/etc/confluent/ca.pem", "Certificate authorities to use, in PEM format")
+	targurl := flag.String("u", "", "Url to connect to")
+	keyfile := flag.String("k", "/etc/confluent/confluent.apikey", "Confluent API key file")
+	nodename := flag.String("n", "", "Node Name")
+	usejson := flag.Bool("j", false, "Use JSON")
+	flag.Parse()
+	certpool := x509.NewCertPool()
+	currcacerts, err := os.ReadFile(*certauthority)
+	if err != nil {
+		panic(err)
+	}
+	confluentapikey, err := os.ReadFile(*keyfile)
+	if confluentapikey[len(confluentapikey) - 1] == 0xa {
+		confluentapikey = confluentapikey[:len(confluentapikey)-1]
+	}
+	if err != nil {
+		panic(err)
+	}
+	certpool.AppendCertsFromPEM(currcacerts)
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: certpool,
+			},
+		},
+	}
+	rq, err := http.NewRequest(http.MethodGet, *targurl, nil)
+	if err != nil { panic(err )}
+	if *usejson { rq.Header.Set("Accept", "application/json") }
+	if *nodename == "" {
+		*nodename, err = os.Hostname()
+	}
+	rq.Header.Set("CONFLUENT_NODENAME", *nodename)
+	fmt.Println(string(confluentapikey))
+	rq.Header.Set("CONFLUENT_APIKEY", string(confluentapikey))
+	if err != nil { panic(err )}
+	rsp, err := client.Do(rq)
+	if err != nil { panic(err )}
+	rspdata, err := io.ReadAll(rsp.Body)
+	rsptxt := string(rspdata)
+	fmt.Println(rsptxt)
+}
+

confluent_osdeploy/utils/gopasshmac/go.mod

@@ -0,0 +1,10 @@
+module genpasshmac
+
+go 1.22
+
+toolchain go1.23.6
+
+require (
+	github.com/go-crypt/crypt v0.3.2 // indirect
+	github.com/go-crypt/x v0.3.2 // indirect
+)

confluent_osdeploy/utils/gopasshmac/go.sum

@@ -0,0 +1,4 @@
+github.com/go-crypt/crypt v0.3.2 h1:I4i0u2g8X9bxCXIjvv19BDVXqQbddDQrURCJrOyyJos=
+github.com/go-crypt/crypt v0.3.2/go.mod h1:U0YhpCizEtaVC4gVfUUN0qGn1Z6+e3at+B5uLYx/sV0=
+github.com/go-crypt/x v0.3.2 h1:m2wn2+8tp28V4yDiW5NSTiyNSXnCoTs1R1+H+cAJA3M=
+github.com/go-crypt/x v0.3.2/go.mod h1:uelN9rbD2e2eqE8KA26B9R6OQ0TdM6msWdPsoMM1ZFk=

confluent_osdeploy/utils/gopasshmac/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"flag"
+	//"fmt"
+	"github.com/go-crypt/crypt/algorithm/shacrypt"
+	"os"
+	"crypto/rand"
+	"encoding/base64"
+	"crypto/hmac"
+	"crypto/sha256"
+)
+
+func main() {
+	hmackeyfile := flag.String("k", "", "Key file for HMAC calculation")
+	passfile := flag.String("p", "", "File to write generated password to")
+	cryptfile := flag.String("c", "", "File to write crypted form of key to")
+	hmacfile := flag.String("m", "", "File to write HMAC value to")
+	flag.Parse()
+	randbytes := make([]byte, 36)
+	_, err := rand.Read(randbytes)
+	if err != nil {
+		panic(err)
+	}
+	newpasswd := base64.StdEncoding.EncodeToString(randbytes)
+	hasher, err := shacrypt.New(shacrypt.WithVariant(shacrypt.VariantSHA256), shacrypt.WithIterations(5000))
+	if err != nil {
+		panic(err)
+	}
+
+	digest, err := hasher.Hash(newpasswd)
+	if err != nil {
+		panic(err)
+	}
+	cryptdata := []byte(digest.Encode())
+	err = os.WriteFile(*passfile, []byte(newpasswd), 0600)
+	if err != nil { panic(err )}
+	err = os.WriteFile(*cryptfile, cryptdata, 0600)
+	if err != nil { panic(err )}
+	keydata, err := os.ReadFile(*hmackeyfile)
+	if err != nil { panic(err )}
+	hmacer := hmac.New(sha256.New, keydata)
+	hmacer.Write(cryptdata)
+	hmacresult := hmacer.Sum(nil)
+	hmacout := []byte(base64.StdEncoding.EncodeToString(hmacresult))
+	err = os.WriteFile(*hmacfile, hmacout, 0600)
+	if err != nil { panic(err )}
+}
+

confluent_server/bin/confluent_selfcheck

@@ -223,7 +223,7 @@ if __name__ == '__main__':
             try:
                 sshutil.prep_ssh_key('/etc/confluent/ssh/automation')
                 print('OK')
-            except subprocess.CalledProcessError:
+            except Exception:
                 emprint('Failed to load confluent automation key, syncfiles and profile ansible plays will not work (Example resolution: osdeploy initialize -a)')
             os.kill(int(sshutil.agent_pid), signal.SIGTERM)
         fprint('Checking for blocked insecure boot: ')

confluent_server/bin/confluentdbutil

@@ -50,6 +50,8 @@ argparser.add_option('-s', '--skipkeys', action='store_true',
                            'protected keys.json file, and only the protected '
                            'data is needed.  keys do not change and as such '
                            'they do not require incremental backup')
+argparser.add_option('-y', '--yaml', action='store_true',
+                      help='Use YAML instead of JSON as file format')
 (options, args) = argparser.parse_args()
 if len(args) != 2 or args[0] not in ('dump', 'restore', 'merge'):
     argparser.print_help()
@@ -73,9 +75,16 @@ if args[0] in ('restore', 'merge'):
         cfm.init(stateless)
         cfm.statelessmode = stateless
         skipped = {'nodes': [], 'nodegroups': []}
+        
+        # Use the format parameter based on the --yaml option
+        format = 'yaml' if options.yaml else 'json'
+        
         cfm.restore_db_from_directory(
             dumpdir, password,
-            merge="skip" if args[0] == 'merge' else False, skipped=skipped)
+            merge="skip" if args[0] == 'merge' else False, 
+            skipped=skipped,
+            format=format)
+            
         if skipped['nodes']:
             skippedn = ','.join(skipped['nodes'])
             print('The following nodes were skipped during merge: '
@@ -114,8 +123,11 @@ elif args[0] == 'dump':
     main._initsecurity(conf.get_config())
     if not os.path.exists(dumpdir):
         os.makedirs(dumpdir)
+    
+    # Use the format parameter based on the --yaml option
+    format = 'yaml' if options.yaml else 'json'
     cfm.dump_db_to_directory(dumpdir, password, options.redact,
-                             options.skipkeys)
+                             options.skipkeys, format=format)
 
 
 

confluent_server/confluent/certutil.py

@@ -218,11 +218,17 @@ def create_certificate(keyout=None, certout=None, csrout=None):
         subprocess.check_call(
             ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
              keyout])
-    san = ['IP:{0}'.format(x) for x in get_ip_addresses()]
+    ipaddrs = list(get_ip_addresses())
+    san = ['IP:{0}'.format(x) for x in ipaddrs]
     # It is incorrect to put IP addresses as DNS type.  However
     # there exists non-compliant clients that fail with them as IP
-    san.extend(['DNS:{0}'.format(x) for x in get_ip_addresses()])
-    san.append('DNS:{0}'.format(shortname))
+    # san.extend(['DNS:{0}'.format(x) for x in ipaddrs])
+    dnsnames = set(ipaddrs)
+    dnsnames.add(shortname)
+    for currip in ipaddrs:
+        dnsnames.add(socket.getnameinfo((currip, 0), 0)[0])
+    for currname in dnsnames:
+        san.append('DNS:{0}'.format(currname))
     #san.append('DNS:{0}'.format(longname))
     san = ','.join(san)
     sslcfg = get_openssl_conf_location()

confluent_server/confluent/config/attributes.py

@@ -94,8 +94,8 @@ node = {
                         'considered a member'),
     },
     'type': {
-        'description': ('Classification of node as server or switch.  By default a node is presumed to be a server.'),
-        'validvalues': ('switch', 'server'),
+        'description': ('The type of node.  This may be switch, server, rackmount, dense, enclosure or not set to be generic.'),
+        'validvalues': ('switch', 'server', 'rackmount', 'dense', 'enclosure', ''),
     },
     'crypted.rootpassword': {
         'description': 'The password of the local root password. '
@@ -265,8 +265,7 @@ node = {
     },
     'discovery.policy': {
         'description':  'Policy to use for auto-configuration of discovered '
-                        'and identified nodes. Valid values are "manual", '
-                        '"permissive", or "open". "manual" means nodes are '
+                        'and identified nodes. "manual" means nodes are '
                         'detected, but not autoconfigured until a user '
                         'approves. "permissive" indicates to allow discovery, '
                         'so long as the node has no existing public key. '
@@ -361,9 +360,8 @@ node = {
 #                        'to suppress serial console configuration')
 #    },
     'console.logging': {
-        'description': ('Indicate logging level to apply to console.  Valid '
-                        'values are currently "full", "interactive", "memory", and '
-                        '"none". Defaults to "full".'),
+        'description': ('Indicate logging level to apply to console. '
+                        'Defaults to "full".'),
         'validvalues': ('full', 'memory', 'interactive', 'none'),
     },
     'console.method': {
@@ -557,7 +555,7 @@ node = {
         'description': 'Indicates that this interface should be a team and what mode or runner to use when teamed. '
                        'If this covers a deployment interface, one of the member interfaces may be brought up as '
                        'a standalone interface until deployment is complete, as supported by the OS deployment profile. '
-                       'To support this scenario, the switch should be set up to allow independent operation of member ports123654 (e.g. lacp bypass mode or fallback mode).',
+                       'To support this scenario, the switch should be set up to allow independent operation of member ports (e.g. lacp bypass mode or fallback mode).',
         'validvalues': ('lacp', 'loadbalance', 'roundrobin', 'activebackup', 'none')
     },
     'power.pdu': {

confluent_server/confluent/config/configmanager.py

@@ -102,7 +102,7 @@ try:
     unicode
 except NameError:
     unicode = str
-
+import yaml
 
 _masterkey = None
 _masterintegritykey = None
@@ -136,7 +136,7 @@ def attrib_supports_expression(attrib):
     if not isinstance(attrib, str):
         attrib = attrib.decode('utf8')
     attrib = _attraliases.get(attrib, attrib)
-    if attrib.startswith('secret.') or attrib.startswith('crypted.'):
+    if attrib.startswith('secret.') or attrib.startswith('crypted.') or attrib.startswith('custom.nodesecret.'):
         return False
     return True
 
@@ -1108,6 +1108,10 @@ class _ExpressionFormat(string.Formatter):
             field_name = val
         parsed = ast.parse(field_name)
         val = self._handle_ast_node(parsed.body[0].value)
+        try:
+            val = int(val)
+        except Exception:
+            pass
         return format(val, format_spec)
 
     def _handle_ast_node(self, node):
@@ -1373,7 +1377,7 @@ class ConfigManager(object):
             attribute, match = expression.split('=')
         else:
             raise Exception('Invalid Expression')
-        if attribute.startswith('secret.'):
+        if attribute.startswith('secret.') or attribute.startswith('custom.nodesecret.'):
             raise Exception('Filter by secret attributes is not supported')
         if attribute_name_is_invalid(attribute):
             raise ValueError(
@@ -1986,6 +1990,9 @@ class ConfigManager(object):
                             delnodes = noderange.NodeRange(
                                 attribmap[group][attr]['remove'],
                                 config=self).nodes
+                            for node in delnodes:
+                                if node not in currnodes:
+                                    raise ValueError('node "{0}" is not a member of {1}'.format(node, group))
                             attribmap[group][attr] = [
                                 x for x in currnodes if x not in delnodes]
                     if not isinstance(attribmap[group][attr], list):
@@ -2020,10 +2027,10 @@ class ConfigManager(object):
                     newdict = {'value': attribmap[group][attr]}
                 else:
                     newdict = attribmap[group][attr]
-                if keydata and attr.startswith('secret.') and 'cryptvalue' in newdict:
+                if keydata and (attr.startswith('secret.') or attr.startswith('custom.nodesecret.')) and 'cryptvalue' in newdict:
                     newdict['value'] = decrypt_value(newdict['cryptvalue'], keydata['cryptkey'], keydata['integritykey'])
                     del newdict['cryptvalue']
-                if 'value' in newdict and attr.startswith("secret."):
+                if 'value' in newdict and (attr.startswith('secret.') or attr.startswith('custom.nodesecret.')):
                     newdict['cryptvalue'] = crypt_value(newdict['value'])
                     del newdict['value']
                 if 'value' in newdict and attr.startswith("crypted."):
@@ -2353,6 +2360,9 @@ class ConfigManager(object):
                 lidx = self._cfgstore['nodes'][node]['groups'].index(name)
                 self._cfgstore['nodes'][node]['groups'][lidx] = renamemap[name]
                 _mark_dirtykey('nodes', node, self.tenant)
+            for node in self._cfgstore['nodegroups'][renamemap[name]].get('nodes', []):
+                 self._node_removed_from_group(node, name, {})
+                 self._node_added_to_group(node, renamemap[name], {})
         self._bg_sync_to_file()
 
 
@@ -2439,6 +2449,9 @@ class ConfigManager(object):
                         elif attribmap[node]['groups'].get('remove', False):
                             delgroups = attribmap[node]['groups'][
                                 'remove'].split(',')
+                            for group in delgroups:
+                                if group not in currgroups:
+                                    raise ValueError("node {0} is not a member of group {1}".format(node, group))
                             newgroups = [
                                 x for x in currgroups if x not in delgroups]
                             attribmap[node]['groups'] = newgroups
@@ -2476,10 +2489,10 @@ class ConfigManager(object):
                 # add check here, skip None attributes
                 if newdict is None:
                     continue
-                if keydata and attrname.startswith('secret.') and 'cryptvalue' in newdict:
+                if keydata and (attrname.startswith('secret.') or attrname.startswith('custom.nodesecret.')) and 'cryptvalue' in newdict:
                     newdict['value'] = decrypt_value(newdict['cryptvalue'], keydata['cryptkey'], keydata['integritykey'])
                     del newdict['cryptvalue']
-                if 'value' in newdict and attrname.startswith("secret."):
+                if 'value' in newdict and (attrname.startswith('secret.') or attrname.startswith('custom.nodesecret.')):
                     newdict['cryptvalue'] = crypt_value(newdict['value'])
                     del newdict['value']
                 if 'value' in newdict and attrname.startswith("crypted."):
@@ -2936,12 +2949,30 @@ def _dump_keys(password, dojson=True):
     return keydata
 
 
-def restore_db_from_directory(location, password, merge=False, skipped=None):
+def restore_db_from_directory(location, password, merge=False, skipped=None, format='json'):
+    """Restore database from a directory
+    
+    :param location: Directory containing the configuration
+    :param password: Password to decrypt sensitive data
+    :param merge: If True, merge with existing configuration
+    :param skipped: List of elements to skip during restore
+    :param format: Format of the files ('json' [default] or 'yaml')
+    """
+    if format not in ('json', 'yaml'):
+        raise ValueError("Format must be 'json' or 'yaml'")
+
     kdd = None
     try:
-        with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
+        keys_file = os.path.join(location, f'keys.{format}')
+        with open(keys_file, 'r') as cfgfile:
             keydata = cfgfile.read()
-            kdd = json.loads(keydata)
+            if format == 'json':
+                kdd = json.loads(keydata)
+            else:
+                kdd = yaml.safe_load(keydata)
+                if kdd is None:
+                    raise ValueError(f"Invalid or empty YAML content in {keys_file}")
+            
             if merge:
                 if 'cryptkey' in kdd:
                     kdd['cryptkey'] = _parse_key(kdd['cryptkey'], password)
@@ -2950,59 +2981,122 @@ def restore_db_from_directory(location, password, merge=False, skipped=None):
                 else:
                     kdd['integritykey'] = None  # GCM
             else:
+                if format == 'json':
+                    _restore_keys(keydata, password)
+                else:
+                    # Convert YAML to JSON string for _restore_keys
+                    _restore_keys(json.dumps(kdd), password)
                 kdd = None
-                _restore_keys(keydata, password)
     except IOError as e:
         if e.errno == 2:
             raise Exception("Cannot restore without keys, this may be a "
                             "redacted dump")
     if not merge:
         try:
-            moreglobals = json.load(open(os.path.join(location, 'globals.json')))
-            for globvar in moreglobals:
-                set_global(globvar, moreglobals[globvar])
+            globals_file = os.path.join(location, f'globals.{format}')
+            with open(globals_file, 'r') as globin:
+                if format == 'json':
+                    moreglobals = json.load(globin)
+                else:
+                    moreglobals = yaml.safe_load(globin)
+                    if moreglobals is None:
+                        raise ValueError(f"Invalid or empty YAML content in {globals_file}")
+                
+                for globvar in moreglobals:
+                    set_global(globvar, moreglobals[globvar])
         except IOError as e:
             if e.errno != 2:
                 raise
         try:
-            collective = json.load(open(os.path.join(location, 'collective.json')))
-            _cfgstore['collective'] = {}
-            for coll in collective:
-                add_collective_member(coll, collective[coll]['address'],
-                                    collective[coll]['fingerprint'])
+            collective_file = os.path.join(location, f'collective.{format}')
+            with open(collective_file, 'r') as collin:
+                if format == 'json':
+                    collective = json.load(collin)
+                else:
+                    collective = yaml.safe_load(collin)
+                    if collective is None:
+                        raise ValueError(f"Invalid or empty YAML content in {collective_file}")
+                
+                _cfgstore['collective'] = {}
+                for coll in collective:
+                    add_collective_member(coll, collective[coll]['address'],
+                                        collective[coll]['fingerprint'])
         except IOError as e:
             if e.errno != 2:
                 raise
-    with open(os.path.join(location, 'main.json'), 'r') as cfgfile:
+    main_file = os.path.join(location, f'main.{format}')
+    with open(main_file, 'r') as cfgfile:
         cfgdata = cfgfile.read()
+        if format == 'yaml':
+            # Convert YAML to JSON string for _load_from_json
+            yaml_data = yaml.safe_load(cfgdata)
+            if yaml_data is None:
+                raise ValueError(f"Invalid or empty YAML content in {main_file}")
+            cfgdata = json.dumps(yaml_data)
         ConfigManager(tenant=None)._load_from_json(cfgdata, merge=merge, keydata=kdd, skipped=skipped)
     ConfigManager.wait_for_sync(True)
 
-
-def dump_db_to_directory(location, password, redact=None, skipkeys=False):
+def dump_db_to_directory(location, password, redact=None, skipkeys=False, format='json'):
+    """Dump database to a directory
+    
+    :param location: Directory to store the configuration
+    :param password: Password to protect sensitive data
+    :param redact: If True, redact sensitive data
+    :param skipkeys: If True, skip dumping keys
+    :param format: Format to use for dumping ('json' [default] or 'yaml')
+    """
+    if format not in ('json', 'yaml'):
+        raise ValueError("Format must be 'json' or 'yaml'")
+    
+    # Handle keys file
     if not redact and not skipkeys:
-        with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
-            cfgfile.write(_dump_keys(password))
+        with open(os.path.join(location, f'keys.{format}'), 'w') as cfgfile:
+            if format == 'json':
+                cfgfile.write(_dump_keys(password))
+            else:
+                keydata = _dump_keys(password, dojson=False)
+                yaml.dump(keydata, cfgfile, default_flow_style=False)
             cfgfile.write('\n')
-    with open(os.path.join(location, 'main.json'), 'wb') as cfgfile:
-        cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
-        cfgfile.write(b'\n')
+    
+    # Handle main config
+    main_data = ConfigManager(tenant=None)._dump_to_json(redact=redact)
+    with open(os.path.join(location, f'main.{format}'), 'wb' if format == 'json' else 'w') as cfgfile:
+        if format == 'json':
+            cfgfile.write(main_data)
+            cfgfile.write(b'\n')
+        else:
+            # Convert JSON to Python object, then dump as YAML
+            yaml.dump(json.loads(main_data.decode('utf-8')), cfgfile, default_flow_style=False)
+    
+    # Handle collective data
     if 'collective' in _cfgstore:
-        with open(os.path.join(location, 'collective.json'), 'w') as cfgfile:
-            cfgfile.write(json.dumps(_cfgstore['collective']))
-            cfgfile.write('\n')
+        with open(os.path.join(location, f'collective.{format}'), 'w') as cfgfile:
+            if format == 'json':
+                cfgfile.write(json.dumps(_cfgstore['collective']))
+                cfgfile.write('\n')
+            else:
+                yaml.dump(_cfgstore['collective'], cfgfile, default_flow_style=False)
+    
+    # Handle globals
     bkupglobals = get_globals()
     if bkupglobals:
-        with open(os.path.join(location, 'globals.json'), 'w') as globout:
-            json.dump(bkupglobals, globout)
+        with open(os.path.join(location, f'globals.{format}'), 'w') as globout:
+            if format == 'json':
+                json.dump(bkupglobals, globout)
+            else:
+                yaml.dump(bkupglobals, globout, default_flow_style=False)
+    
+    # Handle tenants
     try:
         for tenant in os.listdir(
                 os.path.join(ConfigManager._cfgdir, '/tenants/')):
-            with open(os.path.join(location, 'tenants', tenant,
-                                   'main.json'), 'w') as cfgfile:
-                cfgfile.write(ConfigManager(tenant=tenant)._dump_to_json(
-                    redact=redact))
-                cfgfile.write('\n')
+            tenant_data = ConfigManager(tenant=tenant)._dump_to_json(redact=redact)
+            with open(os.path.join(location, 'tenants', tenant, f'main.{format}'), 'wb' if format == 'json' else 'w') as cfgfile:
+                if format == 'json':
+                    cfgfile.write(tenant_data)
+                    cfgfile.write(b'\n')
+                else:
+                    yaml.dump(json.loads(tenant_data.decode('utf-8')), cfgfile, default_flow_style=False)
     except OSError:
         pass
 

confluent_server/confluent/consoleserver.py

@@ -575,12 +575,12 @@ def disconnect_node(node, configmanager):
 
 def _nodechange(added, deleting, renamed, configmanager):
     for node in deleting:
-        disconnect_node(node, configmanager)
+        eventlet.spawn(disconnect_node, node, configmanager)
     for node in renamed:
         disconnect_node(node, configmanager)
-        connect_node(renamed[node], configmanager)
+        eventlet.spawn(connect_node, renamed[node], configmanager)
     for node in added:
-        connect_node(node, configmanager)
+        eventlet.spawn(connect_node, node, configmanager)
 
 
 def _start_tenant_sessions(cfm):
@@ -705,10 +705,13 @@ class ProxyConsole(object):
             if not util.cert_matches(self.managerinfo['fingerprint'],
                                      remote.getpeercert(binary_form=True)):
                 raise Exception('Invalid peer certificate')
-        except Exception:
+        except Exception as e:
+            if _tracelog:
+                _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                              event=log.Events.stacktrace)
             eventlet.sleep(3)
             if self.clisession:
-                self.clisession.detach()
+                self.clisession.detach(False)
             self.detachsession(None)
             return
         tlvdata.recv(remote)
@@ -835,7 +838,7 @@ class ConsoleSession(object):
         self._evt = None
         self.reghdl = None
 
-    def detach(self):
+    def detach(self, reattach=True):
         """Handler for the console handler to detach so it can reattach,
         currently to facilitate changing from one collective.manager to
         another
@@ -843,9 +846,10 @@ class ConsoleSession(object):
         :return:
         """
         self.conshdl.detachsession(self)
-        self.connect_session()
-        self.conshdl.attachsession(self)
-        self.write = self.conshdl.write
+        if reattach:
+            self.connect_session()
+            self.conshdl.attachsession(self)
+            self.write = self.conshdl.write
 
     def got_data(self, data):
         """Receive data from console and buffer

confluent_server/confluent/core.py

@@ -70,7 +70,10 @@ import os
 import eventlet.green.socket as socket
 import struct
 import sys
+import uuid
 import yaml
+import shutil
+
 
 pluginmap = {}
 dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos', b'enos', u'enos')
@@ -161,8 +164,9 @@ def _merge_dict(original, custom):
 
 
 rootcollections = ['deployment/', 'discovery/', 'events/', 'networking/',
-                   'noderange/', 'nodes/', 'nodegroups/', 'storage/', 'usergroups/' ,
-                   'users/', 'uuid', 'version']
+                   'noderange/', 'nodes/', 'nodegroups/', 'storage/', 'usergroups/',
+                   'users/', 'uuid', 'version', 'staging/']
+
 
 
 class PluginRoute(object):
@@ -430,6 +434,18 @@ def _init_core():
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
+            'ikvm': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+            'ikvm_methods': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+            'ikvm_screenshot': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
         },
         'description': PluginRoute({
             'pluginattrs': ['hardwaremanagement.method'],
@@ -478,6 +494,10 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'updatestatus': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
                 'updates': {
                     'active': PluginCollection({
                             'pluginattrs': ['hardwaremanagement.method'],
@@ -1266,6 +1286,97 @@ def handle_discovery(pathcomponents, operation, configmanager, inputdata):
     if pathcomponents[0] == 'detected':
         pass
 
+class Staging:
+    def __init__(self, user, uuid):
+        self.uuid_str = uuid
+        self.storage_folder = '/var/lib/confluent/client_assets/' + self.uuid_str
+        self.filename = None
+        self.user = user
+        self.base_folder = os.path.exists('/var/lib/confluent/client_assets/')
+
+        if not self.base_folder:
+            try:
+                os.mkdir('/var/lib/confluent/client_assets/')
+            except Exception as e:
+                raise OSError(str(e))
+             
+    def getUUID(self):
+        return self.uuid_str
+    
+    def get_push_url(self):
+        return 'staging/{0}/{1}'.format(self.user,self.uuid_str)
+    
+    def create_directory(self):
+        try:
+            os.mkdir(self.storage_folder)
+            return True
+        except OSError as e:
+            raise exc.InvalidArgumentException(str(e))
+    
+    def get_file_name(self):
+        stage_file = '{}/filename.txt'.format(self.storage_folder)
+        try:
+            with open(stage_file, 'r') as f:
+                filename = f.readline()
+            os.remove(stage_file)
+            return self.storage_folder + '/{}'.format(filename)
+        except FileNotFoundError:
+            file = None
+            return False
+        
+    @staticmethod
+    def remove_directory(directory):
+        storage_folder = '/var/lib/confluent/client_assets/' + directory
+        if os.path.exists(storage_folder):
+            shutil.rmtree(storage_folder)
+        else:
+            raise FileNotFoundError
+        return directory
+
+def handle_staging(pathcomponents, operation, configmanager, inputdata):
+    '''
+    e.g push_url: /confluent-api/staging/user/<unique_id> 
+    '''
+    if operation == 'create':
+        if len(pathcomponents) == 1:
+                stage = Staging(inputdata['user'],str(uuid.uuid1()))
+                if stage.create_directory():
+                    if 'filename' in inputdata:
+                        data_file = stage.storage_folder + '/filename.txt'
+                        with open(data_file, 'w') as f:
+                            f.write(inputdata['filename'])     
+                    else:
+                        raise Exception('Error: Missing filename arg')
+                    push_url = stage.get_push_url()
+                    yield msg.CreatedResource(push_url)
+
+        elif len(pathcomponents) == 3:
+            stage = Staging(pathcomponents[1], pathcomponents[2])
+            file = stage.get_file_name()
+            if 'filedata' in inputdata and file:
+                content_length = inputdata['content_length']
+                remaining_length = content_length
+                filedata = inputdata['filedata']
+                chunk_size = 16384
+                progress = 0.0
+                with open(file, 'wb') as f:
+                    while remaining_length > 0:
+                        progress = (1 - (remaining_length/content_length)) * 100
+                        datachunk = filedata['wsgi.input'].read(min(chunk_size, remaining_length))
+                        f.write(datachunk)     
+                        remaining_length -= len(datachunk)
+                        eventlet.sleep(0)
+                        yield msg.FileUploadProgress(progress)
+                    yield msg.FileUploadProgress(100) 
+                 
+
+    elif operation == 'delete':
+        if len(pathcomponents) == 3:
+            asset = Staging.remove_directory(pathcomponents[2])
+            yield msg.DeletedResource(asset)
+        else:
+            raise Exception("Invalid url")
+
 def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
     """Given a full path request, return an object.
 
@@ -1374,5 +1485,7 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
     elif pathcomponents[0] == 'discovery':
         return handle_discovery(pathcomponents[1:], operation, configmanager,
                                 inputdata)
+    elif pathcomponents[0] == 'staging':
+        return handle_staging(pathcomponents, operation, configmanager, inputdata)
     else:
         raise exc.NotFoundException()

confluent_server/confluent/discovery/core.py

@@ -75,6 +75,7 @@ import confluent.discovery.handlers.pxe as pxeh
 import confluent.discovery.handlers.smm as smm
 import confluent.discovery.handlers.xcc as xcc
 import confluent.discovery.handlers.xcc3 as xcc3
+import confluent.discovery.handlers.smm3 as smm3
 import confluent.discovery.handlers.megarac as megarac
 import confluent.exceptions as exc
 import confluent.log as log
@@ -114,6 +115,7 @@ class nesteddict(dict):
 nodehandlers = {
     'service:lenovo-smm': smm,
     'service:lenovo-smm2': smm,
+    'lenovo-smm3': smm3,
     'lenovo-xcc': xcc,
     'lenovo-xcc3': xcc3,
     'megarac-bmc': megarac,
@@ -134,6 +136,7 @@ servicenames = {
     'cumulus-switch': 'cumulus-switch',
     'service:lenovo-smm': 'lenovo-smm',
     'service:lenovo-smm2': 'lenovo-smm2',
+    'lenovo-smm3': 'lenovo-smm3',
     'affluent-switch': 'affluent-switch',
     'lenovo-xcc': 'lenovo-xcc',
     'lenovo-xcc3': 'lenovo-xcc3',
@@ -151,6 +154,7 @@ servicebyname = {
     'cumulus-switch': 'cumulus-switch',
     'lenovo-smm': 'service:lenovo-smm',
     'lenovo-smm2': 'service:lenovo-smm2',
+    'lenovo-smm3': 'lenovo-smm3',
     'affluent-switch': 'affluent-switch',
     'lenovo-xcc': 'lenovo-xcc',
     'lenovo-xcc3': 'lenovo-xcc3',
@@ -525,13 +529,22 @@ def register_remote_addrs(addresses, configmanager):
         nd = {
             'addresses': [(addr, 443)]
         }
-        sd = ssdp.check_fish(('/DeviceDescription.json', nd))
-        if not sd:
+        try:
+            sd = ssdp.check_fish(('/DeviceDescription.json', nd))
+            if not sd:
+                return addr, False
+            if 'macaddress' in sd['attributes']:
+                sd['hwaddr'] = sd['attributes']['macaddress']
+            else:
+                sd['hwaddr'] = sd['attributes']['mac-address']
+            if 'lenovo-xcc3' in sd['services']:
+                nh = xcc3.NodeHandler(sd, configmanager)
+            elif 'lenovo-xcc' in sd['services']:
+                nh = xcc.NodeHandler(sd, configmanager)
+            nh.scan()
+            detected(nh.info)
+        except Exception:
             return addr, False
-        sd['hwaddr'] = sd['attributes']['mac-address']
-        nh = xcc.NodeHandler(sd, configmanager)
-        nh.scan()
-        detected(nh.info)
         return addr, True
     rpool = eventlet.greenpool.GreenPool(512)
     for count in iterate_addrs(addresses, True):
@@ -1083,7 +1096,7 @@ def get_nodename(cfg, handler, info):
         # Ok, see if it is something with a chassis-uuid and discover by
         # chassis
         nodename = get_nodename_from_enclosures(cfg, info)
-    if not nodename and handler.devname == 'SMM':
+    if not nodename and handler.devname in ('SMM', 'SMM3'):
         nodename = get_nodename_from_chained_smms(cfg, handler, info)
     if not nodename:  # as a last resort, search switches for info
         # This is the slowest potential operation, so we hope for the
@@ -1091,7 +1104,7 @@ def get_nodename(cfg, handler, info):
         nodename, macinfo = macmap.find_nodeinfo_by_mac(info['hwaddr'], cfg)
         maccount = macinfo['maccount']
         if nodename:
-            if handler.devname == 'SMM':
+            if handler.devname in ('SMM', 'SMM3'):
                 nl = list(cfg.filter_node_attributes(
                             'enclosure.extends=' + nodename))
                 if nl:
@@ -1114,7 +1127,7 @@ def get_nodename(cfg, handler, info):
                         return None, None
         if (nodename and
                 not handler.discoverable_by_switch(macinfo['maccount'])):
-            if handler.devname == 'SMM':
+            if handler.devname in ('SMM', 'SMM3'):
                 errorstr = 'Attempt to discover SMM by switch, but chained ' \
                            'topology or incorrect net attributes detected, ' \
                            'which is not compatible with switch discovery ' \

confluent_server/confluent/discovery/handlers/redfishbmc.py

@@ -32,7 +32,7 @@ def get_host_interface_urls(wc, mginfo):
     returls = []
     hifurl = mginfo.get('HostInterfaces', {}).get('@odata.id', None)
     if not hifurl:
-        return None
+        return []
     hifinfo = wc.grab_json_response(hifurl)
     hifurls = hifinfo.get('Members', [])
     for hifurl in hifurls:
@@ -109,7 +109,7 @@ class NodeHandler(generic.NodeHandler):
             self.target_account_url(wc))
         acctinfo = acctinfo[0]
         actypes = acctinfo['AccountTypes']
-        candidates = acctinfo['AccountTypes@Redfish.AllowableValues']
+        candidates = acctinfo.get('AccountTypes@Redfish.AllowableValues', [])
         if 'IPMI' not in actypes and 'IPMI' in candidates:
             actypes.append('IPMI')
             acctupd = {
@@ -134,7 +134,14 @@ class NodeHandler(generic.NodeHandler):
                 rsp = json.loads(rsp)
                 currerr = rsp.get('error', {})
                 ecode = currerr.get('code', None)
-                if ecode.endswith('PasswordChangeRequired'):
+                if not ecode:
+                    for msg in rsp['@Message.ExtendedInfo']:
+                        if 'PasswordChangeRequired' in msg['MessageId']:
+                            chgurl = msg['MessageArgs'][0]
+                            break
+                    else:
+                        raise Exception("Failed to ascertain login failure reason")
+                elif ecode.endswith('PasswordChangeRequired'):
                     for einfo in currerr.get('@Message.ExtendedInfo', []):
                         if einfo.get('MessageId', None).endswith('PasswordChangeRequired'):
                             for msgarg in einfo.get('MessageArgs'):
@@ -269,7 +276,25 @@ class NodeHandler(generic.NodeHandler):
                     continue
                 actualnics.append(candnic)
             if len(actualnics) != 1:
-                raise Exception("Multi-interface BMCs are not supported currently")
+                compip = self.ipaddr
+                if ':' in compip:
+                    compip = compip.split('%')[0]
+                    ipkey = 'IPv6Addresses'
+                else:
+                    ipkey = 'IPv6Addresses'
+                actualnic = None
+                for curractnic in actualnics:
+                    currnicinfo = wc.grab_json_response(curractnic)
+                    for targipaddr in currnicinfo.get(ipkey, []):
+                        targipaddr = targipaddr.get('Address', 'Z')
+                        if compip == targipaddr:
+                            actualnic = curractnic
+                            break
+                    if actualnic:
+                        break
+                else:
+                    raise Exception("Unable to detect active NIC of multi-nic bmc")
+                actualnics = [actualnic]
             currnet = wc.grab_json_response(actualnics[0])
             netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
             newconfig = {

confluent_server/confluent/discovery/handlers/smm3.py

@@ -0,0 +1,71 @@
+# Copyright 2024 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.redfishbmc as redfishbmc
+import eventlet.support.greendns
+import confluent.util as util
+
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+
+
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+
+class NodeHandler(redfishbmc.NodeHandler):
+    devname = 'SMM3'
+    maxmacs = 18  # support an enclosure, but try to avoid catching daisy chain
+    is_enclosure = True
+
+    def scan(self):
+        attrs = self.info.get('attributes', {})
+        mtm = attrs.get('enclosure-machinetype-model', None)
+        if mtm:
+            self.info['modelnumber'] = mtm.strip()
+        sn = attrs.get('enclosure-serial-number', None)
+        if sn:
+            self.info['serialnumber'] = sn.strip()
+        modelname = attrs.get('enclosure-component-name', None)
+        if modelname:
+            modelname = modelname.split(' MT:')[0]
+            self.info['modelname'] = modelname
+
+    def get_firmware_default_account_info(self):
+        return ('USERID', 'PASSW0RD')
+
+
+def remote_nodecfg(nodename, cfm):
+    cfg = cfm.get_node_attributes(
+            nodename, 'hardwaremanagement.manager')
+    ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get(
+        'value', None)
+    ipaddr = ipaddr.split('/', 1)[0]
+    ipaddr = getaddrinfo(ipaddr, 0)[0][-1]
+    if not ipaddr:
+        raise Exception('Cannot remote configure a system without known '
+                        'address')
+    info = {'addresses': [ipaddr]}
+    nh = NodeHandler(info, cfm)
+    nh.config(nodename)
+
+
+if __name__ == '__main__':
+    import confluent.config.configmanager as cfm
+    c = cfm.ConfigManager(None)
+    import sys
+    info = {'addresses': [[sys.argv[1]]]}
+    print(repr(info))
+    testr = NodeHandler(info, c)
+    testr.config(sys.argv[2])
+

confluent_server/confluent/discovery/handlers/xcc.py

@@ -479,7 +479,7 @@ class NodeHandler(immhandler.NodeHandler):
                             {'UserName': username}, method='PATCH')
                         if status != 200:
                             rsp = json.loads(rsp)
-                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError'):
+                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError', 'Base.1.18.GeneralError'):
                                 if tries:
                                     eventlet.sleep(4)
                                 elif tmpaccount:
@@ -627,6 +627,7 @@ class NodeHandler(immhandler.NodeHandler):
                         '/redfish/v1/AccountService/Accounts/1',
                         updateinf, method='PATCH')
         if targbmc and not targbmc.startswith('fe80::'):
+            attribsuffix = ''
             newip = targbmc.split('/', 1)[0]
             newipinfo = getaddrinfo(newip, 0)[0]
             newip = newipinfo[-1][0]
@@ -636,6 +637,25 @@ class NodeHandler(immhandler.NodeHandler):
             newmask = netutil.cidr_to_mask(netconfig['prefix'])
             currinfo = wc.grab_json_response('/api/providers/logoninfo')
             currip = currinfo.get('items', [{}])[0].get('ipv4_address', '')
+            curreth1 = wc.grab_json_response('/api/dataset/imm_ethernet')
+            if curreth1:
+                if self.ipaddr.startswith('fe80::'):
+                    ipkey = 'ipv6_link_local_address'
+                elif '.' in self.ipaddr:
+                    ipkey = 'ipv4_address'
+                else:
+                    raise Exception('Non-Link-Local IPv6 TODO')
+                nic1ip = curreth1.get('items', [{}])[0].get(ipkey, None)
+                if nic1ip != self.ipaddr:
+                    # check second nic instead
+                    curreth2 = wc.grab_json_response('/api/dataset/imm_ethernet_2')
+                    if curreth2 and curreth2.get('items', [{}])[0].get('if_second_port_exist', 0):
+                        nic2ip = curreth2.get('items', [{}])[0].get(ipkey + '_2', None)
+                        if nic2ip != self.ipaddr:
+                            raise Exception("Unable to determine which NIC is active")
+                        # ok, second nic is active, target it
+                        currip = curreth2.get('items', [{}])[0].get("ipv4_address", None)
+                        attribsuffix = '_2'
             # do not change the ipv4_config if the current config looks right already
             if currip != newip:
                 statargs = {
@@ -646,9 +666,11 @@ class NodeHandler(immhandler.NodeHandler):
                     statargs['ENET_IPv4GatewayIPAddr'] = netconfig['ipv4_gateway']
                 elif not netutil.address_is_local(newip):
                     raise exc.InvalidArgumentException('Will not remotely configure a device with no gateway')
+                if attribsuffix:
+                    for currkey in list(statargs):
+                        statargs[currkey + attribsuffix] = statargs[currkey]
+                        del statargs[currkey]
                 netset, status = wc.grab_json_response_with_status('/api/dataset', statargs)
-                print(repr(netset))
-                print(repr(status))
 
         elif self.ipaddr.startswith('fe80::'):
             self.configmanager.set_node_attributes(

confluent_server/confluent/discovery/protocols/pxe.py

@@ -22,6 +22,8 @@
 
 # option 97 = UUID (wireformat)
 
+import base64
+import confluent.config.conf as inifile
 import confluent.config.configmanager as cfm
 import confluent.collective.manager as collective
 import confluent.noderange as noderange
@@ -35,6 +37,7 @@ import eventlet
 import eventlet.green.socket as socket
 import eventlet.green.select as select
 import netifaces
+import os
 import struct
 import time
 import traceback
@@ -165,6 +168,18 @@ pxearchs = {
 }
 
 
+shorturls = {}
+def register_shorturl(url):
+    urlid = base64.urlsafe_b64encode(os.urandom(3))
+    while urlid in shorturls:
+        urlid = base64.urlsafe_b64encode(os.urandom(3))
+    urlid = urlid.decode()
+    shorturls[urlid] = url
+    returl = '/'.join(url.split('/')[:3])
+    returl += '/confluent-api/boot/su/' + urlid + '/' + os.path.basename(url)
+    return returl
+
+
 uuidmap = {}
 macmap = {}
 attribwatcher = None
@@ -369,12 +384,15 @@ def proxydhcp(handler, nodeguess):
             elif disco['arch'] == 'uefi-aarch64':
                 bootfile = b'confluent/aarch64/ipxe.efi'
             if len(bootfile) > 127:
-                log.log(
-                    {'info': 'Boot offer cannot be made to {0} as the '
-                    'profile name "{1}" is {2} characters longer than is supported '
-                    'for this boot method.'.format(
-                        node, profile, len(bootfile) - 127)})
-                continue
+                if bootfile.startswith(b'http'):
+                    bootfile = register_shorturl(bootfile.decode('utf8')).encode('utf8')
+                else:
+                    log.log(
+                        {'info': 'Boot offer cannot be made to {0} as the '
+                        'profile name "{1}" is {2} characters longer than is supported '
+                        'for this boot method.'.format(
+                            node, profile, len(bootfile) - 127)})
+                    continue
             rpv[:240] = rqv[:240].tobytes()
             rpv[0:1] = b'\x02'
             rpv[108:108 + len(bootfile)] = bootfile
@@ -393,12 +411,18 @@ def proxydhcp(handler, nodeguess):
 def start_proxydhcp(handler, nodeguess=None):
     eventlet.spawn_n(proxydhcp, handler, nodeguess)
 
-
+ignorenics = None
 def snoop(handler, protocol=None, nodeguess=None):
+    global ignorenics
     #TODO(jjohnson2): ipv6 socket and multicast for DHCPv6, should that be
     #prominent
     #TODO(jjohnson2): enable unicast replies. This would suggest either
     # injection into the neigh table before OFFER or using SOCK_RAW.
+    ignorenics = inifile.get_option('netboot', 'ignorenics')
+    if ignorenics:
+        if not isinstance(ignorenics, bytes):
+            ignorenics = ignorenics.encode()
+        ignorenics = ignorenics.split(b',')
     start_proxydhcp(handler, nodeguess)
     tracelog = log.Logger('trace')
     global attribwatcher
@@ -442,7 +466,10 @@ def snoop(handler, protocol=None, nodeguess=None):
         try:
             # Just need some delay, picked a prime number so that overlap with other
             # timers might be reduced, though it really is probably nothing
-            ready = select.select([net4, net6], [], [], None)
+            ready = select.select([net4, net6], [], [], 1)
+            for txid in list(_recent_txids):
+                if _recent_txids[txid] < time.time():
+                    del _recent_txids[txid]
             if not ready or not ready[0]:
                 continue
             for netc in ready[0]:
@@ -458,6 +485,14 @@ def snoop(handler, protocol=None, nodeguess=None):
                         _, level, typ = struct.unpack('QII', cmsgarr[:16])
                         if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
                             idx, recv = struct.unpack('II', cmsgarr[16:24])
+                            if ignorenics:
+                                ignore = False
+                                for nic in ignorenics:
+                                    if libc.if_nametoindex(nic) == idx:
+                                        ignore = True
+                                        break # ignore DHCP from ignored NIC
+                                if ignore:
+                                    continue
                             recv = ipfromint(recv)
                         rqv = memoryview(rawbuffer)[:i]
                         client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
@@ -613,6 +648,7 @@ def check_reply(node, info, packet, sock, cfg, reqview, addr, requestor):
         requestor = ('0.0.0.0', None)
     if requestor[0] == '0.0.0.0' and not info.get('uuid', None):
         return  # ignore DHCP from local non-PXE segment
+
     httpboot = info.get('architecture', None) == 'uefi-httpboot'
     cfd = cfg.get_node_attributes(node, ('deployment.*', 'collective.managercandidates'))
     profile, stgprofile = get_deployment_profile(node, cfg, cfd)
@@ -715,6 +751,7 @@ def get_my_duid():
         _myuuid = uuid.uuid4().bytes
     return _myuuid
 
+_recent_txids = {}
 
 def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=None, requestor=None):
     replen = 275  # default is going to be 286
@@ -749,6 +786,7 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N
     repview = repview[28:]
     repview[0:1] = b'\x02'
     repview[1:10] = reqview[1:10] # duplicate txid, hwlen, and others
+    thistxid = bytes(repview[4:8])
     repview[10:11] = b'\x80'  # always set broadcast
     repview[28:44] = reqview[28:44]  # copy chaddr field
     relayip = reqview[24:28].tobytes()
@@ -766,7 +804,7 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N
         log.log({'error': nicerr})
     if niccfg.get('ipv4_broken', False):
         # Received a request over a nic with no ipv4 configured, ignore it
-        log.log({'error': 'Skipping boot reply to {0} due to no viable IPv4 configuration on deployment system'.format(node)})
+        log.log({'error': 'Skipping boot reply to {0} due to no viable IPv4 configuration on deployment system on interface index "{}"'.format(node, info['netinfo']['ifidx'])})
         return
     clipn = None
     if niccfg['ipv4_method'] == 'firmwarenone':
@@ -797,12 +835,15 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N
         if not isinstance(bootfile, bytes):
             bootfile = bootfile.encode('utf8')
         if len(bootfile) > 127:
-            log.log(
-                {'info': 'Boot offer cannot be made to {0} as the '
-                'profile name "{1}" is {2} characters longer than is supported '
-                'for this boot method.'.format(
-                    node, profile, len(bootfile) - 127)})
-            return
+                if bootfile.startswith(b'http'):
+                    bootfile = register_shorturl(bootfile.decode('utf8')).encode('utf8')
+                else:
+                    log.log(
+                        {'info': 'Boot offer cannot be made to {0} as the '
+                        'profile name "{1}" is {2} characters longer than is supported '
+                        'for this boot method.'.format(
+                            node, profile, len(bootfile) - 127)})
+                    return
         repview[108:108 + len(bootfile)] = bootfile
     elif info.get('architecture', None) == 'uefi-aarch64' and packet.get(77, None) == b'iPXE':
         if not profile:
@@ -883,12 +924,26 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N
         boottype = 'HTTP'
     else:
         boottype = 'PXE'
+    deferanswer = None
     if clipn:
+        _recent_txids[thistxid] = time.time() + 1
         ipinfo = 'with static address {0}'.format(niccfg['ipv4_address'])
     else:
+        # use txid to track
+        # defer sending for a second if otherwise unserved...
+        deferanswer = thistxid
         ipinfo = 'without address, served from {0}'.format(myip)
     if relayipa:
         ipinfo += ' (relayed to {} via {})'.format(relayipa, requestor[0])
+    eventlet.spawn(send_rsp, repview, replen, requestor, relayip, reqview, info, deferanswer, isboot, node, boottype, ipinfo, sock)
+
+
+def send_rsp(repview, replen, requestor, relayip, reqview, info, defertxid, isboot, node, boottype, ipinfo, sock):
+    if defertxid:
+        eventlet.sleep(0.5)
+        if defertxid in _recent_txids:
+            log.log({'info': 'Skipping reply for {} over interface {} due to better offer being made over other interface'.format(node, info['netinfo']['ifidx'])})
+            return
     if isboot:
         log.log({
             'info': 'Offering {0} boot {1} to {2}'.format(boottype, ipinfo, node)})

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -58,7 +58,7 @@ smsg = ('M-SEARCH * HTTP/1.1\r\n'
 
 def active_scan(handler, protocol=None):
     known_peers = set([])
-    for scanned in scan(['urn:dmtf-org:service:redfish-rest:1', 'urn::service:affluent']):
+    for scanned in scan(['urn:dmtf-org:service:redfish-rest:1', 'urn::dmtf-org:service:redfish-rest:', 'urn::service:affluent']):
         for addr in scanned['addresses']:
             addr = addr[0:1] + addr[2:]
             if addr in known_peers:
@@ -297,7 +297,10 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                                             continue
                                         if not isinstance(reply, bytes):
                                             reply = reply.encode('utf8')
-                                        s.sendto(reply, peer)
+                                        try:
+                                            s.sendto(reply, peer)
+                                        except Exception:
+                                            pass
                                         break
                 r = select.select((net4, net6), (), (), 0.2)
                 if r:
@@ -429,10 +432,10 @@ def _find_service(service, target):
                     mya['enclosure-machinetype-model'] = [val]
             yield peerdata[nid]
             continue
-        if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()):
-            continue
         if '/DeviceDescription.json' in peerdata[nid]['urls']:
             pooltargs.append(('/DeviceDescription.json', peerdata[nid], 'lenovo-xcc'))
+        elif '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()):
+            continue
         else:
             for targurl in peerdata[nid]['urls']:
                 if '/eth' in targurl and targurl.endswith('.xml'):
@@ -463,6 +466,12 @@ def check_fish(urldata, port=443, verifycallback=None):
         return None
     if url == '/DeviceDescription.json':
         if not peerinfo:
+            if data.get('services', None) == ['urn::dmtf-org:service:redfish-rest:']:
+                peerinfo = wc.grab_json_response('/redfish/v1/')
+                if peerinfo:
+                    data['services'] = ['lenovo-smm3']
+                    data['uuid'] = peerinfo['UUID'].lower()
+                    return data
             return None
         try:
             peerinfo = peerinfo[0]
@@ -479,6 +488,12 @@ def check_fish(urldata, port=443, verifycallback=None):
             data['services'] = ['lenovo-xcc'] if 'xcc-variant' not in peerinfo else ['lenovo-xcc' + peerinfo['xcc-variant']]
             return data
         except (IndexError, KeyError):
+            if 'type' in peerinfo and peerinfo['type'].lower() == 'lenovo-smm3':
+                del peerinfo['xcc-variant']
+                data['uuid'] = peerinfo['enclosure-uuid']
+                data['services'] = ['lenovo-smm3']
+                data['attributes'] = peerinfo
+                return data
             return None
             url = '/redfish/v1/'
             peerinfo = wc.grab_json_response('/redfish/v1/')

confluent_server/confluent/httpapi.py

@@ -30,6 +30,7 @@ import confluent.config.attributes as attribs
 import confluent.config.configmanager as configmanager
 import confluent.consoleserver as consoleserver
 import confluent.discovery.core as disco
+import confluent.discovery.protocols.pxe as pxe
 import confluent.forwarder as forwarder
 import confluent.exceptions as exc
 import confluent.log as log
@@ -45,6 +46,7 @@ import eventlet
 import eventlet.greenthread
 import greenlet
 import json
+import os
 import socket
 import sys
 import traceback
@@ -72,6 +74,20 @@ opmap = {
 }
 
 
+def get_user_for_session(sessionid, sessiontok):
+    if not isinstance(sessionid, str):
+        sessionid = sessionid.decode()
+    if not isinstance(sessiontok, str):
+        sessiontok = sessiontok.decode()
+    if not sessiontok or not sessionid:
+        raise Exception("invalid session id or token")
+    if sessiontok != httpsessions.get(sessionid, {}).get('csrftoken', None):
+        raise Exception("Invalid csrf token for session")
+    user = httpsessions[sessionid]['name']
+    if not isinstance(user, str):
+        user = user.decode()
+    return user
+
 def group_creation_resources():
     yield confluent.messages.Attributes(
         kv={'name': None}, desc="Name of the group").html() + '<br>'
@@ -81,7 +97,7 @@ def group_creation_resources():
     for attr in sorted(attribs.node):
         if attr == 'groups':
             continue
-        if attr.startswith("secret."):
+        if attr.startswith('secret.') or attr.startswith('custom.nodesecret.'):
             yield confluent.messages.CryptedAttributes(
                 kv={attr: None},
                 desc=attribs.node[attr]['description']).html() + '<br>\n'
@@ -100,7 +116,7 @@ def node_creation_resources():
     yield confluent.messages.Attributes(
         kv={'name': None}, desc="Name of the node").html() + '<br>'
     for attr in sorted(attribs.node):
-        if attr.startswith("secret."):
+        if attr.startswith('secret.') or attr.startswith('custom.nodesecret.'):
             yield confluent.messages.CryptedAttributes(
                 kv={attr: None},
                 desc=attribs.node[attr]['description']).html() + '<br>\n'
@@ -431,6 +447,8 @@ def websockify_data(data):
             data = data.decode('utf8')
         except UnicodeDecodeError:
             data = data.decode('cp437')
+        except AttributeError:  # already str
+            pass
         data = u' ' + data
     return data
 
@@ -509,7 +527,11 @@ def wsock_handler(ws):
                                     else:
                                         delimit = '/shell/sessions'
                                         shellsession = True
-                                    node = targ.split(delimit, 1)[0]
+                                    nodesess = targ.split(delimit, 1)
+                                    node = nodesess[0]
+                                    sessidx = None
+                                    if len(nodesess) == 2 and len(nodesess[1]) > 1:
+                                        sessidx = nodesess[1][1:]
                                     node = node.rsplit('/', 1)[-1]
                                     auditmsg = {'operation': 'start', 'target': targ,
                                                 'user': util.stringify(username)}
@@ -520,7 +542,7 @@ def wsock_handler(ws):
                                             node=node, configmanager=cfgmgr,
                                             username=username, skipreplay=skipreplay,
                                             datacallback=datacallback,
-                                            width=width, height=height)
+                                            width=width, height=height, sessionid=sessidx)
                                     else:
                                         consession = consoleserver.ConsoleSession(
                                             node=node, configmanager=cfgmgr,
@@ -606,6 +628,14 @@ def wsock_handler(ws):
 
 
 def resourcehandler(env, start_response):
+    if env['PATH_INFO'].startswith('/confluent-api'):
+        env['PATH_INFO'] = env['PATH_INFO'].replace('/confluent-api', '')
+    for hdr in env['headers_raw']:
+        if hdr[0].lower().startswith('confluent'):
+            hdrname = hdr[0].upper()
+            if '_' not in hdrname:
+                hdrname = hdrname.replace('CONFLUENT', 'CONFLUENT_')
+            env['HTTP_' + hdrname] = hdr[1]
     try:
         if 'HTTP_SEC_WEBSOCKET_VERSION' in env: 
             for rsp in wsock_handler(env, start_response):
@@ -620,6 +650,8 @@ def resourcehandler(env, start_response):
         yield '500 - ' + str(e)
         return
 
+
+
 def resourcehandler_backend(env, start_response):
     """Function to handle new wsgi requests
     """
@@ -628,7 +660,7 @@ def resourcehandler_backend(env, start_response):
                ('Pragma', 'no-cache'),
                ('X-Content-Type-Options', 'nosniff'),
                ('Content-Security-Policy', "default-src 'self'"),
-               ('X-XSS-Protection', '1; mode=block'), ('X-Frame-Options', 'deny'),
+               ('X-XySS-Protection', '1; mode=block'), ('X-Frame-Options', 'deny'),
                ('Strict-Transport-Security', 'max-age=86400'),
                ('X-Permitted-Cross-Domain-Policies', 'none')]
     reqbody = None
@@ -638,10 +670,29 @@ def resourcehandler_backend(env, start_response):
             yield res
         return
     reqpath = env.get('PATH_INFO', '')
+    if reqpath == '/httpapi_initialized':
+        if (len(configmanager.ConfigManager(None).list_usergroups()) > 0
+                or len(configmanager.ConfigManager(None).list_users()) > 0):
+            start_response('200 OK', headers)
+            yield ''
+            return
+        start_response('500 No authorized users', headers)
+        yield ''
+        return
     if reqpath.startswith('/boot/'):
         request = env['PATH_INFO'].split('/')
         if not request[0]:
             request = request[1:]
+        if request[1] == 'su':  # shorturl
+            targurl = pxe.shorturls.get(request[2], None)
+            if not targurl:
+                start_response('404 Not Found', headers)
+                yield ''
+                return
+            headers.append(('Location', targurl))
+            start_response('302 Found', headers)
+            yield ''
+            return
         if len(request) != 4:
             start_response('400 Bad Request', headers)
             yield ''
@@ -667,7 +718,7 @@ def resourcehandler_backend(env, start_response):
         start_response('302 Found', headers)
         yield ''
         return
-    if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:
+    if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0 and not '/staging' in env['PATH_INFO']:
         reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
         reqtype = env['CONTENT_TYPE']
     operation = opmap.get(env['REQUEST_METHOD'], None)
@@ -921,6 +972,70 @@ def resourcehandler_backend(env, start_response):
             start_response('200 OK', headers)
             yield rsp
             return
+    
+
+    elif (operation == 'create' and ('/firmware/updates/active' in env['PATH_INFO'])):    
+        url = env['PATH_INFO']
+        if 'application/json' in reqtype:
+            if not isinstance(reqbody, str):
+                reqbody = reqbody.decode('utf8')
+            pbody = json.loads(reqbody)
+            args = pbody['args']
+            file_directory = '/var/lib/confluent/client_assets/{}'.format(args.split('/')[-1])  
+            filepath = '{0}/{1}'.format(file_directory, os.listdir(file_directory)[0])     # TODO find a way to validate that the file is found and its the expected one
+            args_dict = {'filename': filepath}
+            noderrs = {}
+            nodeurls = {}
+            hdlr = pluginapi.handle_path(url, operation, cfgmgr, args_dict)
+            for res in hdlr:
+                if isinstance(res, confluent.messages.CreatedResource):
+                    watchurl = res.kvpairs['created']
+                    currnode = watchurl.split('/')[1]
+                    nodeurls[currnode] = '/' + watchurl
+            yield json.dumps({'data': nodeurls})
+            start_response('200 OK', headers)
+            return
+    
+    elif (operation == 'create' and ('/staging' in env['PATH_INFO'])):
+        url = env['PATH_INFO']
+        args_dict = {}
+        content_length = int(env.get('CONTENT_LENGTH', 0))
+        if content_length > 0 and (len(url.split('/')) > 2):
+            # check if the user and the url defined user are the same 
+            if authorized['username'] == url.split('/')[2]:
+                args_dict.update({'filedata':env, 'content_length': content_length})
+                hdlr = pluginapi.handle_path(url, operation, cfgmgr, args_dict)
+                for resp in hdlr:
+                    if isinstance(resp, confluent.messages.FileUploadProgress):
+                        if resp.kvpairs['progress']['value'] == 100:
+                            progress = resp.kvpairs['progress']['value']
+                start_response('200 OK', headers)
+                yield json.dumps({'data': 'done'})
+                return       
+            else:
+                start_response('401 Unauthorized', headers)
+                yield json.dumps({'data': 'You do not have permission to write to file'})
+                return 
+        elif len(url.split('/')) == 2:
+            reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
+            reqtype = env['CONTENT_TYPE']
+            if not isinstance(reqbody, str):
+                reqbody = reqbody.decode('utf8')
+            pbody = json.loads(reqbody)
+            args = pbody['args']
+            args_dict.update({'filename': args, 'user': authorized['username']})
+            try:
+                args_dict.update({'bank': pbody['bank']})
+            except KeyError:
+                pass
+            hdlr = pluginapi.handle_path(url, operation, cfgmgr, args_dict)    
+            for res in hdlr:
+                if isinstance(res, confluent.messages.CreatedResource):
+                    stageurl = res.kvpairs['created']
+            start_response('200 OK', headers)
+            yield json.dumps({'data': stageurl})
+            return
+
     else:
         # normal request
         url = env['PATH_INFO']
@@ -1102,8 +1217,17 @@ def serve(bind_host, bind_port):
     sock = None
     while not sock:
         try:
-            sock = eventlet.listen(
-                (bind_host, bind_port, 0, 0), family=socket.AF_INET6)
+            if '/' in bind_host:
+                try:
+                    os.remove(bind_host)
+                except Exception:
+                    pass
+                sock = eventlet.listen(
+                    bind_host, family=socket.AF_UNIX)
+                os.chmod(bind_host, 0o666)
+            else:
+                sock = eventlet.listen(
+                    (bind_host, bind_port, 0, 0), family=socket.AF_INET6)
         except socket.error as e:
             if e.errno != 98:
                 raise

confluent_server/confluent/messages.py

@@ -18,6 +18,7 @@
 # This module implements client/server messages emitted from plugins.
 # Things are defined here to 'encourage' developers to coordinate information
 # format.  This is also how different data formats are supported
+import base64
 import confluent.exceptions as exc
 import confluent.config.configmanager as cfm
 import confluent.config.conf as cfgfile
@@ -86,7 +87,13 @@ def _htmlify_structure(indict):
 
 
 def msg_deserialize(packed):
-    m = msgpack.unpackb(packed, raw=False)
+    try:
+        m = msgpack.unpackb(packed, raw=False)
+    except UnicodeDecodeError:  # binary data, likely imagedata
+        # strings will be made binary, so binary messages
+        # must tolerate either string or bytes
+        m = msgpack.unpackb(packed)
+        m[0] = m[0].decode()
     cls = globals()[m[0]]
     if issubclass(cls, ConfluentMessage) or issubclass(cls, ConfluentNodeError):
         return cls(*m[1:])
@@ -262,10 +269,10 @@ class Generic(ConfluentMessage):
 
     def json(self):
         return json.dumps(self.data)
-    
+
     def raw(self):
         return self.data
-    
+
     def html(self):
         return json.dumps(self.data)
 
@@ -344,10 +351,10 @@ class ConfluentResourceCount(ConfluentMessage):
         self.myargs = [count]
         self.desc = 'Resource Count'
         self.kvpairs = {'count': count}
-    
+
     def strip_node(self, node):
         pass
-    
+
 class CreatedResource(ConfluentMessage):
     notnode = True
     readonly = True
@@ -569,6 +576,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputLicense(path, nodes, inputdata, configmanager)
     elif path == ['deployment', 'ident_image']:
         return InputIdentImage(path, nodes, inputdata)
+    elif path == ['console', 'ikvm']:
+        return InputIkvmParams(path, nodes, inputdata)
     elif inputdata:
         raise exc.InvalidArgumentException(
             'No known input handler for request')
@@ -638,6 +647,18 @@ class SavedFile(ConfluentMessage):
         self.myargs = (node, file)
         self.kvpairs = {node: {'filename': file}}
 
+class FileUploadProgress(ConfluentMessage):
+    readonly = True
+
+    def __init__(self, progress, name=None):
+        self.myargs = (progress)
+        self.stripped = False
+        self.notnode = name is None
+        if self.notnode:
+            self.kvpairs = {'progress': {'value': progress}}
+        else:
+            self.kvpairs = {name: {'progress': {'value': progress}}}
+
 class InputAlertData(ConfluentMessage):
 
     def __init__(self, path, inputdata, nodes=None):
@@ -936,6 +957,9 @@ class InputIdentImage(ConfluentInputMessage):
     keyname = 'ident_image'
     valid_values = ['create']
 
+class InputIkvmParams(ConfluentInputMessage):
+    keyname = 'method'
+    valid_values = ['unix', 'wss', 'url']
 
 class InputIdentifyMessage(ConfluentInputMessage):
     valid_values = set([
@@ -1043,7 +1067,7 @@ class InputReseatMessage(ConfluentInputMessage):
     keyname = 'reseat'
 
     def is_valid_key(self, key):
-        return key in self.valid_values or isinstance(key, int)
+        return key in self.valid_values or isinstance(key, int) or len(key) < 4
 
 
 class InputBMCReset(ConfluentInputMessage):
@@ -1865,6 +1889,18 @@ class GraphicalConsole(ConfluentMessage):
         else:
             self.kvpairs = {name: {'Launcher': kv}}
 
+class ScreenShot(ConfluentMessage):
+    readonly = True
+
+    def __init__(self, imgdata, node, imgformat=None):
+        if isinstance(node, bytes):
+            node = node.decode()
+        if isinstance(imgformat, bytes):
+            imgformat = imgformat.decode()
+        self.myargs = (imgdata, node, imgformat)
+        self.kvpairs = {node: {'image': {'imgformat': imgformat, 'imgdata': base64.b64encode(imgdata).decode()}}}
+
+
 class CryptedAttributes(Attributes):
     defaulttype = 'password'
 

confluent_server/confluent/netutil.py

@@ -317,6 +317,20 @@ def add_netmask(ncfg):
 def get_full_net_config(configmanager, node, serverip=None):
     cfd = configmanager.get_node_attributes(node, ['net.*'])
     cfd = cfd.get(node, {})
+    bmc = configmanager.get_node_attributes(
+        node, 'hardwaremanagement.manager').get(node, {}).get(
+            'hardwaremanagement.manager', {}).get('value', None)
+    bmc4 = None
+    bmc6 = None
+    if bmc:
+        try:
+            bmc4 = socket.getaddrinfo(bmc, 0, socket.AF_INET, socket.SOCK_DGRAM)[0][-1][0]
+        except Exception:
+            pass
+        try:
+            bmc6 = socket.getaddrinfo(bmc, 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][0]
+        except Exception:
+            pass
     attribs = {}
     for attrib in cfd:
         val = cfd[attrib].get('value', None)
@@ -346,6 +360,12 @@ def get_full_net_config(configmanager, node, serverip=None):
     for netname in sorted(attribs):
         ppool.spawn(nm.process_attribs, netname, attribs[netname])
     ppool.waitall()
+    for iface in list(nm.myattribs):
+        if bmc4 and nm.myattribs[iface].get('ipv4_address', None) == bmc4:
+            del nm.myattribs[iface]
+            continue
+        if bmc6 and nm.myattribs[iface].get('ipv6_address', None) == bmc6:
+            del nm.myattribs[iface]
     retattrs = {}
     if None in nm.myattribs:
         retattrs['default'] = nm.myattribs[None]
@@ -454,6 +474,19 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None,
             clientipn = socket.inet_pton(clientfam, clientip)
     nodenetattribs = configmanager.get_node_attributes(
         node, 'net*').get(node, {})
+    bmc = configmanager.get_node_attributes(
+        node, 'hardwaremanagement.manager').get(node, {}).get('hardwaremanagement.manager', {}).get('value', None)
+    bmc4 = None
+    bmc6 = None
+    if bmc:
+        try:
+            bmc4 = socket.getaddrinfo(bmc, 0, socket.AF_INET, socket.SOCK_DGRAM)[0][-1][0]
+        except Exception:
+            pass
+        try:
+            bmc6 = socket.getaddrinfo(bmc, 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][0]
+        except Exception:
+            pass
     cfgbyname = {}
     for attrib in nodenetattribs:
         segs = attrib.split('.')
@@ -554,6 +587,10 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None,
                         continue
                 candgw = cfgbyname[candidate].get('ipv{}_gateway'.format(nver), None)
                 if candip:
+                    if bmc4 and candip == bmc4:
+                        continue
+                    if bmc6 and candip == bmc6:
+                        continue
                     try:
                         for inf in socket.getaddrinfo(candip, 0, fam, socket.SOCK_STREAM):
                             candipn = socket.inet_pton(fam, inf[-1][0])